games: Move FMOD fixing from Bastion to setup hook

Thanks to @layus for pointing out that there is at least one other game
(Epistory) which has the same FMOD issue as Bastion has.

So I decided to move this into a setup hook that automatically discovers
whether it's the affected FMOD version and NOPs out the calls to
system().

In summary: If another game is affected, all that's needed now is to add
fixFmodHook to nativeBuildInputs.

Signed-off-by: aszlig <aszlig@nix.build>
Cc: @layus

4 files changed, 76 insertions, 50 deletions

diff --git a/pkgs/games/build-support/default.nix b/pkgs/games/build-support/default.nix
index 0e7574ec..ac152659 100644
--- a/pkgs/games/build-support/default.nix
+++ b/pkgs/games/build-support/default.nix
@@ -7,5 +7,5 @@
   buildUnity = callPackage ./build-unity.nix {};
   monogamePatcher = callPackage ./monogame-patcher {};
 
-  inherit (callPackages ./setup-hooks {}) gogUnpackHook;
+  inherit (callPackages ./setup-hooks {}) fixFmodHook gogUnpackHook;
 }
diff --git a/pkgs/games/build-support/setup-hooks/default.nix b/pkgs/games/build-support/setup-hooks/default.nix
index 9f35dad7..a125f644 100644
--- a/pkgs/games/build-support/setup-hooks/default.nix
+++ b/pkgs/games/build-support/setup-hooks/default.nix
@@ -1,6 +1,10 @@
 { makeSetupHook, libarchive, innoextract }:
 
 {
+  fixFmodHook = makeSetupHook {
+    deps = [];
+  } ./fix-fmod.sh;
+
   gogUnpackHook = makeSetupHook {
     deps = [ libarchive innoextract ];
   } ./gog-unpack.sh;
diff --git a/pkgs/games/build-support/setup-hooks/fix-fmod.sh b/pkgs/games/build-support/setup-hooks/fix-fmod.sh
new file mode 100644
index 00000000..bc490a64
--- /dev/null
+++ b/pkgs/games/build-support/setup-hooks/fix-fmod.sh
@@ -0,0 +1,68 @@
+# FMOD tries to run /bin/sh -c 'pulseaudio --check > /dev/null 2>&1', so we
+# need to prevent this by replacing the system() call with a successful return
+# value (0). If someone doesn't have or want to have PulseAudio, FMOD still
+# falls back to ALSA if it can't load libpulse-simple.so.
+_doPatchFmod() {
+    set -e
+    set -o pipefail
+
+    # Let's make sure it's really the affected FMOD version:
+    objdump -T "$1" 2> /dev/null | grep -q "\\<FMOD_System_Init\\>"
+    grep -qF "pulseaudio --check" "$1"
+
+    case "$(objdump -f "$1" | sed -n -e 's/^architecture: *//p')" in
+        i386:x86-64,*)
+            local addr="$(objdump -d "$1" | sed -n -e '
+                /callq.*system@plt/s/^ *\([^:]\+\).*/\1/p
+            ')"
+
+            # This is quite easy, just replace the system() call with XOR EAX
+            # so we get a return value of 0 and pad the rest with NOP.
+            local offset=$(("0x$addr"))
+            ( printf '\x31\xc0'     # XOR the EAX register
+              printf '\x90\x90\x90' # Fill with NOPs
+            ) | dd of="$1" obs=1 seek=$offset conv=notrunc status=none
+            ;;
+        i386,*)
+            local relocSystem="$(readelf -r "$1" | sed -n -e '
+                /system@/s/^0*\([^ ]\+\).*/\1/p
+            ')"
+            local addr="$(objdump -d "$1" | sed -n -e '
+                /call *'"$relocSystem"' /s/^ *\([^:]\+\).*/\1/p
+            ')"
+
+            # For the 32 bit library it's not so easy as the 4 bytes coming
+            # after the CALL opcode will be replaced by the dynamic linker, so
+            # we just XOR the EAX register with the relocation address and
+            # replace the TEST opcode afterwards.
+            local offset=$(("0x$addr"))
+            ( printf '\x35\xfc\xff\xff\xff' # XOR EAX with the relocation addr
+              printf '\x39\xc0'             # CMP EAX, EAX
+            ) | dd of="$1" obs=1 seek=$offset conv=notrunc status=none
+            ;;
+        *) return ;;
+    esac
+
+    # Only needed if the library is used with dlopen().
+    if [ -n "$runtimeDependencies" ]; then
+        local dep rpath="$(patchelf --print-rpath "$1")"
+        for dep in $runtimeDependencies; do
+          rpath="$rpath''${rpath:+:}$dep/lib"
+        done
+        patchelf --set-rpath "$rpath" "$1"
+    fi
+
+    echo "$1: removed call to system()" >&2
+}
+
+patchFmod() {
+    export -f _doPatchFmod
+    echo "patching out system() calls in FMOD" >&2
+    find "$prefix" \( -iname '*fmod*.so' -o -iname '*fmod*.so.*' \) \
+        -exec "$SHELL" -c '_doPatchFmod "$1"' -- {} \;
+}
+
+# Needs to come after the setup hook of patchelf.
+postFixupHooks+=(
+    'for output in $outputs; do prefix="${!output}" patchFmod; done'
+)
diff --git a/pkgs/games/humblebundle/bastion.nix b/pkgs/games/humblebundle/bastion.nix
index 3a4af115..207dd105 100644
--- a/pkgs/games/humblebundle/bastion.nix
+++ b/pkgs/games/humblebundle/bastion.nix
@@ -1,5 +1,5 @@
-{ lib, stdenv, buildGame, fetchHumbleBundle, makeWrapper, unzip, imagemagick
-, mono, SDL2, SDL2_image, openal, libvorbis, libGL
+{ lib, stdenv, buildGame, fixFmodHook, fetchHumbleBundle, makeWrapper
+, unzip, imagemagick, mono, SDL2, SDL2_image, openal, libvorbis, libGL
 }:
 
 buildGame rec {
@@ -15,7 +15,7 @@ buildGame rec {
 
   unpackCmd = "${unzip}/bin/unzip -qq \"$curSrc\" 'data/*' || :";
 
-  nativeBuildInputs = [ makeWrapper imagemagick ];
+  nativeBuildInputs = [ makeWrapper imagemagick fixFmodHook ];
 
   libDir =
     if stdenv.system == "i686-linux" then "lib"
@@ -58,39 +58,6 @@ buildGame rec {
     patchelf --remove-needed libsteam_api.so "$libDir/libSteamWrapper.so"
     patchelf --add-needed libSteamStub.so "$libDir/libSteamWrapper.so"
 
-    # FMOD Ex tries to run /bin/sh -c 'pulseaudio --check > /dev/null 2>&1', so
-    # we need to prevent this by replacing the system() call with a successful
-    # return value (0). If someone doesn't have or want to have PulseAudio,
-    # FMOD Ex still falls back to ALSA if it can't load libpulse-simple.so.
-    if [ "$libDir" = lib64 ]; then
-      addr="$(objdump -d lib64/libfmodex.so | sed -n -e '
-        /callq.*system@plt/s/^ *\([^:]\+\).*/\1/p
-      ')"
-
-      # This is quite easy, just replace the system() call with XOR EAX so we
-      # get a return value of 0 and pad the rest with NOP.
-      offset=$(("0x$addr"))
-      ( printf '\x31\xc0'     # XOR the EAX register
-        printf '\x90\x90\x90' # Fill with NOPs
-      ) | dd of="$libDir/libfmodex.so" obs=1 seek=$offset count=5 conv=notrunc
-    else
-      relocSystem="$(readelf -r lib/libfmodex.so | sed -n -e '
-        /system@/s/^0*\([^ ]\+\).*/\1/p
-      ')"
-      addr="$(objdump -d lib/libfmodex.so | sed -n -e '
-        /call *'"$relocSystem"' /s/^ *\([^:]\+\).*/\1/p
-      ')"
-
-      # For the 32 bit library it's not so easy as the 4 bytes coming after the
-      # CALL opcode will be replaced by the dynamic linker, so we just XOR the
-      # EAX register with the relocation address and replace the TEST opcode
-      # afterwards.
-      offset=$(("0x$addr"))
-      ( printf '\x35\xfc\xff\xff\xff' # XOR EAX with the relocation address
-        printf '\x39\xc0'             # CMP EAX, EAX
-      ) | dd of="$libDir/libfmodex.so" obs=1 seek=$offset conv=notrunc
-    fi
-
     # For the Desktop icon
     convert Bastion.bmp Bastion.png
   '';
@@ -127,18 +94,5 @@ buildGame rec {
     EOF
   '';
 
-  # Add all of the libraries in $runtimeDependencies to the FMOD Ex library,
-  # because it tries to dlopen() libpulse-simple and libasound.so and we don't
-  # have a main ELF binary where we could add that search path.
-  postPhases = [ "fixFModex" ];
-  fixFModex = ''
-    rpath="$(patchelf --print-rpath "$out/libexec/bastion/libfmodex.so")"
-    for dep in $runtimeDependencies; do
-      rpath="$rpath''${rpath:+:}$dep/lib"
-    done
-    echo "setting RPATH for libfmodex to: $rpath" >&2
-    patchelf --set-rpath "$rpath" "$out/libexec/bastion/libfmodex.so"
-  '';
-
   sandbox.paths.required = [ "$XDG_DATA_HOME/Bastion" ];
 }