games: Move FMOD fixing from Bastion to setup hook

Thanks to @layus for pointing out that there is at least one other game
(Epistory) which has the same FMOD issue as Bastion has.

So I decided to move this into a setup hook that automatically discovers
whether it's the affected FMOD version and NOPs out the calls to
system().

In summary: If another game is affected, all that's needed now is to add
fixFmodHook to nativeBuildInputs.

Signed-off-by: aszlig <aszlig@nix.build>
Cc: @layus

3 files changed, 73 insertions, 1 deletions

diff --git a/pkgs/games/build-support/default.nix b/pkgs/games/build-support/default.nix
index 0e7574ec..ac152659 100644
--- a/pkgs/games/build-support/default.nix
+++ b/pkgs/games/build-support/default.nix
@@ -7,5 +7,5 @@
   buildUnity = callPackage ./build-unity.nix {};
   monogamePatcher = callPackage ./monogame-patcher {};
 
-  inherit (callPackages ./setup-hooks {}) gogUnpackHook;
+  inherit (callPackages ./setup-hooks {}) fixFmodHook gogUnpackHook;
 }
diff --git a/pkgs/games/build-support/setup-hooks/default.nix b/pkgs/games/build-support/setup-hooks/default.nix
index 9f35dad7..a125f644 100644
--- a/pkgs/games/build-support/setup-hooks/default.nix
+++ b/pkgs/games/build-support/setup-hooks/default.nix
@@ -1,6 +1,10 @@
 { makeSetupHook, libarchive, innoextract }:
 
 {
+  fixFmodHook = makeSetupHook {
+    deps = [];
+  } ./fix-fmod.sh;
+
   gogUnpackHook = makeSetupHook {
     deps = [ libarchive innoextract ];
   } ./gog-unpack.sh;
diff --git a/pkgs/games/build-support/setup-hooks/fix-fmod.sh b/pkgs/games/build-support/setup-hooks/fix-fmod.sh
new file mode 100644
index 00000000..bc490a64
--- /dev/null
+++ b/pkgs/games/build-support/setup-hooks/fix-fmod.sh
@@ -0,0 +1,68 @@
+# FMOD tries to run /bin/sh -c 'pulseaudio --check > /dev/null 2>&1', so we
+# need to prevent this by replacing the system() call with a successful return
+# value (0). If someone doesn't have or want to have PulseAudio, FMOD still
+# falls back to ALSA if it can't load libpulse-simple.so.
+_doPatchFmod() {
+    set -e
+    set -o pipefail
+
+    # Let's make sure it's really the affected FMOD version:
+    objdump -T "$1" 2> /dev/null | grep -q "\\<FMOD_System_Init\\>"
+    grep -qF "pulseaudio --check" "$1"
+
+    case "$(objdump -f "$1" | sed -n -e 's/^architecture: *//p')" in
+        i386:x86-64,*)
+            local addr="$(objdump -d "$1" | sed -n -e '
+                /callq.*system@plt/s/^ *\([^:]\+\).*/\1/p
+            ')"
+
+            # This is quite easy, just replace the system() call with XOR EAX
+            # so we get a return value of 0 and pad the rest with NOP.
+            local offset=$(("0x$addr"))
+            ( printf '\x31\xc0'     # XOR the EAX register
+              printf '\x90\x90\x90' # Fill with NOPs
+            ) | dd of="$1" obs=1 seek=$offset conv=notrunc status=none
+            ;;
+        i386,*)
+            local relocSystem="$(readelf -r "$1" | sed -n -e '
+                /system@/s/^0*\([^ ]\+\).*/\1/p
+            ')"
+            local addr="$(objdump -d "$1" | sed -n -e '
+                /call *'"$relocSystem"' /s/^ *\([^:]\+\).*/\1/p
+            ')"
+
+            # For the 32 bit library it's not so easy as the 4 bytes coming
+            # after the CALL opcode will be replaced by the dynamic linker, so
+            # we just XOR the EAX register with the relocation address and
+            # replace the TEST opcode afterwards.
+            local offset=$(("0x$addr"))
+            ( printf '\x35\xfc\xff\xff\xff' # XOR EAX with the relocation addr
+              printf '\x39\xc0'             # CMP EAX, EAX
+            ) | dd of="$1" obs=1 seek=$offset conv=notrunc status=none
+            ;;
+        *) return ;;
+    esac
+
+    # Only needed if the library is used with dlopen().
+    if [ -n "$runtimeDependencies" ]; then
+        local dep rpath="$(patchelf --print-rpath "$1")"
+        for dep in $runtimeDependencies; do
+          rpath="$rpath''${rpath:+:}$dep/lib"
+        done
+        patchelf --set-rpath "$rpath" "$1"
+    fi
+
+    echo "$1: removed call to system()" >&2
+}
+
+patchFmod() {
+    export -f _doPatchFmod
+    echo "patching out system() calls in FMOD" >&2
+    find "$prefix" \( -iname '*fmod*.so' -o -iname '*fmod*.so.*' \) \
+        -exec "$SHELL" -c '_doPatchFmod "$1"' -- {} \;
+}
+
+# Needs to come after the setup hook of patchelf.
+postFixupHooks+=(
+    'for output in $outputs; do prefix="${!output}" patchFmod; done'
+)