autoPatchelfHook: improve arch/ABI compatibility

Fully enabling crossSystem support for autoPatchelfHook came with some
perhaps unintended consequences of being a bit more aggressive about
patching ELF files from architectures/ABIs that differ from the target
(previously, those files would be ignored because ldd usually couldn't
handle them).

This change adds architecture and rough OS ABI detection to the script
so that it doesn't try to blindly replace the interpreter of files that
can't possibly use that interpreter, and also makes sure it doesn't
accidentally use libraries of other architectures/ABIs.

1 files changed, 78 insertions, 10 deletions

diff --git a/pkgs/build-support/setup-hooks/auto-patchelf.sh b/pkgs/build-support/setup-hooks/auto-patchelf.sh
index d310f8255224f..7f95dc8043a21 100644
--- a/pkgs/build-support/setup-hooks/auto-patchelf.sh
+++ b/pkgs/build-support/setup-hooks/auto-patchelf.sh
@@ -63,10 +63,9 @@ getRpathFromElfBinary() {
     # NOTE: This does not use runPatchelf because it may encounter non-ELF
     # files. Caller is expected to check the return code if needed.
     local rpath
-    rpath="$(patchelf --print-rpath "$1" 2> /dev/null)" || return $?
+    IFS=':' read -ra rpath < <(patchelf --print-rpath "$1" 2> /dev/null) || return $?
 
-    local IFS=':'
-    printf "%s\n" $rpath
+    printf "%s\n" "${rpath[@]}"
 }
 
 populateCacheForDep() {
@@ -115,8 +114,52 @@ populateCacheWithRecursiveDeps() {
     done
 }
 
-getSoArch() {
-    $OBJDUMP -f "$1" | sed -ne 's/^architecture: *\([^,]\+\).*/\1/p'
+getBinArch() {
+    $OBJDUMP -f "$1" 2> /dev/null | sed -ne 's/^architecture: *\([^,]\+\).*/\1/p'
+}
+
+# Returns the specific OS ABI for an ELF file in the format produced by
+# readelf(1), like "UNIX - System V" or "UNIX - GNU".
+getBinOsabi() {
+    $READELF -h "$1" 2> /dev/null | sed -ne 's/^[ \t]*OS\/ABI:[ \t]*\(.*\)/\1/p'
+}
+
+# Tests whether two OS ABIs are compatible, taking into account the generally
+# accepted compatibility of SVR4 ABI with other ABIs.
+areBinOsabisCompatible() {
+    local wanted="$1"
+    local got="$2"
+
+    if [[ -z "$wanted" || -z "$got" ]]; then
+        # One of the types couldn't be detected, so as a fallback we'll assume
+        # they're compatible.
+        return 0
+    fi
+
+    # Generally speaking, the base ABI (0x00), which is represented by
+    # readelf(1) as "UNIX - System V", indicates broad compatibility with other
+    # ABIs.
+    #
+    # TODO: This isn't always true. For example, some OSes embed ABI
+    # compatibility into SHT_NOTE sections like .note.tag and .note.ABI-tag.
+    # It would be prudent to add these to the detection logic to produce better
+    # ABI information.
+    if [[ "$wanted" == "UNIX - System V" ]]; then
+        return 0
+    fi
+
+    # Similarly here, we should be able to link against a superset of features,
+    # so even if the target has another ABI, this should be fine.
+    if [[ "$got" == "UNIX - System V" ]]; then
+        return 0
+    fi
+
+    # Otherwise, we simply return whether the ABIs are identical.
+    if [[ "$wanted" == "$got" ]]; then
+        return 0
+    fi
+
+    return 1
 }
 
 # NOTE: If you want to use this function outside of the autoPatchelf function,
@@ -127,6 +170,7 @@ getSoArch() {
 findDependency() {
     local filename="$1"
     local arch="$2"
+    local osabi="$3"
     local lib dep
 
     if [ $depCacheInitialised -eq 0 ]; then
@@ -138,7 +182,7 @@ findDependency() {
 
     for dep in "${autoPatchelfCachedDeps[@]}"; do
         if [ "$filename" = "${dep##*/}" ]; then
-            if [ "$(getSoArch "$dep")" = "$arch" ]; then
+            if [ "$(getBinArch "$dep")" = "$arch" ] && areBinOsabisCompatible "$osabi" "$(getBinOsabi "$dep")"; then
                 foundDependency="$dep"
                 return 0
             fi
@@ -163,6 +207,23 @@ autoPatchelfFile() {
 
     local interpreter
     interpreter="$(< "$NIX_CC/nix-support/dynamic-linker")"
+
+    local interpreterArch interpreterOsabi toPatchArch toPatchOsabi
+    interpreterArch="$(getBinArch "$interpreter")"
+    interpreterOsabi="$(getBinOsabi "$interpreter")"
+    toPatchArch="$(getBinArch "$toPatch")"
+    toPatchOsabi="$(getBinOsabi "$toPatch")"
+
+    if [ "$interpreterArch" != "$toPatchArch" ]; then
+        # Our target architecture is different than this file's architecture,
+        # so skip it.
+        echo "skipping $toPatch because its architecture ($toPatchArch) differs from target ($interpreterArch)" >&2
+        return 0
+    elif ! areBinOsabisCompatible "$interpreterOsabi" "$toPatchOsabi"; then
+        echo "skipping $toPatch because its OS ABI ($toPatchOsabi) is not compatible with target ($interpreterOsabi)" >&2
+        return 0
+    fi
+
     if isExecutable "$toPatch"; then
         runPatchelf --set-interpreter "$interpreter" "$toPatch"
         # shellcheck disable=SC2154
@@ -187,14 +248,21 @@ autoPatchelfFile() {
     # new package where you don't yet know its dependencies.
 
     for dep in $missing; do
-        # Check whether this library exists in libc. If so, we don't need to do
-        # any futher searching -- it will be resolved correctly by the linker.
-        if [ -f "$libcLib/$dep" ]; then
+        if [[ "$dep" == /* ]]; then
+            # This is an absolute path. If it exists, just use it. Otherwise,
+            # we probably want this to produce an error when checked (because
+            # just updating the rpath won't satisfy it).
+            if [ -f "$dep" ]; then
+                continue
+            fi
+        elif [ -f "$libcLib/$dep" ]; then
+            # This library exists in libc, and will be correctly resolved by
+            # the linker.
             continue
         fi
 
         echo -n "  $dep -> " >&2
-        if findDependency "$dep" "$(getSoArch "$toPatch")"; then
+        if findDependency "$dep" "$toPatchArch" "$toPatchOsabi"; then
             rpath="$rpath${rpath:+:}${foundDependency%/*}"
             echo "found: $foundDependency" >&2
         else