stdenv.darwin.make-boostrap-tools: update tools and unpin LLVM11

- unpin LLVM11. fix discrepancy with freshBootstrapTools and the tools
  built on hydra. pinning the stdenv for the hydra build doesn't pin the
  tools as the included packages are able to change.

- remove unused LLVM tools & libs which reduces the uncompressed and
  compressed file sizes by more than 1/2. compressed tarball is now 40M
  and uncompressed is around 200M

- add @loader_path/. to dylibs that reference other libs in the archive.
  this is needed for libraries with re-exports.

- validate shared objects with @rpath references contain the reference
  in lib

- add a test to verify that the @loader_path/ works for libc++ as it
  re-exports libc++abi

2 files changed, 148 insertions, 108 deletions

diff --git a/pkgs/stdenv/darwin/make-bootstrap-tools.nix b/pkgs/stdenv/darwin/make-bootstrap-tools.nix
index d00e6d9392599..7725665ae4368 100644
--- a/pkgs/stdenv/darwin/make-bootstrap-tools.nix
+++ b/pkgs/stdenv/darwin/make-bootstrap-tools.nix
@@ -16,30 +16,10 @@ let cross = if crossSystem != null
 in with import pkgspath ({ inherit localSystem; } // cross // custom-bootstrap);
 
 rec {
-  coreutils_ = (coreutils.override (args: {
-    # We want coreutils without ACL support.
-    aclSupport = false;
-    # Cannot use a single binary build, or it gets dynamically linked against gmp.
-    singleBinary = false;
-  })).overrideAttrs (oa: {
-    # Increase header size to be able to inject extra RPATHs. Otherwise
-    # x86_64-darwin build fails as:
-    #    https://cache.nixos.org/log/g5wyq9xqshan6m3kl21bjn1z88hx48rh-stdenv-bootstrap-tools.drv
-    NIX_LDFLAGS = (oa.NIX_LDFLAGS or "") + " -headerpad_max_install_names";
-  });
-
-  cctools_ = darwin.cctools;
-
-  # Avoid debugging larger changes for now.
-  bzip2_ = bzip2.override (args: { enableStatic = true; enableShared = false; });
-
-  # Avoid messing with libkrb5 and libnghttp2.
-  curl_ = curlMinimal.override (args: { gssSupport = false; http2Support = false; });
-
   build = stdenv.mkDerivation {
     name = "stdenv-bootstrap-tools";
 
-    nativeBuildInputs = [ nukeReferences dumpnar ];
+    nativeBuildInputs = [ dumpnar nukeReferences ];
 
     buildCommand = let
       inherit (lib)
@@ -47,6 +27,35 @@ rec {
         getDev
         getLib
         ;
+
+      coreutils_ = (coreutils.override (args: {
+        # We want coreutils without ACL support.
+        aclSupport = false;
+        # Cannot use a single binary build, or it gets dynamically linked against gmp.
+        singleBinary = false;
+      })).overrideAttrs (oa: {
+        # Increase header size to be able to inject extra RPATHs. Otherwise
+        # x86_64-darwin build fails as:
+        #    https://cache.nixos.org/log/g5wyq9xqshan6m3kl21bjn1z88hx48rh-stdenv-bootstrap-tools.drv
+        NIX_LDFLAGS = (oa.NIX_LDFLAGS or "") + " -headerpad_max_install_names";
+      });
+
+      cctools_ = darwin.cctools;
+
+      # Avoid messing with libkrb5 and libnghttp2.
+      curl_ = curlMinimal.override (args: {
+        gssSupport = false;
+        http2Support = false;
+        scpSupport = false;
+      });
+
+      gnutar_ = (gnutar.override { libintl = null; }).overrideAttrs (old: {
+        configureFlags = [
+          "--disable-nls"
+        ] ++ old.configureFlags or [];
+      });
+
+      xz_ = xz.override { enableStatic = true; };
     in
     ''
       mkdir -p $out/bin $out/lib $out/lib/system $out/lib/darwin
@@ -61,7 +70,6 @@ rec {
 
       cp -rL ${getDev darwin.Libsystem}/include $out
       chmod -R u+w $out/include
-      cp -rL ${getDev darwin.ICU}/include* $out/include
       cp -rL ${getDev libiconv}/include/* $out/include
       cp -rL ${getDev gnugrep.pcre2}/include/* $out/include
       mv $out/include $out/include-Libsystem
@@ -81,7 +89,7 @@ rec {
       cp -d ${getBin gawk}/bin/awk $out/bin
       cp ${getBin gnutar}/bin/tar $out/bin
       cp ${getBin gzip}/bin/.gzip-wrapped $out/bin/gzip
-      cp ${getBin bzip2_}/bin/bzip2 $out/bin
+      cp ${getBin bzip2}/bin/bzip2 $out/bin
       ln -s bzip2 $out/bin/bunzip2
       cp -d ${getBin gnumake}/bin/* $out/bin
       cp -d ${getBin patch}/bin/* $out/bin
@@ -92,7 +100,6 @@ rec {
       # because I can't be bothered to make it partially static
       cp ${getBin curl_}/bin/curl $out/bin
       cp -d ${getLib curl_}/lib/libcurl*.dylib $out/lib
-      cp -d ${getLib libssh2}/lib/libssh*.dylib $out/lib
       cp -d ${getLib openssl}/lib/*.dylib $out/lib
 
       cp -d ${getLib gnugrep.pcre2}/lib/libpcre2*.dylib $out/lib
@@ -103,17 +110,14 @@ rec {
       cp -d ${getLib libxml2}/lib/libxml2*.dylib $out/lib
 
       # Copy what we need of clang
-      cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang* $out/bin
-      cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/* $out/lib
+      cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang{,++,-cl,-cpp,-[0-9]*} $out/bin
+      cp -d ${getLib llvmPackages.clang-unwrapped}/lib/libclang-cpp*.dylib $out/lib
+      cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/clang $out/lib
 
       cp -d ${getLib llvmPackages.libcxx}/lib/libc++*.dylib $out/lib
-    ''
-    # libc++abi is contained in libcxx for LLVM12+. Remove once unpinned from LLVM11
-    + lib.optionalString (llvmPackages ? libcxxabi) ''
-      cp -d ${getLib llvmPackages.libcxxabi}/lib/libc++abi*.dylib $out/lib
-    '' + ''
-      cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt* $out/lib/darwin
-      cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt* $out/lib
+      mkdir -p $out/lib/darwin
+      cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt.{,profile_}osx.a  $out/lib/darwin
+      cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt.{,profile_}osx.a $out/lib
       cp -d ${getLib llvmPackages.llvm}/lib/libLLVM.dylib $out/lib
       cp -d ${getLib libffi}/lib/libffi*.dylib $out/lib
 
@@ -133,7 +137,6 @@ rec {
       cp -d ${getBin pkgs.darwin.sigtool}/bin/sigtool $out/bin
       cp -d ${getBin pkgs.darwin.sigtool}/bin/codesign $out/bin
 
-      cp -d ${getLib darwin.ICU}/lib/libicu*.dylib $out/lib
       cp -d ${getLib zlib}/lib/libz.*       $out/lib
       cp -d ${getLib gmpxx}/lib/libgmp*.*   $out/lib
       cp -d ${getLib xz}/lib/liblzma*.*     $out/lib
@@ -145,63 +148,103 @@ rec {
 
       cp -d ${getLib darwin.libtapi}/lib/libtapi* $out/lib
 
-      cp -rd ${getLib pkgs.darwin.CF}/Library $out
-      ${lib.optionalString stdenv.targetPlatform.isAarch64 ''
-        cp -rd ${getLib pkgs.darwin.libobjc}/lib/* $out/lib/
-      ''}
+      # tools needed to unpack bootstrap archive. they should not contain any
+      # external references. we will process them like the other tools but
+      # perform some additional checks and will not pack them into the archive.
+      mkdir -p unpack/bin
+      cp ${getBin bash}/bin/bash unpack/bin
+      ln -s bash unpack/bin/sh
+      cp ${getBin coreutils_}/bin/mkdir unpack/bin
+      cp ${getBin gnutar_}/bin/tar unpack/bin
+      cp ${getBin xz_}/bin/xz unpack/bin
 
-      chmod -R u+w $out
+      #
+      # All files copied. Perform processing to update references to point into
+      # the archive
+      #
 
-      nuke-refs $out/bin/*
+      chmod -R u+w $out unpack
 
+      # - change nix store library paths to use @rpath/library
+      # - if needed add an rpath containing lib/
+      # - strip executable
       rpathify() {
-        local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*") || true
-        local newlib
+        local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*" || true)
+        local lib rpath
         for lib in $libs; do
           ${stdenv.cc.targetPrefix}install_name_tool -change $lib "@rpath/$(basename "$lib")" "$1"
         done
-      }
 
-      # Strip executables even further
-      for i in $out/bin/*; do
-        if [[ ! -L $i ]] && isMachO "$i"; then
-          chmod +w $i
-          ${stdenv.cc.targetPrefix}strip $i || true
+        case "$(dirname "$1")" in
+        */bin)
+          # Strip executables even further
+          ${stdenv.cc.targetPrefix}strip "$i"
+          rpath='@executable_path/../lib'
+          ;;
+        */lib)
+          # the '/.' suffix is required
+          rpath='@loader_path/.'
+          ;;
+        */lib/darwin)
+          rpath='@loader_path/..'
+          ;;
+        *)
+          echo unkown executable $1 >&2
+          exit 1
+          ;;
+        esac
+
+        # if shared object contains references add an rpath to lib/
+        if ${stdenv.cc.targetPrefix}otool -l "$1"| grep -q '@rpath/'; then
+          ${stdenv.cc.targetPrefix}install_name_tool -add_rpath "$rpath" "$1"
         fi
-      done
+      }
 
-      for i in $out/bin/* $out/lib/*.dylib $out/lib/darwin/*.dylib; do
-        if [[ ! -L "$i" ]]; then
-          rpathify $i
-        fi
-      done
+      # check that linked library paths exist in $out/lib
+      # must be run after rpathify is performed
+      checkDeps() {
+        local deps=$(${stdenv.cc.targetPrefix}otool -l "$1"| grep -o '@rpath/[^      ]*' || true)
+        local lib
+        for lib in $deps; do
+          if [[ ! -e $out/''${lib/@rpath/lib} ]]; then
+            echo "error: $1 missing lib for $lib" >&2
+            exit 1
+          fi
+        done
+      }
 
-      for i in $out/bin/*; do
-        if [[ ! -L "$i" ]] && isMachO "$i"; then
-          ${stdenv.cc.targetPrefix}install_name_tool -add_rpath '@executable_path/../lib' $i
+      for i in $out/bin/* unpack/bin/* $out/lib{,/darwin}/*.dylib; do
+        if [[ ! -L $i ]] && isMachO "$i"; then
+          rpathify "$i"
+          checkDeps "$i"
         fi
       done
 
-      ${if stdenv.targetPlatform.isx86_64 then ''
-        rpathify $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation
-      '' else ''
-        sed -i -e 's|/nix/store/.*/libobjc.A.dylib|@executable_path/../libobjc.A.dylib|g' \
-          $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation.tbd
-      ''}
-
+      nuke-refs $out/bin/*
       nuke-refs $out/lib/*
-      nuke-refs $out/lib/system/*
       nuke-refs $out/lib/darwin/*
-      ${lib.optionalString stdenv.targetPlatform.isx86_64 ''
-        nuke-refs $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation
-      ''}
+      nuke-refs $out/lib/system/*
+      nuke-refs unpack/bin/*
 
       mkdir $out/.pack
       mv $out/* $out/.pack
       mv $out/.pack $out/pack
 
+      # validate that tools contain no references into the archive
+      for tool in unpack/bin/*; do
+        deps=$(${stdenv.cc.targetPrefix}otool -l "$tool"| grep '@rpath/' || true)
+        if [[ -n "$deps" ]]; then
+          printf "error: $tool is not self contained\n$deps\n" >&2
+          exit 1
+        fi
+      done
+
       mkdir $out/on-server
-      dumpnar $out/pack | ${getBin xz}/bin/xz > $out/on-server/bootstrap-tools.nar.xz
+      cp -r unpack $out
+
+      XZ_OPT="-9 -T $NIX_BUILD_CORES" tar cvJf $out/on-server/bootstrap-tools.tar.xz \
+        --hard-dereference --sort=name --numeric-owner --owner=0 --group=0 --mtime=@1 -C $out/pack .
+      dumpnar $out/unpack | xz -9 -T $NIX_BUILD_CORES > $out/on-server/unpack.nar.xz
     '';
 
     allowedReferences = [];
@@ -211,44 +254,42 @@ rec {
     };
   };
 
-  dist = stdenv.mkDerivation {
-    name = "stdenv-bootstrap-tools";
-
-    buildCommand = ''
-      mkdir -p $out/nix-support
-      echo "file tools ${build}/on-server/bootstrap-tools.nar.xz" >> $out/nix-support/hydra-build-products
-    '';
-  };
+  dist = runCommand "stdenv-bootstrap-tools" {} ''
+    mkdir -p $out/nix-support
+    echo "file tarball ${build}/on-server/*.tar.xz" >> $out/nix-support/hydra-build-products
+    echo "file unpack ${build}/on-server/unpack.* " >> $out/nix-support/hydra-build-products
+  '';
 
   bootstrapFiles = {
-    tools = "${build}/pack";
+    bootstrapTools = "${build}/on-server/bootstrap-tools.tar.xz";
+    unpack = runCommand "unpack" { allowedReferences = []; } ''
+      cp -r ${build}/unpack $out
+    '';
   };
 
   bootstrapTools = derivation {
     inherit (stdenv.hostPlatform) system;
 
     name = "bootstrap-tools";
-    builder = "${bootstrapFiles.tools}/bin/bash";
+    builder = "${bootstrapFiles.unpack}/bin/bash";
 
-    # This is by necessity a near-duplicate of patch-bootstrap-tools.sh. If we refer to it directly,
-    # we can't make any changes to it due to our testing stdenv depending on it. Think of this as the
-    # patch-bootstrap-tools.sh for the next round of bootstrap tools.
-    args = [ ./patch-bootstrap-tools-next.sh ];
-
-    inherit (bootstrapFiles) tools;
+    args = [ ./patch-bootstrap-tools.sh bootstrapFiles.bootstrapTools ];
+    PATH = lib.makeBinPath [ (placeholder "out") bootstrapFiles.unpack ];
 
     allowedReferences = [ "out" ];
   };
 
-  test = stdenv.mkDerivation {
-    name = "test";
-
-    realBuilder = "${bootstrapTools}/bin/bash";
-
+  test = derivation {
+    name = "test-bootstrap-tools";
+    inherit (stdenv.hostPlatform) system;
+    builder = "${bootstrapTools}/bin/bash";
+    args = [ "-euo" "pipefail" "-c" "eval \"$buildCommand\"" ];
+    PATH = lib.makeBinPath [ bootstrapTools ];
     tools = bootstrapTools;
+    "${stdenv.cc.darwinMinVersionVariable}" = stdenv.cc.darwinMinVersion;
+
+    # Create a pure environment where we use just what's in the bootstrap tools.
     buildCommand = ''
-      # Create a pure environment where we use just what's in the bootstrap tools.
-      export PATH=$tools/bin
 
       ls -l
       mkdir $out
@@ -275,16 +316,15 @@ rec {
         ${stdenv.cc.libc_dev}/lib/system \
         libSystem-boot
 
-      substituteInPlace libSystem-boot/libSystem.B.tbd \
-        --replace "/usr/lib/system/" "$PWD/libSystem-boot/system/"
+      sed -i "s|/usr/lib/system/|$PWD/libSystem-boot/system/|g" libSystem-boot/libSystem.B.tbd
       ln -s libSystem.B.tbd libSystem-boot/libSystem.tbd
       # End of bootstrap libSystem
 
       export flags="-idirafter $tools/include-Libsystem --sysroot=$tools -L$tools/lib -L$PWD/libSystem-boot"
 
       export CPP="clang -E $flags"
-      export CC="clang $flags -rpath $tools/lib"
-      export CXX="clang++ $flags --stdlib=libc++ -lc++abi -isystem$tools/include/c++/v1 -rpath $tools/lib"
+      export CC="clang $flags"
+      export CXX="clang++ $flags --stdlib=libc++ -isystem$tools/include/c++/v1"
 
       # NOTE: These tests do a separate 'install' step (using cp), because
       # having clang write directly to the final location apparently will make
@@ -301,22 +341,23 @@ rec {
       cp hello1 $out/bin/
       $out/bin/hello1
 
-      echo '#include <CoreFoundation/CoreFoundation.h>' >> hello2.c
-      echo 'int main() { CFShow(CFSTR("Hullo")); return 0; }' >> hello2.c
-      $CC -F$tools/Library/Frameworks -framework CoreFoundation -o hello2 hello2.c
-      cp hello2 $out/bin/
-      $out/bin/hello2
-
       echo '#include <iostream>' >> hello3.cc
       echo 'int main() { std::cout << "Hello World\n"; }' >> hello3.cc
       $CXX -v -o hello3 hello3.cc
       cp hello3 $out/bin/
       $out/bin/hello3
 
+      # test that libc++.dylib rpaths are correct so it can reference libc++abi.dylib when linked.
+      # using -Wl,-flat_namespace is required to generate an error
+      mkdir libtest/
+      ln -s $tools/lib/libc++.dylib libtest/
+      clang++ -Wl,-flat_namespace -idirafter $tools/include-Libsystem -isystem$tools/include/c++/v1 \
+        --sysroot=$tools -L./libtest -L$PWD/libSystem-boot hello3.cc
+
       tar xvf ${hello.src}
       cd hello-*
-      # stdenv bootstrap tools ship a broken libiconv.dylib https://github.com/NixOS/nixpkgs/issues/158331
-      am_cv_func_iconv=no ./configure --prefix=$out
+      # hello configure detects -liconv is needed but doesn't add to the link step
+      LDFLAGS=-liconv ./configure --prefix=$out
       make
       make install
       $out/bin/hello
diff --git a/pkgs/stdenv/darwin/patch-bootstrap-tools-next.sh b/pkgs/stdenv/darwin/patch-bootstrap-tools.sh
index a5b9edff7cd5f..555c340a7e752 100644
--- a/pkgs/stdenv/darwin/patch-bootstrap-tools-next.sh
+++ b/pkgs/stdenv/darwin/patch-bootstrap-tools.sh
@@ -1,9 +1,8 @@
 set -euo pipefail
 
-export PATH=$tools/bin
-
-cp -R $tools $out
-chmod -R u+w $out
+echo Unpacking the bootstrap tools... >&2
+mkdir $out
+tar xf "$1" -C $out
 
 updateInstallName() {
   local path="$1"