diff options
Diffstat (limited to 'pkgs/stdenv')
-rw-r--r-- | pkgs/stdenv/adapters.nix | 31 | ||||
-rw-r--r-- | pkgs/stdenv/darwin/make-bootstrap-tools.nix | 212 | ||||
-rw-r--r-- | pkgs/stdenv/darwin/patch-bootstrap-tools-next.sh | 38 | ||||
-rw-r--r-- | pkgs/stdenv/generic/check-meta.nix | 13 | ||||
-rw-r--r-- | pkgs/stdenv/generic/make-derivation.nix | 82 | ||||
-rw-r--r-- | pkgs/stdenv/generic/setup.sh | 80 | ||||
-rw-r--r-- | pkgs/stdenv/linux/bootstrap-files/aarch64.nix | 8 | ||||
-rw-r--r-- | pkgs/stdenv/linux/bootstrap-files/mips64el-n32.nix | 25 | ||||
-rw-r--r-- | pkgs/stdenv/linux/bootstrap-tools-musl/default.nix | 1 | ||||
-rw-r--r-- | pkgs/stdenv/linux/bootstrap-tools/default.nix | 1 | ||||
-rw-r--r-- | pkgs/stdenv/linux/bootstrap-tools/scripts/unpack-bootstrap-tools.sh | 7 | ||||
-rw-r--r-- | pkgs/stdenv/linux/default.nix | 294 | ||||
-rw-r--r-- | pkgs/stdenv/linux/make-bootstrap-tools.nix | 7 |
13 files changed, 501 insertions, 298 deletions
diff --git a/pkgs/stdenv/adapters.nix b/pkgs/stdenv/adapters.nix index 42d19a0fad4b9..0c5645e5a487f 100644 --- a/pkgs/stdenv/adapters.nix +++ b/pkgs/stdenv/adapters.nix @@ -142,7 +142,7 @@ rec { Example: stdenvNoOptimise = addAttrsToDerivation - { NIX_CFLAGS_COMPILE = "-O0"; } + { env.NIX_CFLAGS_COMPILE = "-O0"; } stdenv; */ addAttrsToDerivation = extraAttrs: stdenv: stdenv.override (old: { @@ -176,7 +176,7 @@ rec { stdenv.override (old: { mkDerivationFromStdenv = extendMkDerivationArgs old (args: { dontStrip = true; - NIX_CFLAGS_COMPILE = toString (args.NIX_CFLAGS_COMPILE or "") + " -ggdb -Og"; + env = (args.env or {}) // { NIX_CFLAGS_COMPILE = toString (args.env.NIX_CFLAGS_COMPILE or "") + " -ggdb -Og"; }; }); }); @@ -189,6 +189,28 @@ rec { }); }); + useMoldLinker = stdenv: let + bintools = stdenv.cc.bintools.override { + extraBuildCommands = '' + wrap ld.mold ${../build-support/bintools-wrapper/ld-wrapper.sh} ${pkgs.mold}/bin/ld.mold + wrap ld ${../build-support/bintools-wrapper/ld-wrapper.sh} ${pkgs.mold}/bin/ld.mold + ''; + }; + in stdenv.override (old: { + cc = stdenv.cc.override { + inherit bintools; + }; + allowedRequisites = + lib.mapNullable (rs: rs ++ [ bintools pkgs.mold (lib.getLib pkgs.mimalloc) (lib.getLib pkgs.openssl) ]) (stdenv.allowedRequisites or null); + # gcc >12.1.0 supports '-fuse-ld=mold' + # the wrap ld above in bintools supports gcc <12.1.0 and shouldn't harm >12.1.0 + # https://github.com/rui314/mold#how-to-use + } // lib.optionalAttrs (stdenv.cc.isClang || (stdenv.cc.isGNU && lib.versionAtLeast stdenv.cc.version "12")) { + mkDerivationFromStdenv = extendMkDerivationArgs old (args: { + NIX_CFLAGS_LINK = toString (args.NIX_CFLAGS_LINK or "") + " -fuse-ld=mold"; + }); + }); + /* Modify a stdenv so that it builds binaries optimized specifically for the machine they are built on. @@ -197,7 +219,8 @@ rec { impureUseNativeOptimizations = stdenv: stdenv.override (old: { mkDerivationFromStdenv = extendMkDerivationArgs old (args: { - NIX_CFLAGS_COMPILE = toString (args.NIX_CFLAGS_COMPILE or "") + " -march=native"; + env = (args.env or {}) // { NIX_CFLAGS_COMPILE = toString (args.env.NIX_CFLAGS_COMPILE or "") + " -march=native"; }; + NIX_ENFORCE_NO_NATIVE = false; preferLocalBuild = true; @@ -222,7 +245,7 @@ rec { withCFlags = compilerFlags: stdenv: stdenv.override (old: { mkDerivationFromStdenv = extendMkDerivationArgs old (args: { - NIX_CFLAGS_COMPILE = toString (args.NIX_CFLAGS_COMPILE or "") + " ${toString compilerFlags}"; + env = (args.env or {}) // { NIX_CFLAGS_COMPILE = toString (args.env.NIX_CFLAGS_COMPILE or "") + " ${toString compilerFlags}"; }; }); }); } diff --git a/pkgs/stdenv/darwin/make-bootstrap-tools.nix b/pkgs/stdenv/darwin/make-bootstrap-tools.nix index 2cc44b319ffaa..5433e7afecf9f 100644 --- a/pkgs/stdenv/darwin/make-bootstrap-tools.nix +++ b/pkgs/stdenv/darwin/make-bootstrap-tools.nix @@ -37,8 +37,7 @@ in rec { build = stdenv.mkDerivation { name = "stdenv-bootstrap-tools"; - nativeBuildInputs = [ buildPackages.nukeReferences buildPackages.cpio ] - ++ lib.optionals targetPlatform.isAarch64 [ buildPackages.darwin.sigtool ]; + nativeBuildInputs = [ nukeReferences dumpnar ]; buildCommand = '' mkdir -p $out/bin $out/lib $out/lib/system $out/lib/darwin @@ -49,20 +48,21 @@ in rec { # Resolv is actually a link to another package, so let's copy it properly cp -L ${lib.getLib darwin.Libsystem}/lib/libresolv.9.dylib $out/lib - - cp -rL ${darwin.Libsystem}/include $out - chmod -R u+w $out/include - cp -rL ${darwin.ICU}/include* $out/include - cp -rL ${libiconv}/include/* $out/include - cp -rL ${gnugrep.pcre.dev}/include/* $out/include - mv $out/include $out/include-Libsystem ''} + cp -rL ${darwin.Libsystem}/include $out + chmod -R u+w $out/include + cp -rL ${darwin.ICU}/include* $out/include + cp -rL ${libiconv}/include/* $out/include + cp -rL ${lib.getDev gnugrep.pcre}/include/* $out/include + mv $out/include $out/include-Libsystem + # Copy coreutils, bash, etc. cp ${coreutils_}/bin/* $out/bin (cd $out/bin && rm vdir dir sha*sum pinky factor pathchk runcon shuf who whoami shred users) cp ${bash}/bin/bash $out/bin + ln -s bash $out/bin/sh cp ${findutils}/bin/find $out/bin cp ${findutils}/bin/xargs $out/bin cp -d ${diffutils}/bin/* $out/bin @@ -73,9 +73,11 @@ in rec { cp ${gnutar}/bin/tar $out/bin cp ${gzip}/bin/.gzip-wrapped $out/bin/gzip cp ${bzip2_.bin}/bin/bzip2 $out/bin + ln -s bzip2 $out/bin/bunzip2 cp -d ${gnumake}/bin/* $out/bin cp -d ${patch}/bin/* $out/bin cp -d ${xz.bin}/bin/xz $out/bin + cp ${cpio}/bin/cpio $out/bin # This used to be in-nixpkgs, but now is in the bundle # because I can't be bothered to make it partially static @@ -114,11 +116,9 @@ in rec { cp -d ${lib.getLib pkgs.xar}/lib/libxar*.dylib $out/lib cp -d ${pkgs.bzip2.out}/lib/libbz2*.dylib $out/lib - ${lib.optionalString targetPlatform.isAarch64 '' - # copy sigtool - cp -d ${pkgs.darwin.sigtool}/bin/sigtool $out/bin - cp -d ${pkgs.darwin.sigtool}/bin/codesign $out/bin - ''} + # copy sigtool + cp -d ${pkgs.darwin.sigtool}/bin/sigtool $out/bin + cp -d ${pkgs.darwin.sigtool}/bin/codesign $out/bin cp -d ${lib.getLib darwin.ICU}/lib/libicu*.dylib $out/lib cp -d ${zlib.out}/lib/libz.* $out/lib @@ -132,8 +132,9 @@ in rec { cp -d ${lib.getLib darwin.libtapi}/lib/libtapi* $out/lib - ${lib.optionalString targetPlatform.isx86_64 '' - cp -rd ${pkgs.darwin.CF}/Library $out + cp -rd ${pkgs.darwin.CF}/Library $out + ${lib.optionalString stdenv.targetPlatform.isAarch64 '' + cp -rd ${pkgs.darwin.libobjc}/lib/* $out/lib/ ''} chmod -R u+w $out @@ -150,46 +151,44 @@ in rec { # Strip executables even further for i in $out/bin/*; do - if test -x $i -a ! -L $i; then + if [[ ! -L $i ]]; then chmod +w $i ${stdenv.cc.targetPrefix}strip $i || true fi done - for i in $out/bin/* $out/lib/*.dylib $out/lib/darwin/*.dylib $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation; do - if test -x "$i" -a ! -L "$i"; then - echo "Adding rpath to $i" + for i in $out/bin/* $out/lib/*.dylib $out/lib/darwin/*.dylib; do + if [[ ! -L "$i" ]]; then rpathify $i fi done for i in $out/bin/*; do - if test -x "$i" -a ! -L "$i"; then - echo "Adding @executable_path to rpath in $i" + if [[ ! -L "$i" ]]; then ${stdenv.cc.targetPrefix}install_name_tool -add_rpath '@executable_path/../lib' $i fi done + ${if stdenv.targetPlatform.isx86_64 then '' + rpathify $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation + '' else '' + sed -i -e 's|/nix/store/.*/libobjc.A.dylib|@executable_path/../libobjc.A.dylib|g' \ + $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation.tbd + ''} + nuke-refs $out/lib/* nuke-refs $out/lib/system/* nuke-refs $out/lib/darwin/* - nuke-refs $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation + ${lib.optionalString stdenv.targetPlatform.isx86_64 '' + nuke-refs $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation + ''} mkdir $out/.pack mv $out/* $out/.pack mv $out/.pack $out/pack mkdir $out/on-server - cp ${stdenv.shell} $out/on-server/sh - cp ${cpio}/bin/cpio $out/on-server - cp ${coreutils_}/bin/mkdir $out/on-server - cp ${bzip2_.bin}/bin/bzip2 $out/on-server - - chmod u+w $out/on-server/* - ${stdenv.cc.targetPrefix}strip $out/on-server/* - nuke-refs $out/on-server/* - - (cd $out/pack && (find | cpio -o -H newc)) | bzip2 > $out/on-server/bootstrap-tools.cpio.bz2 + dumpnar $out/pack | ${xz}/bin/xz > $out/on-server/bootstrap-tools.nar.xz ''; allowedReferences = []; @@ -204,98 +203,42 @@ in rec { buildCommand = '' mkdir -p $out/nix-support - echo "file tarball ${build}/on-server/bootstrap-tools.cpio.bz2" >> $out/nix-support/hydra-build-products - echo "file sh ${build}/on-server/sh" >> $out/nix-support/hydra-build-products - echo "file cpio ${build}/on-server/cpio" >> $out/nix-support/hydra-build-products - echo "file mkdir ${build}/on-server/mkdir" >> $out/nix-support/hydra-build-products - echo "file bzip2 ${build}/on-server/bzip2" >> $out/nix-support/hydra-build-products + echo "file tools ${build}/on-server/bootstrap-tools.nar.xz" >> $out/nix-support/hydra-build-products ''; }; bootstrapLlvmVersion = llvmPackages.llvm.version; bootstrapFiles = { - sh = "${build}/on-server/sh"; - bzip2 = "${build}/on-server/bzip2"; - mkdir = "${build}/on-server/mkdir"; - cpio = "${build}/on-server/cpio"; - tarball = "${build}/on-server/bootstrap-tools.cpio.bz2"; + tools = "${build}/pack"; }; - unpack = stdenv.mkDerivation (bootstrapFiles // { - name = "unpack"; - - # This is by necessity a near-duplicate of unpack-bootstrap-tools.sh. If we refer to it directly, - # we can't make any changes to it due to our testing stdenv depending on it. Think of this as the - # unpack-bootstrap-tools.sh for the next round of bootstrap tools. - # TODO: think through alternate designs, such as hosting this script as an output of the process. - buildCommand = '' - # Unpack the bootstrap tools tarball. - echo Unpacking the bootstrap tools... - $mkdir $out - $bzip2 -d < $tarball | (cd $out && $cpio -i) - - # Set the ELF interpreter / RPATH in the bootstrap binaries. - echo Patching the tools... + bootstrapTools = derivation { + inherit system; - export PATH=$out/bin - - for i in $out/bin/*; do - if ! test -L $i; then - echo patching $i - install_name_tool -add_rpath $out/lib $i || true - fi - done + name = "bootstrap-tools"; + builder = "${bootstrapFiles.tools}/bin/bash"; - ln -s libresolv.9.dylib $out/lib/libresolv.dylib - - for i in $out/lib/*.dylib $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation; do - if test ! -L "$i" -a "$i" != "$out/lib/libSystem*.dylib"; then - echo "Patching $i" - - id=$(otool -D "$i" | tail -n 1) - install_name_tool -id "$(dirname $i)/$(basename $id)" $i - - libs=$(otool -L "$i" | tail -n +2 | grep -v libSystem | cat) - if [ -n "$libs" ]; then - install_name_tool -add_rpath $out/lib $i - fi - fi - done - - ln -s bash $out/bin/sh - ln -s bzip2 $out/bin/bunzip2 - - # Provide a gunzip script. - cat > $out/bin/gunzip <<EOF - #!$out/bin/sh - exec $out/bin/gzip -d "\$@" - EOF - chmod +x $out/bin/gunzip - - # Provide fgrep/egrep. - echo "#! $out/bin/sh" > $out/bin/egrep - echo "exec $out/bin/grep -E \"\$@\"" >> $out/bin/egrep - echo "#! $out/bin/sh" > $out/bin/fgrep - echo "exec $out/bin/grep -F \"\$@\"" >> $out/bin/fgrep + # This is by necessity a near-duplicate of patch-bootstrap-tools.sh. If we refer to it directly, + # we can't make any changes to it due to our testing stdenv depending on it. Think of this as the + # patch-bootstrap-tools.sh for the next round of bootstrap tools. + args = [ ./patch-bootstrap-tools-next.sh ]; - cat >$out/bin/dsymutil << EOF - #!$out/bin/sh - EOF - - chmod +x $out/bin/egrep $out/bin/fgrep $out/bin/dsymutil - ''; + inherit (bootstrapFiles) tools; allowedReferences = [ "out" ]; - }); + }; test = stdenv.mkDerivation { name = "test"; - realBuilder = "${unpack}/bin/bash"; + realBuilder = "${bootstrapTools}/bin/bash"; + tools = bootstrapTools; buildCommand = '' - export PATH=${unpack}/bin + # Create a pure environment where we use just what's in the bootstrap tools. + export PATH=$tools/bin + ls -l mkdir $out mkdir $out/bin @@ -313,14 +256,12 @@ in rec { # an SSL-capable curl curl --version | grep SSL - ${build}/on-server/sh -c 'echo Hello World' - # This approximates a bootstrap version of libSystem can that be # assembled via fetchurl. Adapted from main libSystem expression. mkdir libSystem-boot cp -vr \ - ${darwin.darwin-stubs}/usr/lib/libSystem.B.tbd \ - ${darwin.darwin-stubs}/usr/lib/system \ + ${stdenv.cc.libc_dev}/lib/libSystem.B.tbd \ + ${stdenv.cc.libc_dev}/lib/system \ libSystem-boot substituteInPlace libSystem-boot/libSystem.B.tbd \ @@ -328,28 +269,38 @@ in rec { ln -s libSystem.B.tbd libSystem-boot/libSystem.tbd # End of bootstrap libSystem - export flags="-idirafter ${unpack}/include-Libsystem --sysroot=${unpack} -L${unpack}/lib -L$PWD/libSystem-boot" + export flags="-idirafter $tools/include-Libsystem --sysroot=$tools -L$tools/lib -L$PWD/libSystem-boot" export CPP="clang -E $flags" - export CC="clang $flags -Wl,-rpath,${unpack}/lib -Wl,-v -Wl,-sdk_version,10.10" - export CXX="clang++ $flags --stdlib=libc++ -lc++abi -isystem${unpack}/include/c++/v1 -Wl,-rpath,${unpack}/lib -Wl,-v -Wl,-sdk_version,10.10" - - echo '#include <stdio.h>' >> foo.c - echo '#include <float.h>' >> foo.c - echo '#include <limits.h>' >> foo.c - echo 'int main() { printf("Hello World\n"); return 0; }' >> foo.c - $CC -o $out/bin/foo foo.c - $out/bin/foo - - echo '#include <CoreFoundation/CoreFoundation.h>' >> bar.c - echo 'int main() { CFShow(CFSTR("Hullo")); return 0; }' >> bar.c - $CC -F${unpack}/Library/Frameworks -framework CoreFoundation -o $out/bin/bar bar.c - $out/bin/bar - - echo '#include <iostream>' >> bar.cc - echo 'int main() { std::cout << "Hello World\n"; }' >> bar.cc - $CXX -v -o $out/bin/bar bar.cc - $out/bin/bar + export CC="clang $flags -rpath $tools/lib" + export CXX="clang++ $flags --stdlib=libc++ -lc++abi -isystem$tools/include/c++/v1 -rpath $tools/lib" + + # NOTE: These tests do a separate 'install' step (using cp), because + # having clang write directly to the final location apparently will make + # running the executable fail signature verification. (SIGKILL'd) + # + # Suspect this is creating a corrupt entry in the kernel cache, but it is + # unique to cctools ld. (The problem goes away with `-fuse-ld=lld`.) + + echo '#include <stdio.h>' >> hello1.c + echo '#include <float.h>' >> hello1.c + echo '#include <limits.h>' >> hello1.c + echo 'int main() { printf("Hello World\n"); return 0; }' >> hello1.c + $CC -o hello1 hello1.c + cp hello1 $out/bin/ + $out/bin/hello1 + + echo '#include <CoreFoundation/CoreFoundation.h>' >> hello2.c + echo 'int main() { CFShow(CFSTR("Hullo")); return 0; }' >> hello2.c + $CC -F$tools/Library/Frameworks -framework CoreFoundation -o hello2 hello2.c + cp hello2 $out/bin/ + $out/bin/hello2 + + echo '#include <iostream>' >> hello3.cc + echo 'int main() { std::cout << "Hello World\n"; }' >> hello3.cc + $CXX -v -o hello3 hello3.cc + cp hello3 $out/bin/ + $out/bin/hello3 tar xvf ${hello.src} cd hello-* @@ -357,7 +308,6 @@ in rec { am_cv_func_iconv=no ./configure --prefix=$out make make install - $out/bin/hello ''; }; diff --git a/pkgs/stdenv/darwin/patch-bootstrap-tools-next.sh b/pkgs/stdenv/darwin/patch-bootstrap-tools-next.sh new file mode 100644 index 0000000000000..a5b9edff7cd5f --- /dev/null +++ b/pkgs/stdenv/darwin/patch-bootstrap-tools-next.sh @@ -0,0 +1,38 @@ +set -euo pipefail + +export PATH=$tools/bin + +cp -R $tools $out +chmod -R u+w $out + +updateInstallName() { + local path="$1" + + cp "$path" "$path.new" + install_name_tool -id "$path" "$path.new" + codesign -f -i "$(basename "$path")" -s - "$path.new" + mv -f "$path.new" "$path" +} + +find $out/lib -type f -name '*.dylib' -print0 | while IFS= read -r -d $'\0' lib; do + updateInstallName "$lib" +done + +# Provide a gunzip script. +cat > $out/bin/gunzip <<EOF +#!$out/bin/sh +exec $out/bin/gzip -d "\$@" +EOF +chmod +x $out/bin/gunzip + +# Provide fgrep/egrep. +echo "#! $out/bin/sh" > $out/bin/egrep +echo "exec $out/bin/grep -E \"\$@\"" >> $out/bin/egrep +echo "#! $out/bin/sh" > $out/bin/fgrep +echo "exec $out/bin/grep -F \"\$@\"" >> $out/bin/fgrep + +cat >$out/bin/dsymutil << EOF +#!$out/bin/sh +EOF + +chmod +x $out/bin/egrep $out/bin/fgrep $out/bin/dsymutil diff --git a/pkgs/stdenv/generic/check-meta.nix b/pkgs/stdenv/generic/check-meta.nix index 4b0ca649ee94d..7f317c787b04e 100644 --- a/pkgs/stdenv/generic/check-meta.nix +++ b/pkgs/stdenv/generic/check-meta.nix @@ -368,7 +368,18 @@ let else if !allowBroken && attrs.meta.broken or false then { valid = "no"; reason = "broken"; errormsg = "is marked as broken"; } else if !allowUnsupportedSystem && hasUnsupportedPlatform attrs then - { valid = "no"; reason = "unsupported"; errormsg = "is not supported on ‘${hostPlatform.system}’"; } + let toPretty = lib.generators.toPretty { + allowPrettyValues = true; + indent = " "; + }; + in { valid = "no"; reason = "unsupported"; + errormsg = '' + is not available on the requested hostPlatform: + hostPlatform.config = "${hostPlatform.config}" + package.meta.platforms = ${toPretty (attrs.meta.platforms or [])} + package.meta.badPlatforms = ${toPretty (attrs.meta.badPlatforms or [])} + ''; + } else if !(hasAllowedInsecure attrs) then { valid = "no"; reason = "insecure"; errormsg = "is marked as insecure"; } diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 4131067cb423e..6cb494d46e5f2 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -18,33 +18,34 @@ let # separate lines, because Nix would only show the last line of the comment. # An infinite recursion here can be caused by having the attribute names of expression `e` in `.overrideAttrs(finalAttrs: previousAttrs: e)` depend on `finalAttrs`. Only the attribute values of `e` can depend on `finalAttrs`. - args = rattrs (args // { inherit finalPackage; }); + args = rattrs (args // { inherit finalPackage overrideAttrs; }); # ^^^^ - finalPackage = - mkDerivationSimple - (f0: - let - f = self: super: - # Convert f0 to an overlay. Legacy is: - # overrideAttrs (super: {}) - # We want to introduce self. We follow the convention of overlays: - # overrideAttrs (self: super: {}) - # Which means the first parameter can be either self or super. - # This is surprising, but far better than the confusion that would - # arise from flipping an overlay's parameters in some cases. - let x = f0 super; - in - if builtins.isFunction x - then - # Can't reuse `x`, because `self` comes first. - # Looks inefficient, but `f0 super` was a cheap thunk. - f0 self super - else x; + overrideAttrs = f0: + let + f = self: super: + # Convert f0 to an overlay. Legacy is: + # overrideAttrs (super: {}) + # We want to introduce self. We follow the convention of overlays: + # overrideAttrs (self: super: {}) + # Which means the first parameter can be either self or super. + # This is surprising, but far better than the confusion that would + # arise from flipping an overlay's parameters in some cases. + let x = f0 super; in - makeDerivationExtensible - (self: let super = rattrs self; in super // f self super)) - args; + if builtins.isFunction x + then + # Can't reuse `x`, because `self` comes first. + # Looks inefficient, but `f0 super` was a cheap thunk. + f0 self super + else x; + in + makeDerivationExtensible + (self: let super = rattrs self; in super // f self super); + + finalPackage = + mkDerivationSimple overrideAttrs args; + in finalPackage; # makeDerivationExtensibleConst == makeDerivationExtensible (_: attrs), @@ -170,7 +171,7 @@ let doCheck' = doCheck && stdenv.buildPlatform.canExecute stdenv.hostPlatform; doInstallCheck' = doInstallCheck && stdenv.buildPlatform.canExecute stdenv.hostPlatform; - separateDebugInfo' = separateDebugInfo && stdenv.hostPlatform.isLinux && !(stdenv.hostPlatform.useLLVM or false); + separateDebugInfo' = separateDebugInfo && stdenv.hostPlatform.isLinux; outputs' = outputs ++ lib.optional separateDebugInfo' "debug"; # Turn a derivation into its outPath without a string context attached. @@ -185,27 +186,35 @@ let ++ buildInputs ++ propagatedBuildInputs ++ depsTargetTarget ++ depsTargetTargetPropagated) == 0; dontAddHostSuffix = attrs ? outputHash && !noNonNativeDeps || !stdenv.hasCC; - supportedHardeningFlags = [ "fortify" "stackprotector" "pie" "pic" "strictoverflow" "format" "relro" "bindnow" ]; + + hardeningDisable' = if lib.any (x: x == "fortify") hardeningDisable + # disabling fortify implies fortify3 should also be disabled + then lib.unique (hardeningDisable ++ [ "fortify3" ]) + else hardeningDisable; + supportedHardeningFlags = [ "fortify" "fortify3" "stackprotector" "pie" "pic" "strictoverflow" "format" "relro" "bindnow" ]; # Musl-based platforms will keep "pie", other platforms will not. # If you change this, make sure to update section `{#sec-hardening-in-nixpkgs}` # in the nixpkgs manual to inform users about the defaults. - defaultHardeningFlags = if stdenv.hostPlatform.isMusl && - # Except when: - # - static aarch64, where compilation works, but produces segfaulting dynamically linked binaries. - # - static armv7l, where compilation fails. - !(stdenv.hostPlatform.isAarch && stdenv.hostPlatform.isStatic) - then supportedHardeningFlags - else lib.remove "pie" supportedHardeningFlags; + defaultHardeningFlags = let + # not ready for this by default + supportedHardeningFlags' = lib.remove "fortify3" supportedHardeningFlags; + in if stdenv.hostPlatform.isMusl && + # Except when: + # - static aarch64, where compilation works, but produces segfaulting dynamically linked binaries. + # - static armv7l, where compilation fails. + !(stdenv.hostPlatform.isAarch && stdenv.hostPlatform.isStatic) + then supportedHardeningFlags' + else lib.remove "pie" supportedHardeningFlags'; enabledHardeningOptions = - if builtins.elem "all" hardeningDisable + if builtins.elem "all" hardeningDisable' then [] - else lib.subtractLists hardeningDisable (defaultHardeningFlags ++ hardeningEnable); + else lib.subtractLists hardeningDisable' (defaultHardeningFlags ++ hardeningEnable); # hardeningDisable additionally supports "all". erroneousHardeningFlags = lib.subtractLists supportedHardeningFlags (hardeningEnable ++ lib.remove "all" hardeningDisable); checkDependencyList = checkDependencyList' []; checkDependencyList' = positions: name: deps: lib.flip lib.imap1 deps (index: dep: - if lib.isDerivation dep || isNull dep || builtins.typeOf dep == "string" || builtins.typeOf dep == "path" then dep + if lib.isDerivation dep || dep == null || builtins.typeOf dep == "string" || builtins.typeOf dep == "path" then dep else if lib.isList dep then checkDependencyList' ([index] ++ positions) name dep else throw "Dependency is not of a valid type: ${lib.concatMapStrings (ix: "element ${toString ix} of ") ([index] ++ positions)}${name} for ${attrs.name or attrs.pname}"); in if builtins.length erroneousHardeningFlags != 0 @@ -436,6 +445,7 @@ else let } // lib.optionalAttrs (enableParallelBuilding) { inherit enableParallelBuilding; enableParallelChecking = attrs.enableParallelChecking or true; + enableParallelInstalling = attrs.enableParallelInstalling or true; } // lib.optionalAttrs (hardeningDisable != [] || hardeningEnable != [] || stdenv.hostPlatform.isMusl) { NIX_HARDENING_ENABLE = enabledHardeningOptions; } // lib.optionalAttrs (stdenv.hostPlatform.isx86_64 && stdenv.hostPlatform ? gcc.arch) { diff --git a/pkgs/stdenv/generic/setup.sh b/pkgs/stdenv/generic/setup.sh index cbf2a2ceb8dee..734abb890c24f 100644 --- a/pkgs/stdenv/generic/setup.sh +++ b/pkgs/stdenv/generic/setup.sh @@ -28,10 +28,16 @@ if [ -n "$__structuredAttrs" ]; then # ex: out=/nix/store/... export "$outputName=${outputs[$outputName]}" done - # $NIX_ATTRS_JSON_FILE points to the wrong location in sandbox - # https://github.com/NixOS/nix/issues/6736 - export NIX_ATTRS_JSON_FILE="$NIX_BUILD_TOP/.attrs.json" - export NIX_ATTRS_SH_FILE="$NIX_BUILD_TOP/.attrs.sh" + + # $NIX_ATTRS_JSON_FILE pointed to the wrong location in sandbox + # https://github.com/NixOS/nix/issues/6736; please keep around until the + # fix reaches *every patch version* that's >= lib/minver.nix + if ! [[ -e "$NIX_ATTRS_JSON_FILE" ]]; then + export NIX_ATTRS_JSON_FILE="$NIX_BUILD_TOP/.attrs.json" + fi + if ! [[ -e "$NIX_ATTRS_SH_FILE" ]]; then + export NIX_ATTRS_SH_FILE="$NIX_BUILD_TOP/.attrs.sh" + fi else : "${outputs:=out}" fi @@ -989,6 +995,39 @@ stripHash() { } +recordPropagatedDependencies() { + # Propagate dependencies into the development output. + declare -ra flatVars=( + # Build + depsBuildBuildPropagated + propagatedNativeBuildInputs + depsBuildTargetPropagated + # Host + depsHostHostPropagated + propagatedBuildInputs + # Target + depsTargetTargetPropagated + ) + declare -ra flatFiles=( + "${propagatedBuildDepFiles[@]}" + "${propagatedHostDepFiles[@]}" + "${propagatedTargetDepFiles[@]}" + ) + + local propagatedInputsIndex + for propagatedInputsIndex in "${!flatVars[@]}"; do + local propagatedInputsSlice="${flatVars[$propagatedInputsIndex]}[@]" + local propagatedInputsFile="${flatFiles[$propagatedInputsIndex]}" + + [[ "${!propagatedInputsSlice}" ]] || continue + + mkdir -p "${!outputDev}/nix-support" + # shellcheck disable=SC2086 + printWords ${!propagatedInputsSlice} > "${!outputDev}/nix-support/$propagatedInputsFile" + done +} + + unpackCmdHooks+=(_defaultUnpack) _defaultUnpack() { local fn="$1" @@ -1337,6 +1376,7 @@ installPhase() { # shellcheck disable=SC2086 local flagsArray=( + ${enableParallelInstalling:+-j${NIX_BUILD_CORES}} SHELL=$SHELL ) _accumFlagsArray makeFlags makeFlagsArray installFlags installFlagsArray @@ -1373,36 +1413,8 @@ fixupPhase() { done - # Propagate dependencies & setup hook into the development output. - declare -ra flatVars=( - # Build - depsBuildBuildPropagated - propagatedNativeBuildInputs - depsBuildTargetPropagated - # Host - depsHostHostPropagated - propagatedBuildInputs - # Target - depsTargetTargetPropagated - ) - declare -ra flatFiles=( - "${propagatedBuildDepFiles[@]}" - "${propagatedHostDepFiles[@]}" - "${propagatedTargetDepFiles[@]}" - ) - - local propagatedInputsIndex - for propagatedInputsIndex in "${!flatVars[@]}"; do - local propagatedInputsSlice="${flatVars[$propagatedInputsIndex]}[@]" - local propagatedInputsFile="${flatFiles[$propagatedInputsIndex]}" - - [[ "${!propagatedInputsSlice}" ]] || continue - - mkdir -p "${!outputDev}/nix-support" - # shellcheck disable=SC2086 - printWords ${!propagatedInputsSlice} > "${!outputDev}/nix-support/$propagatedInputsFile" - done - + # record propagated dependencies & setup hook into the development output. + recordPropagatedDependencies if [ -n "${setupHook:-}" ]; then mkdir -p "${!outputDev}/nix-support" diff --git a/pkgs/stdenv/linux/bootstrap-files/aarch64.nix b/pkgs/stdenv/linux/bootstrap-files/aarch64.nix index aa81cbd84af31..6719e7ca30722 100644 --- a/pkgs/stdenv/linux/bootstrap-files/aarch64.nix +++ b/pkgs/stdenv/linux/bootstrap-files/aarch64.nix @@ -1,11 +1,11 @@ { busybox = import <nix/fetchurl.nix> { - url = "http://tarballs.nixos.org/stdenv-linux/aarch64/c7c997a0662bf88264db52cbc41e67884eb7a1ff/busybox"; - sha256 = "sha256-4EN2vLvXUkelZZR2eKaAQA5kCEuHNvRZN6dcohxVY+c="; + url = "http://tarballs.nixos.org/stdenv-linux/aarch64/21ec906463ea8f11abf3f9091ddd4c3276516e58/busybox"; executable = true; + hash = "sha256-0MuIeQlBUaeisqoFSu8y+8oB6K4ZG5Lhq8RcS9JqkFQ="; }; bootstrapTools = import <nix/fetchurl.nix> { - url = "http://tarballs.nixos.org/stdenv-linux/aarch64/c7c997a0662bf88264db52cbc41e67884eb7a1ff/bootstrap-tools.tar.xz"; - sha256 = "sha256-AjOvmaW8JFVZaBSRUMKufr9kJozg/tsZr7PvUEBQyi4="; + url = "http://tarballs.nixos.org/stdenv-linux/aarch64/21ec906463ea8f11abf3f9091ddd4c3276516e58/bootstrap-tools.tar.xz"; + hash = "sha256-aJvtsWeuQHbb14BGZ2EiOKzjQn46h3x3duuPEawG0eE="; }; } diff --git a/pkgs/stdenv/linux/bootstrap-files/mips64el-n32.nix b/pkgs/stdenv/linux/bootstrap-files/mips64el-n32.nix new file mode 100644 index 0000000000000..cf26c2113ec8d --- /dev/null +++ b/pkgs/stdenv/linux/bootstrap-files/mips64el-n32.nix @@ -0,0 +1,25 @@ +# +# Files came from this Hydra build: +# +# https://hydra.nixos.org/build/188389586 +# +# Which used nixpkgs revision 97d9c84e1df4397b43ecb39359f1bd003cd44585 +# to instantiate: +# +# /nix/store/hakn8s85s9011v61r6svp5qy8x1y64fv-stdenv-bootstrap-tools-mips64el-unknown-linux-gnuabin32.drv +# +# and then built: +# +# /nix/store/rjgybpnf3yiqyhvl2n2lx31jf800fii2-stdenv-bootstrap-tools-mips64el-unknown-linux-gnuabin32 +# +{ + busybox = import <nix/fetchurl.nix> { + url = "http://tarballs.nixos.org/stdenv-linux/mips64el-n32/97d9c84e1df4397b43ecb39359f1bd003cd44585/busybox"; + sha256 = "sha256-4N3G1qYA7vitjhsIW17pR6UixIuzrq4vZXa8F0/X4iI="; + executable = true; + }; + bootstrapTools = import <nix/fetchurl.nix> { + url = "http://tarballs.nixos.org/stdenv-linux/mips64el-n32/97d9c84e1df4397b43ecb39359f1bd003cd44585/bootstrap-tools.tar.xz"; + sha256 = "sha256-LWrpN6su2yNVurUyhZP34OiZyzgh7MfN13fIIbou8KI="; + }; +} diff --git a/pkgs/stdenv/linux/bootstrap-tools-musl/default.nix b/pkgs/stdenv/linux/bootstrap-tools-musl/default.nix index d690f40267217..569f0c6f31e2f 100644 --- a/pkgs/stdenv/linux/bootstrap-tools-musl/default.nix +++ b/pkgs/stdenv/linux/bootstrap-tools-musl/default.nix @@ -15,4 +15,5 @@ derivation ({ langC = true; langCC = true; isGNU = true; + hardeningUnsupportedFlags = [ "fortify3" ]; } // extraAttrs) diff --git a/pkgs/stdenv/linux/bootstrap-tools/default.nix b/pkgs/stdenv/linux/bootstrap-tools/default.nix index d690f40267217..569f0c6f31e2f 100644 --- a/pkgs/stdenv/linux/bootstrap-tools/default.nix +++ b/pkgs/stdenv/linux/bootstrap-tools/default.nix @@ -15,4 +15,5 @@ derivation ({ langC = true; langCC = true; isGNU = true; + hardeningUnsupportedFlags = [ "fortify3" ]; } // extraAttrs) diff --git a/pkgs/stdenv/linux/bootstrap-tools/scripts/unpack-bootstrap-tools.sh b/pkgs/stdenv/linux/bootstrap-tools/scripts/unpack-bootstrap-tools.sh index 5b5677eef1366..09bf25f52153f 100644 --- a/pkgs/stdenv/linux/bootstrap-tools/scripts/unpack-bootstrap-tools.sh +++ b/pkgs/stdenv/linux/bootstrap-tools/scripts/unpack-bootstrap-tools.sh @@ -30,6 +30,13 @@ LD_LIBRARY_PATH=$out/lib $LD_BINARY $out/bin/mv $out/lib/libstdc++.* $LIBSTDCXX_ # use a copy of patchelf. LD_LIBRARY_PATH=$out/lib $LD_BINARY $out/bin/cp $out/bin/patchelf . +# Older versions of the bootstrap-files did not compile their +# patchelf with -static-libgcc, so we have to be very careful not to +# run patchelf on the same copy of libgcc_s that it links against. +LD_LIBRARY_PATH=$out/lib $LD_BINARY $out/bin/cp $out/lib/libgcc_s.so.1 . +LD_LIBRARY_PATH=.:$out/lib:$LIBSTDCXX_SO_DIR $LD_BINARY \ + ./patchelf --set-rpath $out/lib --force-rpath $out/lib/libgcc_s.so.1 + for i in $out/bin/* $out/libexec/gcc/*/*/*; do if [ -L "$i" ]; then continue; fi if [ -z "${i##*/liblto*}" ]; then continue; fi diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index 8781907382862..e227b6850773c 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -10,13 +10,10 @@ # # Goals of the bootstrap process: # 1. final stdenv must not reference any of the bootstrap files. -# 2. final stdenv must not contain any of the bootstrap files -# (the only current violation is libgcc_s.so in glibc). +# 2. final stdenv must not contain any of the bootstrap files. # 3. final stdenv must not contain any of the files directly # generated by the bootstrap code generators (assembler, linker, -# compiler). The only current violations are: libgcc_s.so in glibc, -# the lib{mpfr,mpc,gmp,isl} which are statically linked -# into the final gcc). +# compiler). # # These goals ensure that final packages and final stdenv are built # exclusively using nixpkgs package definitions and don't depend @@ -68,8 +65,11 @@ armv6l-linux = import ./bootstrap-files/armv6l.nix; armv7l-linux = import ./bootstrap-files/armv7l.nix; aarch64-linux = import ./bootstrap-files/aarch64.nix; - mipsel-linux = import ./bootstrap-files/loongson2f.nix; - mips64el-linux = import ./bootstrap-files/mips64el.nix; + mipsel-linux = import ./bootstrap-files/mipsel.nix; + mips64el-linux = import + (if localSystem.isMips64n32 + then ./bootstrap-files/mips64el-n32.nix + else ./bootstrap-files/mips64el.nix); powerpc64le-linux = import ./bootstrap-files/powerpc64le.nix; riscv64-linux = import ./bootstrap-files/riscv64.nix; }; @@ -100,6 +100,29 @@ assert crossSystem == localSystem; let inherit (localSystem) system; + isFromNixpkgs = pkg: !(isFromBootstrapFiles pkg); + isFromBootstrapFiles = + pkg: pkg.passthru.isFromBootstrapFiles or false; + isBuiltByNixpkgsCompiler = + pkg: isFromNixpkgs pkg && isFromNixpkgs pkg.stdenv.cc.cc; + isBuiltByBootstrapFilesCompiler = + pkg: isFromNixpkgs pkg && isFromBootstrapFiles pkg.stdenv.cc.cc; + + commonGccOverrides = { + # Use a deterministically built compiler + # see https://github.com/NixOS/nixpkgs/issues/108475 for context + reproducibleBuild = true; + profiledCompiler = false; + + # It appears that libcc1 (which is not a g++ plugin; it is a gdb plugin) gets linked against + # the libstdc++ from the compiler that *built* g++, not the libstdc++ which was just built. + # This causes a reference chain from stdenv to the bootstrapFiles: + # + # stdenv -> gcc-lib -> xgcc-lib -> bootstrapFiles + # + disableGdbPlugin = true; + }; + commonPreHook = '' export NIX_ENFORCE_PURITY="''${NIX_ENFORCE_PURITY-1}" @@ -117,16 +140,14 @@ let # Download and unpack the bootstrap tools (coreutils, GCC, Glibc, ...). - bootstrapTools = import (if localSystem.libc == "musl" then ./bootstrap-tools-musl else ./bootstrap-tools) { + bootstrapTools = (import (if localSystem.libc == "musl" then ./bootstrap-tools-musl else ./bootstrap-tools) { inherit system bootstrapFiles; - extraAttrs = lib.optionalAttrs - config.contentAddressedByDefault - { - __contentAddressed = true; - outputHashAlgo = "sha256"; - outputHashMode = "recursive"; - }; - }; + extraAttrs = lib.optionalAttrs config.contentAddressedByDefault { + __contentAddressed = true; + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + }; + }) // { passthru.isFromBootstrapFiles = true; }; getLibc = stage: stage.${localSystem.libc}; @@ -161,7 +182,7 @@ let cc = if prevStage.gcc-unwrapped == null then null - else lib.makeOverridable (import ../../build-support/cc-wrapper) { + else (lib.makeOverridable (import ../../build-support/cc-wrapper) { name = "${name}-gcc-wrapper"; nativeTools = false; nativeLibc = false; @@ -175,7 +196,12 @@ let inherit lib; inherit (prevStage) coreutils gnugrep; stdenvNoCC = prevStage.ccWrapperStdenv; - }; + }).overrideAttrs(a: lib.optionalAttrs (prevStage.gcc-unwrapped.passthru.isXgcc or false) { + # This affects only `xgcc` (the compiler which compiles the final compiler). + postFixup = (a.postFixup or "") + '' + echo "--sysroot=${lib.getDev (getLibc prevStage)}" >> $out/nix-support/cc-cflags + ''; + }); overrides = self: super: (overrides self super) // { fetchurl = thisStdenv.fetchurlBoot; }; }; @@ -186,7 +212,7 @@ let }; in - + assert bootstrapTools.passthru.isFromBootstrapFiles or false; # sanity check [ ({}: { @@ -200,9 +226,6 @@ in # Build a dummy stdenv with no GCC or working fetchurl. This is # because we need a stdenv to build the GCC wrapper and fetchurl. - # - # resulting stage0 stdenv: - # - coreutils, binutils, glibc, gcc: from bootstrapFiles (prevStage: stageFun prevStage { name = "bootstrap-stage0"; @@ -220,7 +243,7 @@ in ${localSystem.libc} = self.stdenv.mkDerivation { pname = "bootstrap-stage0-${localSystem.libc}"; strictDeps = true; - version = "bootstrap"; + version = "bootstrapFiles"; enableParallelBuilding = true; buildCommand = '' mkdir -p $out @@ -230,6 +253,7 @@ in '' + lib.optionalString (localSystem.libc == "musl") '' ln -s ${bootstrapTools}/include-libc $out/include ''; + passthru.isFromBootstrapFiles = true; }; gcc-unwrapped = bootstrapTools; binutils = import ../../build-support/bintools-wrapper { @@ -258,10 +282,14 @@ in # If we ever need to use a package from more than one stage back, we # simply re-export those packages in the middle stage(s) using the # overrides attribute and the inherit syntax. - # - # resulting stage1 stdenv: - # - coreutils, binutils, glibc, gcc: from bootstrapFiles - (prevStage: stageFun prevStage { + (prevStage: + # previous stage0 stdenv: + assert isFromBootstrapFiles prevStage.binutils.bintools; + assert isFromBootstrapFiles prevStage."${localSystem.libc}"; + assert isFromBootstrapFiles prevStage.gcc-unwrapped; + assert isFromBootstrapFiles prevStage.coreutils; + assert isFromBootstrapFiles prevStage.gnugrep; + stageFun prevStage { name = "bootstrap-stage1"; # Rebuild binutils to use from stage2 onwards. @@ -271,7 +299,7 @@ in }; inherit (prevStage) ccWrapperStdenv - gcc-unwrapped coreutils gnugrep; + gcc-unwrapped coreutils gnugrep binutils; ${localSystem.libc} = getLibc prevStage; @@ -284,21 +312,101 @@ in }; }) + # First rebuild of gcc; this is linked against all sorts of junk + # from the bootstrap-files, but we only care about the code that + # this compiler *emits*. The `gcc` binary produced in this stage + # is not part of the final stdenv. + (prevStage: + assert isBuiltByBootstrapFilesCompiler prevStage.binutils-unwrapped; + assert isFromBootstrapFiles prevStage."${localSystem.libc}"; + assert isFromBootstrapFiles prevStage.gcc-unwrapped; + assert isFromBootstrapFiles prevStage.coreutils; + assert isFromBootstrapFiles prevStage.gnugrep; + stageFun prevStage { + name = "bootstrap-stage-xgcc"; + overrides = final: prev: { + inherit (prevStage) ccWrapperStdenv coreutils gnugrep gettext bison texinfo zlib gnum4 perl; + patchelf = bootstrapTools; + ${localSystem.libc} = getLibc prevStage; + gmp = prev.gmp.override { cxx = false; }; + gcc-unwrapped = + (prev.gcc-unwrapped.override (commonGccOverrides // { + # The most logical name for this package would be something like + # "gcc-stage1". Unfortunately "stage" is already reserved for the + # layers of stdenv, so using "stage" in the name of this package + # would cause massive confusion. + # + # Gcc calls its "stage1" compiler `xgcc` (--disable-bootstrap results + # in `xgcc` being copied to $prefix/bin/gcc). So we imitate that. + # + name = "xgcc"; + + # xgcc uses ld linked against nixpkgs' glibc and gcc built + # against bootstrapTools glibc. We can't allow loading + # $out/libexec/gcc/x86_64-unknown-linux-gnu/13.0.1/liblto_plugin.so + # to mix libc.so: + # ...-binutils-patchelfed-ld-2.40/bin/ld: ...-xgcc-13.0.0/libexec/gcc/x86_64-unknown-linux-gnu/13.0.1/liblto_plugin.so: + # error loading plugin: ...-bootstrap-tools/lib/libpthread.so.0: undefined symbol: __libc_vfork, version GLIBC_PRIVATE + enableLTO = false; + })).overrideAttrs (a: { + + # This signals to cc-wrapper (as overridden above in this file) to add `--sysroot` + # to `$out/nix-support/cc-cflags`. + passthru = a.passthru // { isXgcc = true; }; + + # Gcc will look for the C library headers in + # + # ${with_build_sysroot}${native_system_header_dir} + # + # The ordinary gcc expression sets `--with-build-sysroot=/` and sets + # `native-system-header-dir` to `"${lib.getDev stdenv.cc.libc}/include`. + # + # Unfortunately the value of "--with-native-system-header-dir=" gets "burned in" to the + # compiler, and it is quite difficult to get the compiler to change or ignore it + # afterwards. On the other hand, the `sysroot` is very easy to change; you can just pass + # a `--sysroot` flag to `gcc`. + # + # So we override the expression to remove the default settings for these flags, and + # replace them such that the concatenated value will be the same as before, but we split + # the value between the two variables differently: `--native-system-header-dir=/include`, + # and `--with-build-sysroot=${lib.getDev stdenv.cc.libc}`. + # + configureFlags = (a.configureFlags or []) ++ [ + "--with-native-system-header-dir=/include" + "--with-build-sysroot=${lib.getDev final.stdenv.cc.libc}" + ]; + + # This is a separate phase because gcc assembles its phase scripts + # in bash instead of nix (we should fix that). + preFixupPhases = (a.preFixupPhases or []) ++ [ "preFixupXgccPhase" ]; + + # This is needed to prevent "error: cycle detected in build of '...-xgcc-....drv' + # in the references of output 'lib' from output 'out'" + preFixupXgccPhase = '' + find $lib/lib/ -name \*.so\* -exec patchelf --shrink-rpath {} \; || true + ''; + }); + }; + }) # 2nd stdenv that contains our own rebuilt binutils and is used for # compiling our own Glibc. # - # resulting stage2 stdenv: - # - coreutils, glibc, gcc: from bootstrapFiles - # - binutils: from nixpkgs, built by bootstrapFiles toolchain - (prevStage: stageFun prevStage { + (prevStage: + # previous stage1 stdenv: + assert isBuiltByBootstrapFilesCompiler prevStage.binutils-unwrapped; + assert isFromBootstrapFiles prevStage."${localSystem.libc}"; + assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped; + assert isFromBootstrapFiles prevStage.coreutils; + assert isFromBootstrapFiles prevStage.gnugrep; + stageFun prevStage { name = "bootstrap-stage2"; overrides = self: super: { inherit (prevStage) ccWrapperStdenv gettext gcc-unwrapped coreutils gnugrep - perl gnum4 bison; + perl gnum4 bison texinfo which; dejagnu = super.dejagnu.overrideAttrs (a: { doCheck = false; } ); # We need libidn2 and its dependency libunistring as glibc dependency. @@ -334,6 +442,7 @@ in bintools = self.stdenvNoCC.mkDerivation { pname = prevStage.bintools.bintools.pname + "-patchelfed-ld"; inherit (prevStage.bintools.bintools) version; + passthru = { inherit (prevStage.bintools.passthru) isFromBootstrapFiles; }; enableParallelBuilding = true; dontUnpack = true; dontBuild = true; @@ -349,6 +458,14 @@ in ''; }; }; + + # TODO(amjoseph): It is not yet entirely clear why this is necessary. + # Something strange is going on with xgcc and libstdc++ on pkgsMusl. + patchelf = super.patchelf.overrideAttrs(previousAttrs: + lib.optionalAttrs super.stdenv.hostPlatform.isMusl { + NIX_CFLAGS_COMPILE = (previousAttrs.NIX_CFLAGS_COMPILE or "") + " -static-libstdc++"; + }); + }; # `libtool` comes with obsolete config.sub/config.guess that don't recognize Risc-V. @@ -360,36 +477,35 @@ in # Construct a third stdenv identical to the 2nd, except that this # one uses the rebuilt Glibc from stage2. It still uses the recent # binutils and rest of the bootstrap tools, including GCC. - # - # resulting stage3 stdenv: - # - coreutils, gcc: from bootstrapFiles - # - glibc, binutils: from nixpkgs, built by bootstrapFiles toolchain - (prevStage: stageFun prevStage { + (prevStage: + # previous stage2 stdenv: + assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.${localSystem.libc}; + assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped; + assert isFromBootstrapFiles prevStage.coreutils; + assert isFromBootstrapFiles prevStage.gnugrep; + assert lib.all isBuiltByNixpkgsCompiler (with prevStage; [ gmp isl_0_20 libmpc mpfr ]); + stageFun prevStage { name = "bootstrap-stage3"; overrides = self: super: rec { inherit (prevStage) ccWrapperStdenv binutils coreutils gnugrep gettext - perl patchelf linuxHeaders gnum4 bison libidn2 libunistring; + perl patchelf linuxHeaders gnum4 bison libidn2 libunistring libxcrypt; + # We build a special copy of libgmp which doesn't use libstdc++, because + # xgcc++'s libstdc++ references the bootstrap-files (which is what + # compiles xgcc++). + gmp = super.gmp.override { cxx = false; }; + } // { ${localSystem.libc} = getLibc prevStage; - gcc-unwrapped = - let makeStaticLibrariesAndMark = pkg: - lib.makeOverridable (pkg.override { stdenv = self.makeStaticLibraries self.stdenv; }) - .overrideAttrs (a: { pname = "${a.pname}-stage3"; }); - in super.gcc-unwrapped.override { - # Link GCC statically against GMP etc. This makes sense because - # these builds of the libraries are only used by GCC, so it - # reduces the size of the stdenv closure. - gmp = makeStaticLibrariesAndMark super.gmp; - mpfr = makeStaticLibrariesAndMark super.mpfr; - libmpc = makeStaticLibrariesAndMark super.libmpc; - isl = makeStaticLibrariesAndMark super.isl_0_20; - # Use a deterministically built compiler - # see https://github.com/NixOS/nixpkgs/issues/108475 for context - reproducibleBuild = true; - profiledCompiler = false; - }; + gcc-unwrapped = (super.gcc-unwrapped.override (commonGccOverrides // { + inherit (prevStage) which; + } + )).overrideAttrs (a: { + # so we can add them to allowedRequisites below + passthru = a.passthru // { inherit (self) gmp mpfr libmpc isl; }; + }); }; extraNativeBuildInputs = [ prevStage.patchelf ] ++ # Many tarballs come with obsolete config.sub/config.guess that don't recognize aarch64. @@ -401,17 +517,14 @@ in # Construct a fourth stdenv that uses the new GCC. But coreutils is # still from the bootstrap tools. # - # resulting stage4 stdenv: - # - coreutils: from bootstrapFiles - # - glibc, binutils: from nixpkgs, built by bootstrapFiles toolchain - # - gcc: from nixpkgs, built by bootstrapFiles toolchain. Can assume - # it has almost no code from bootstrapTools as gcc bootstraps - # internally. The only exceptions are crt files from glibc - # built by bootstrapTools used to link executables and libraries, - # and the bootstrapTools-built, statically-linked - # lib{mpfr,mpc,gmp,isl}.a which are linked into the final gcc - # (see commit cfde88976ba4cddd01b1bb28b40afd12ea93a11d). - (prevStage: stageFun prevStage { + (prevStage: + # previous stage3 stdenv: + assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.${localSystem.libc}; + assert isBuiltByNixpkgsCompiler prevStage.gcc-unwrapped; + assert isFromBootstrapFiles prevStage.coreutils; + assert isFromBootstrapFiles prevStage.gnugrep; + stageFun prevStage { name = "bootstrap-stage4"; overrides = self: super: { @@ -430,11 +543,6 @@ in }; }; - # force gmp to rebuild so we have the option of dynamically linking - # libgmp without creating a reference path from: - # stage5.gcc -> stage4.coreutils -> stage3.glibc -> bootstrap - gmp = lib.makeOverridable (super.gmp.override { stdenv = self.stdenv; }).overrideAttrs (a: { pname = "${a.pname}-stage4"; }); - # To allow users' overrides inhibit dependencies too heavy for # bootstrap, like guile: https://github.com/NixOS/nixpkgs/issues/181188 gnumake = super.gnumake.override { inBootstrap = true; }; @@ -468,17 +576,15 @@ in # dependency (`nix-store -qR') on bootstrapTools or the first # binutils built. # - # resulting stage5 (final) stdenv: - # - coreutils, binutils: from nixpkgs, built by nixpkgs toolchain - # - glibc: from nixpkgs, built by bootstrapFiles toolchain - # - gcc: from nixpkgs, built by bootstrapFiles toolchain. Can assume - # it has almost no code from bootstrapTools as gcc bootstraps - # internally. The only exceptions are crt files from glibc - # built by bootstrapTools used to link executables and libraries, - # and the bootstrapTools-built, statically-linked - # lib{mpfr,mpc,gmp,isl}.a which are linked into the final gcc - # (see commit cfde88976ba4cddd01b1bb28b40afd12ea93a11d). - (prevStage: { + (prevStage: + # previous stage4 stdenv; see stage3 comment regarding gcc, + # which applies here as well. + assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.${localSystem.libc}; + assert isBuiltByNixpkgsCompiler prevStage.gcc-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.coreutils; + assert isBuiltByNixpkgsCompiler prevStage.gnugrep; + { inherit config overlays; stdenv = import ../generic rec { name = "stdenv-linux"; @@ -525,11 +631,15 @@ in ) # More complicated cases ++ (map (x: getOutput x (getLibc prevStage)) [ "out" "dev" "bin" ] ) - ++ [ /*propagated from .dev*/ linuxHeaders - binutils gcc gcc.cc gcc.cc.lib gcc.expand-response-params + ++ [ linuxHeaders # propagated from .dev + binutils gcc gcc.cc gcc.cc.lib gcc.expand-response-params gcc.cc.libgcc glibc.passthru.libgcc ] - ++ lib.optionals (!localSystem.isx86 || localSystem.libc == "musl") - [ prevStage.updateAutotoolsGnuConfigScriptsHook prevStage.gnu-config ]; + ++ lib.optionals (!localSystem.isx86 || localSystem.libc == "musl") + [ prevStage.updateAutotoolsGnuConfigScriptsHook prevStage.gnu-config ] + ++ (with gcc-unwrapped.passthru; [ + gmp libmpc mpfr isl + ]) + ; overrides = self: super: { inherit (prevStage) @@ -554,4 +664,14 @@ in }; }) + # This "no-op" stage is just a place to put the assertions about stage5. + (prevStage: + # previous stage5 stdenv; see stage3 comment regarding gcc, + # which applies here as well. + assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.${localSystem.libc}; + assert isBuiltByNixpkgsCompiler prevStage.gcc-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.coreutils; + assert isBuiltByNixpkgsCompiler prevStage.gnugrep; + { inherit (prevStage) config overlays stdenv; }) ] diff --git a/pkgs/stdenv/linux/make-bootstrap-tools.nix b/pkgs/stdenv/linux/make-bootstrap-tools.nix index 3aa7f6a3df537..091130ebf93a8 100644 --- a/pkgs/stdenv/linux/make-bootstrap-tools.nix +++ b/pkgs/stdenv/linux/make-bootstrap-tools.nix @@ -2,6 +2,10 @@ let libc = pkgs.stdenv.cc.libc; + patchelf = pkgs.patchelf.overrideAttrs(previousAttrs: { + NIX_CFLAGS_COMPILE = (previousAttrs.NIX_CFLAGS_COMPILE or []) ++ [ "-static-libgcc" "-static-libstdc++" ]; + NIX_CFLAGS_LINK = (previousAttrs.NIX_CFLAGS_LINK or []) ++ [ "-static-libgcc" "-static-libstdc++" ]; + }); in with pkgs; rec { @@ -127,7 +131,7 @@ in with pkgs; rec { cp -d ${bootGCC.out}/bin/gcc $out/bin cp -d ${bootGCC.out}/bin/cpp $out/bin cp -d ${bootGCC.out}/bin/g++ $out/bin - cp -d ${bootGCC.lib}/lib/libgcc_s.so* $out/lib + cp ${bootGCC.lib}/lib/libgcc_s.so* $out/lib cp -d ${bootGCC.lib}/lib/libstdc++.so* $out/lib cp -d ${bootGCC.out}/lib/libssp.a* $out/lib cp -d ${bootGCC.out}/lib/libssp_nonshared.a $out/lib @@ -149,6 +153,7 @@ in with pkgs; rec { rm -rf $out/include/c++/*/ext/parallel cp -d ${gmpxx.out}/lib/libgmp*.so* $out/lib + cp -d ${isl.out}/lib/libisl*.so* $out/lib cp -d ${mpfr.out}/lib/libmpfr*.so* $out/lib cp -d ${libmpc.out}/lib/libmpc*.so* $out/lib cp -d ${zlib.out}/lib/libz.so* $out/lib |