Merge master into haskell-updates

19 files changed, 96 insertions, 92 deletions

diff --git a/pkgs/build-support/deterministic-uname/default.nix b/pkgs/build-support/deterministic-uname/default.nix
index 164136c937b92..6d150557aa9d6 100644
--- a/pkgs/build-support/deterministic-uname/default.nix
+++ b/pkgs/build-support/deterministic-uname/default.nix
@@ -39,6 +39,7 @@ substituteAll {
 
   meta = with lib; {
     description = "Print certain system information (hardcoded with lib/system values)";
+    mainProgram = "uname";
     longDescription = ''
       This package provides a replacement for `uname` whose output depends only
       on `stdenv.buildPlatform`.  It is meant to be used from within derivations.
diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix
index ea464ebea18c8..8e04944bc8100 100644
--- a/pkgs/build-support/docker/default.nix
+++ b/pkgs/build-support/docker/default.nix
@@ -29,7 +29,7 @@
 , tarsum
 , util-linux
 , vmTools
-, writeReferencesToFile
+, writeClosure
 , writeScript
 , writeShellScriptBin
 , writeText
@@ -630,7 +630,7 @@ rec {
           imageName = lib.toLower name;
           imageTag = lib.optionalString (tag != null) tag;
           inherit fromImage baseJson;
-          layerClosure = writeReferencesToFile layer;
+          layerClosure = writeClosure [ layer ];
           passthru.buildArgs = args;
           passthru.layer = layer;
           passthru.imageTag =
diff --git a/pkgs/build-support/docker/nix-prefetch-docker.nix b/pkgs/build-support/docker/nix-prefetch-docker.nix
index 61e917461ed9b..18accd135c04e 100644
--- a/pkgs/build-support/docker/nix-prefetch-docker.nix
+++ b/pkgs/build-support/docker/nix-prefetch-docker.nix
@@ -18,6 +18,7 @@ stdenv.mkDerivation {
 
   meta = with lib; {
     description = "Script used to obtain source hashes for dockerTools.pullImage";
+    mainProgram = "nix-prefetch-docker";
     maintainers = with maintainers; [ offline ];
     platforms = platforms.unix;
   };
diff --git a/pkgs/build-support/docker/tarsum.nix b/pkgs/build-support/docker/tarsum.nix
index 734c6b3d5aeb8..f62a8d49389aa 100644
--- a/pkgs/build-support/docker/tarsum.nix
+++ b/pkgs/build-support/docker/tarsum.nix
@@ -39,4 +39,5 @@ stdenv.mkDerivation {
   };
 
   meta.platforms = go.meta.platforms;
+  meta.mainProgram = "tarsum";
 }
diff --git a/pkgs/build-support/java/canonicalize-jar.nix b/pkgs/build-support/java/canonicalize-jar.nix
deleted file mode 100644
index 1edd9a6e0d20f..0000000000000
--- a/pkgs/build-support/java/canonicalize-jar.nix
+++ /dev/null
@@ -1,9 +0,0 @@
-{ substituteAll, unzip, zip }:
-
-substituteAll {
-  name = "canonicalize-jar";
-  src = ./canonicalize-jar.sh;
-
-  unzip = "${unzip}/bin/unzip";
-  zip = "${zip}/bin/zip";
-}
diff --git a/pkgs/build-support/java/canonicalize-jar.sh b/pkgs/build-support/java/canonicalize-jar.sh
deleted file mode 100644
index af010bcd2b26c..0000000000000
--- a/pkgs/build-support/java/canonicalize-jar.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-# Canonicalize the manifest & repack with deterministic timestamps.
-canonicalizeJar() {
-    local input='' outer=''
-    input="$(realpath -sm -- "$1")"
-    outer="$(pwd)"
-    # -qq: even quieter
-    @unzip@ -qq "$input" -d "$input-tmp"
-    canonicalizeJarManifest "$input-tmp/META-INF/MANIFEST.MF"
-    # Sets all timestamps to Jan 1 1980, the earliest mtime zips support.
-    find -- "$input-tmp" -exec touch -t 198001010000.00 {} +
-    rm "$input"
-    pushd "$input-tmp" 2>/dev/null
-    # -q|--quiet, -r|--recurse-paths
-    # -o|--latest-time: canonicalizes overall archive mtime
-    # -X|--no-extra: don't store platform-specific extra file attribute fields
-    @zip@ -qroX "$outer/tmp-out.jar" . 2> /dev/null
-    popd 2>/dev/null
-    rm -rf "$input-tmp"
-    mv "$outer/tmp-out.jar" "$input"
-}
-
-# See also the Java specification's JAR requirements:
-# https://docs.oracle.com/javase/8/docs/technotes/guides/jar/jar.html#Notes_on_Manifest_and_Signature_Files
-canonicalizeJarManifest() {
-    local input=''
-    input="$(realpath -sm -- "$1")"
-    (head -n 1 "$input" && tail -n +2 "$input" | sort | grep -v '^\s*$') > "$input-tmp"
-    mv "$input-tmp" "$input"
-}
diff --git a/pkgs/build-support/kernel/make-initrd-ng-tool.nix b/pkgs/build-support/kernel/make-initrd-ng-tool.nix
index b1fbee92b32e5..5e08c091c0549 100644
--- a/pkgs/build-support/kernel/make-initrd-ng-tool.nix
+++ b/pkgs/build-support/kernel/make-initrd-ng-tool.nix
@@ -11,6 +11,7 @@ rustPlatform.buildRustPackage {
 
   meta = {
     description = "Tool for copying binaries and their dependencies";
+    mainProgram = "make-initrd-ng";
     maintainers = with lib.maintainers; [ das_j elvishjerricco k900 lheckemann ];
     license = lib.licenses.mit;
   };
diff --git a/pkgs/build-support/node/fetch-npm-deps/default.nix b/pkgs/build-support/node/fetch-npm-deps/default.nix
index 725f9ba3bb017..373d63cc59b8f 100644
--- a/pkgs/build-support/node/fetch-npm-deps/default.nix
+++ b/pkgs/build-support/node/fetch-npm-deps/default.nix
@@ -141,6 +141,7 @@
 
     meta = with lib; {
       description = "Prefetch dependencies from npm (for use with `fetchNpmDeps`)";
+      mainProgram = "prefetch-npm-deps";
       maintainers = with maintainers; [ lilyinstarlight winter ];
       license = licenses.mit;
     };
diff --git a/pkgs/build-support/oci-tools/default.nix b/pkgs/build-support/oci-tools/default.nix
index 67e081522d64c..1f5507f2eb753 100644
--- a/pkgs/build-support/oci-tools/default.nix
+++ b/pkgs/build-support/oci-tools/default.nix
@@ -1,4 +1,4 @@
-{ lib, writeText, runCommand, writeReferencesToFile }:
+{ lib, writeText, runCommand, writeClosure }:
 
 {
   buildContainer =
@@ -72,7 +72,7 @@
       set -o pipefail
       mkdir -p $out/rootfs/{dev,proc,sys}
       cp ${config} $out/config.json
-      xargs tar c < ${writeReferencesToFile args} | tar -xC $out/rootfs/
+      xargs tar c < ${writeClosure args} | tar -xC $out/rootfs/
     '';
 }
 
diff --git a/pkgs/build-support/references-by-popularity/closure-graph.py b/pkgs/build-support/references-by-popularity/closure-graph.py
index 579f3b041fa83..4f8efd42ed816 100644
--- a/pkgs/build-support/references-by-popularity/closure-graph.py
+++ b/pkgs/build-support/references-by-popularity/closure-graph.py
@@ -8,8 +8,8 @@
 # and how deep in the tree they live. Equally-"popular" paths are then
 # sorted by name.
 #
-# The existing writeReferencesToFile prints the paths in a simple
-# ascii-based sorting of the paths.
+# The existing writeClosure prints the paths in a simple ascii-based
+# sorting of the paths.
 #
 # Sorting the paths by graph improves the chances that the difference
 # between two builds appear near the end of the list, instead of near
diff --git a/pkgs/build-support/remove-references-to/default.nix b/pkgs/build-support/remove-references-to/default.nix
index f022611ef9132..1277cdb04fd7c 100644
--- a/pkgs/build-support/remove-references-to/default.nix
+++ b/pkgs/build-support/remove-references-to/default.nix
@@ -32,4 +32,5 @@ stdenv.mkDerivation {
   inherit (builtins) storeDir;
   shell = lib.getBin shell + (shell.shellPath or "");
   signingUtils = if darwinCodeSign then signingUtils else null;
+  meta.mainProgram = "remove-references-to";
 }
diff --git a/pkgs/build-support/setup-hooks/canonicalize-jars.sh b/pkgs/build-support/setup-hooks/canonicalize-jars.sh
deleted file mode 100644
index 5137bfc94b011..0000000000000
--- a/pkgs/build-support/setup-hooks/canonicalize-jars.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-# This setup hook causes the fixup phase to repack all JAR files in a
-# canonical & deterministic fashion, e.g. resetting mtimes (like with normal
-# store files) and avoiding impure metadata.
-
-fixupOutputHooks+=('if [ -z "$dontCanonicalizeJars" -a -e "$prefix" ]; then canonicalizeJarsIn "$prefix"; fi')
-
-canonicalizeJarsIn() {
-  local dir="$1"
-  echo "canonicalizing jars in $dir"
-  dir="$(realpath -sm -- "$dir")"
-  while IFS= read -rd '' f; do
-    canonicalizeJar "$f"
-  done < <(find -- "$dir" -type f -name '*.jar' -print0)
-}
-
-source @canonicalize_jar@
diff --git a/pkgs/build-support/setup-hooks/strip-java-archives.sh b/pkgs/build-support/setup-hooks/strip-java-archives.sh
new file mode 100644
index 0000000000000..22322468f76dd
--- /dev/null
+++ b/pkgs/build-support/setup-hooks/strip-java-archives.sh
@@ -0,0 +1,16 @@
+# This setup hook makes the fixup phase to repack all java archives in a
+# deterministic fashion. The most important change being done is the resetting
+# of the modification times of the archive entries
+
+fixupOutputHooks+=('stripJavaArchivesIn $prefix')
+
+stripJavaArchivesIn() {
+    local dir="$1"
+    echo "stripping java archives in $dir"
+    find $dir -type f -regextype posix-egrep -regex ".*\.(jar|war|hpi|apk)$" -print0 |
+    while IFS= read -rd '' f; do
+        echo "stripping java archive $f"
+        strip-nondeterminism --type jar "$f"
+    done
+}
+
diff --git a/pkgs/build-support/singularity-tools/default.nix b/pkgs/build-support/singularity-tools/default.nix
index 8d7ad9e742a1a..c9e53a4cb706f 100644
--- a/pkgs/build-support/singularity-tools/default.nix
+++ b/pkgs/build-support/singularity-tools/default.nix
@@ -4,7 +4,7 @@
 , storeDir ? builtins.storeDir
 , writeScript
 , singularity
-, writeReferencesToFile
+, writeClosure
 , bash
 , vmTools
 , gawk
@@ -50,18 +50,13 @@ rec {
     }:
     let
       projectName = singularity.projectName or "singularity";
-      layer = mkLayer {
-        inherit name;
-        contents = contents ++ [ bash runScriptFile ];
-        inherit projectName;
-      };
       runAsRootFile = shellScript "run-as-root.sh" runAsRoot;
       runScriptFile = shellScript "run-script.sh" runScript;
       result = vmTools.runInLinuxVM (
         runCommand "${projectName}-image-${name}.img"
           {
             buildInputs = [ singularity e2fsprogs util-linux gawk ];
-            layerClosure = writeReferencesToFile layer;
+            layerClosure = writeClosure contents;
             preVM = vmTools.createEmptyImage {
               size = diskSize;
               fullName = "${projectName}-run-disk";
diff --git a/pkgs/build-support/trivial-builders/default.nix b/pkgs/build-support/trivial-builders/default.nix
index df81d67d868da..d7438923a54b8 100644
--- a/pkgs/build-support/trivial-builders/default.nix
+++ b/pkgs/build-support/trivial-builders/default.nix
@@ -1,4 +1,4 @@
-{ lib, stdenv, stdenvNoCC, lndir, runtimeShell, shellcheck-minimal }:
+{ lib, config, stdenv, stdenvNoCC, jq, lndir, runtimeShell, shellcheck-minimal }:
 
 let
   inherit (lib)
@@ -625,18 +625,22 @@ rec {
 
   # Docs in doc/build-helpers/trivial-build-helpers.chapter.md
   # See https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-writeReferencesToFile
-  writeReferencesToFile = path: runCommand "runtime-deps"
+  # TODO: Convert to throw after Nixpkgs 24.05 branch-off.
+  writeReferencesToFile = (if config.allowAliases then lib.warn else throw)
+    "writeReferencesToFile is deprecated in favour of writeClosure"
+    (path: writeClosure [ path ]);
+
+  # Docs in doc/build-helpers/trivial-build-helpers.chapter.md
+  # See https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-writeClosure
+  writeClosure = paths: runCommand "runtime-deps"
     {
-      exportReferencesGraph = [ "graph" path ];
+      # Get the cleaner exportReferencesGraph interface
+      __structuredAttrs = true;
+      exportReferencesGraph.graph = paths;
+      nativeBuildInputs = [ jq ];
     }
     ''
-      touch $out
-      while read path; do
-        echo $path >> $out
-        read dummy
-        read nrRefs
-        for ((i = 0; i < nrRefs; i++)); do read ref; done
-      done < graph
+      jq -r ".graph | map(.path) | sort | .[]" "$NIX_ATTRS_JSON_FILE" > "$out"
     '';
 
   # Docs in doc/build-helpers/trivial-build-helpers.chapter.md
diff --git a/pkgs/build-support/trivial-builders/test/default.nix b/pkgs/build-support/trivial-builders/test/default.nix
index 59dbba3f18410..f41372d922bb1 100644
--- a/pkgs/build-support/trivial-builders/test/default.nix
+++ b/pkgs/build-support/trivial-builders/test/default.nix
@@ -26,6 +26,9 @@ recurseIntoAttrs {
     then references
     else {};
   writeCBin = callPackage ./writeCBin.nix {};
+  writeClosure-union = callPackage ./writeClosure-union.nix {
+    inherit (references) samples;
+  };
   writeShellApplication = callPackage ./writeShellApplication.nix {};
   writeScriptBin = callPackage ./writeScriptBin.nix {};
   writeShellScript = callPackage ./write-shell-script.nix {};
diff --git a/pkgs/build-support/trivial-builders/test/references/default.nix b/pkgs/build-support/trivial-builders/test/references/default.nix
index 3e21c905321e4..928cc1d9461f0 100644
--- a/pkgs/build-support/trivial-builders/test/references/default.nix
+++ b/pkgs/build-support/trivial-builders/test/references/default.nix
@@ -12,7 +12,7 @@
 , cleanSamples ? lib.filterAttrs (n: lib.isStringLike)
   # Test targets
 , writeDirectReferencesToFile
-, writeReferencesToFile
+, writeClosure
 }:
 
 # -------------------------------------------------------------------------- #
@@ -46,8 +46,9 @@ let
   samplesToString = attrs:
     lib.concatMapStringsSep " " (name: "[${name}]=${lib.escapeShellArg "${attrs.${name}}"}") (builtins.attrNames attrs);
 
-  references = lib.mapAttrs (n: v: writeReferencesToFile v) samples;
+  closures = lib.mapAttrs (n: v: writeClosure [ v ]) samples;
   directReferences = lib.mapAttrs (n: v: writeDirectReferencesToFile v) samples;
+  collectiveClosure = writeClosure (lib.attrValues samples);
 
   testScriptBin = stdenvNoCC.mkDerivation (finalAttrs: {
     name = "references-test";
@@ -61,8 +62,9 @@ let
       mkdir -p "$out/bin"
       substitute "$src" "$out/bin/${finalAttrs.meta.mainProgram}" \
         --replace "@SAMPLES@" ${lib.escapeShellArg (samplesToString samples)} \
-        --replace "@REFERENCES@" ${lib.escapeShellArg (samplesToString references)} \
-        --replace "@DIRECT_REFS@" ${lib.escapeShellArg (samplesToString directReferences)}
+        --replace "@CLOSURES@" ${lib.escapeShellArg (samplesToString closures)} \
+        --replace "@DIRECT_REFS@" ${lib.escapeShellArg (samplesToString directReferences)} \
+        --replace "@COLLECTIVE_CLOSURE@" ${lib.escapeShellArg collectiveClosure}
       runHook postInstall
       chmod +x "$out/bin/${finalAttrs.meta.mainProgram}"
     '';
@@ -79,8 +81,9 @@ let
 
     passthru = {
       inherit
+        collectiveClosure
         directReferences
-        references
+        closures
         samples
         ;
     };
@@ -109,8 +112,9 @@ testers.nixosTest {
     '';
   passthru = {
     inherit
+      collectiveClosure
       directReferences
-      references
+      closures
       samples
       testScriptBin
       ;
diff --git a/pkgs/build-support/trivial-builders/test/references/references-test.sh b/pkgs/build-support/trivial-builders/test/references/references-test.sh
index 1b8f8e1504ec8..92e4467287edc 100755
--- a/pkgs/build-support/trivial-builders/test/references/references-test.sh
+++ b/pkgs/build-support/trivial-builders/test/references/references-test.sh
@@ -33,16 +33,17 @@ set -euo pipefail
 
 cd "$(dirname "${BASH_SOURCE[0]}")"  # nixpkgs root
 
-  # Injected by Nix (to avoid evaluating in a derivation)
-  # turn them into arrays
-  # shellcheck disable=SC2206 # deliberately unquoted
+  # Inject the path to compare from the Nix expression
+
+  # Associative Arrays
   declare -A samples=( @SAMPLES@ )
-  # shellcheck disable=SC2206 # deliberately unquoted
   declare -A directRefs=( @DIRECT_REFS@ )
-  # shellcheck disable=SC2206 # deliberately unquoted
-  declare -A references=( @REFERENCES@ )
+  declare -A closures=( @CLOSURES@ )
+
+  # Path string
+  collectiveClosure=@COLLECTIVE_CLOSURE@
 
-echo >&2 Testing direct references...
+echo >&2 Testing direct closures...
 for i in "${!samples[@]}"; do
   echo >&2 Checking "$i" "${samples[$i]}" "${directRefs[$i]}"
   diff -U3 \
@@ -52,10 +53,16 @@ done
 
 echo >&2 Testing closure...
 for i in "${!samples[@]}"; do
-  echo >&2 Checking "$i" "${samples[$i]}" "${references[$i]}"
+  echo >&2 Checking "$i" "${samples[$i]}" "${closures[$i]}"
   diff -U3 \
-    <(sort <"${references[$i]}") \
+    <(sort <"${closures[$i]}") \
     <(nix-store -q --requisites "${samples[$i]}" | sort)
 done
 
+echo >&2 Testing mixed closures...
+echo >&2 Checking all samples "(${samples[*]})" "$collectiveClosure"
+diff -U3 \
+  <(sort <"$collectiveClosure") \
+  <(nix-store -q --requisites "${samples[@]}" | sort)
+
 echo 'OK!'
diff --git a/pkgs/build-support/trivial-builders/test/writeClosure-union.nix b/pkgs/build-support/trivial-builders/test/writeClosure-union.nix
new file mode 100644
index 0000000000000..92a2bf9f0988d
--- /dev/null
+++ b/pkgs/build-support/trivial-builders/test/writeClosure-union.nix
@@ -0,0 +1,23 @@
+{ lib
+, runCommandLocal
+  # Test targets
+, writeClosure
+, samples
+}:
+runCommandLocal "test-trivial-builders-writeClosure-union" {
+  __structuredAttrs = true;
+  closures = lib.mapAttrs (n: v: writeClosure [ v ]) samples;
+  collectiveClosure = writeClosure (lib.attrValues samples);
+  inherit samples;
+  meta.maintainers = with lib.maintainers; [
+    ShamrockLee
+  ];
+} ''
+  set -eu -o pipefail
+  echo >&2 Testing mixed closures...
+  echo >&2 Checking all samples "(''${samples[*]})" "$collectiveClosure"
+  diff -U3 \
+    <(sort <"$collectiveClosure") \
+    <(cat "''${closures[@]}" | sort | uniq)
+  touch "$out"
+''