Merge pull request #306095 from K900/you-can-just-leave

Tarball job optimizations

3 files changed, 29 insertions, 38 deletions

diff --git a/pkgs/top-level/make-tarball.nix b/pkgs/top-level/make-tarball.nix
index 17df40fa0fb16..78b90082547be 100644
--- a/pkgs/top-level/make-tarball.nix
+++ b/pkgs/top-level/make-tarball.nix
@@ -1,10 +1,5 @@
-/* Hydra job to build a tarball for Nixpkgs from a Git checkout.  It
-   also builds the documentation and tests whether the Nix expressions
-   evaluate correctly. */
-
 { nixpkgs
 , officialRelease
-, supportedSystems
 , pkgs ? import nixpkgs.outPath {}
 , nix ? pkgs.nix
 , lib-tests ? import ../../lib/tests/release.nix { inherit pkgs; }
@@ -32,54 +27,50 @@ pkgs.releaseTools.sourceTarball {
     echo "git-revision is $(cat .git-revision)"
   '';
 
-  requiredSystemFeatures = [ "big-parallel" ]; # 1 thread but ~36G RAM (!) see #227945
+  dontUnpack = true;
 
   dontBuild = false;
 
   doCheck = true;
 
   checkPhase = ''
-    set -o pipefail
-
-    export NIX_STATE_DIR=$TMPDIR
-    export NIX_PATH=nixpkgs=$TMPDIR/barf.nix
-    opts=(--option build-users-group "")
-    nix-store --init
+    echo "generating packages.json"
 
-    echo "checking eval-release.nix"
-    nix-instantiate --eval --strict --show-trace ./maintainers/scripts/eval-release.nix > /dev/null
+    (
+      echo -n '{"version":2,"packages":'
+      NIX_STATE_DIR=$TMPDIR NIX_PATH= nix-env -f $src -qa --meta --json --show-trace --arg config 'import ${./packages-config.nix}'
+      echo -n '}'
+    ) | sed "s|$src/||g" | jq -c > packages.json
 
-    echo "checking find-tarballs.nix"
-    nix-instantiate --readonly-mode --eval --strict --show-trace --json \
-       ./maintainers/scripts/find-tarballs.nix \
-      --arg expr 'import ./maintainers/scripts/all-tarballs.nix' > $TMPDIR/tarballs.json
-    nrUrls=$(jq -r '.[].url' < $TMPDIR/tarballs.json | wc -l)
-    echo "found $nrUrls URLs"
-    if [ "$nrUrls" -lt 10000 ]; then
-      echo "suspiciously low number of URLs"
+    # Arbitrary number. The index has ~115k packages as of April 2024.
+    if [ $(jq -r '.packages | length' < packages.json) -lt 100000 ]; then
+      echo "ERROR: not enough packages in the search index, bailing out!"
       exit 1
     fi
 
-    echo "generating packages.json"
-    mkdir -p $out/nix-support
-    echo -n '{"version":2,"packages":' > tmp
-    nix-env -f . -I nixpkgs=$src -qa --meta --json --show-trace --arg config 'import ${./packages-config.nix}' "''${opts[@]}" >> tmp
-    echo -n '}' >> tmp
     packages=$out/packages.json.br
-    < tmp sed "s|$(pwd)/||g" | jq -c | brotli -9 > $packages
-    rm tmp
 
+    mkdir -p $out/nix-support
+    brotli -9 < packages.json > $packages
     echo "file json-br $packages" >> $out/nix-support/hydra-build-products
   '';
 
   distPhase = ''
     mkdir -p $out/tarballs
-    mkdir ../$releaseName
-    cp -prd . ../$releaseName
-    (cd .. && tar cfa $out/tarballs/$releaseName.tar.xz $releaseName) || false
+    XZ_OPT="-T0" tar \
+      --create \
+      --xz \
+      --file=$out/tarballs/$releaseName.tar.xz \
+      --absolute-names \
+      --transform="s|^$src|$releaseName|g" \
+      --transform="s|^$(pwd)|$releaseName|g" \
+      --owner=0 \
+      --group=0 \
+      --numeric-owner \
+      --format=gnu \
+      --sort=name \
+      --mtime="@$SOURCE_DATE_EPOCH" \
+      --mode=ug+w \
+      $src $(pwd)/{.version-suffix,.git-revision}
   '';
-
-  meta = {
-    maintainers = [ ];
-  };
 }
diff --git a/pkgs/top-level/release-small.nix b/pkgs/top-level/release-small.nix
index 2b143248948fa..0212464acb62d 100644
--- a/pkgs/top-level/release-small.nix
+++ b/pkgs/top-level/release-small.nix
@@ -18,7 +18,7 @@ in
 {
 
   tarball = import ./make-tarball.nix {
-    inherit nixpkgs supportedSystems;
+    inherit nixpkgs;
     officialRelease = false;
   };
 
diff --git a/pkgs/top-level/release.nix b/pkgs/top-level/release.nix
index 2da2d7d64675c..d11d1d1dbd43b 100644
--- a/pkgs/top-level/release.nix
+++ b/pkgs/top-level/release.nix
@@ -78,7 +78,7 @@ let
   ] (arch: elem "${arch}-darwin" supportedSystems);
 
   nonPackageJobs =
-    { tarball = import ./make-tarball.nix { inherit pkgs nixpkgs officialRelease supportedSystems; };
+    { tarball = import ./make-tarball.nix { inherit pkgs nixpkgs officialRelease; };
 
       release-checks = import ./nixpkgs-basic-release-checks.nix { inherit pkgs nixpkgs supportedSystems; };