trivial-builders: Add writeStringReferencesToFile

Add writeStringReferencesToFile, a builder which extracts a string's
references to derivations and paths and writes them to a text file,
removing the input string itself from the dependency graph. This is
useful when you want to make a derivation depend on the string's
references, but not its content (to avoid unnecessary rebuilds, for
example).

3 files changed, 109 insertions, 5 deletions

diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix
index f06d2136b8c6c..58cdeb269d58c 100644
--- a/pkgs/build-support/trivial-builders.nix
+++ b/pkgs/build-support/trivial-builders.nix
@@ -465,6 +465,88 @@ rec {
     '';
 
 
+  /*
+   * Extract a string's references to derivations and paths (its
+   * context) and write them to a text file, removing the input string
+   * itself from the dependency graph. This is useful when you want to
+   * make a derivation depend on the string's references, but not its
+   * contents (to avoid unnecessary rebuilds, for example).
+   *
+   * Note that this only works as intended on Nix >= 2.3.
+   */
+  writeStringReferencesToFile = string:
+    /*
+    * The basic operation this performs is to copy the string context
+    * from `string' to a second string and wrap that string in a
+    * derivation. However, that alone is not enough, since nothing in the
+    * string refers to the output paths of the derivations/paths in its
+    * context, meaning they'll be considered build-time dependencies and
+    * removed from the wrapper derivation's closure. Putting the
+    * necessary output paths in the new string is however not very
+    * straightforward - the attrset returned by `getContext' contains
+    * only references to derivations' .drv-paths, not their output
+    * paths. In order to "convert" them, we try to extract the
+    * corresponding paths from the original string using regex.
+    */
+    let
+      # Taken from https://github.com/NixOS/nix/blob/130284b8508dad3c70e8160b15f3d62042fc730a/src/libutil/hash.cc#L84
+      nixHashChars = "0123456789abcdfghijklmnpqrsvwxyz";
+      context = builtins.getContext string;
+      derivations = lib.filterAttrs (n: v: v ? outputs) context;
+      # Objects copied from outside of the store, such as paths and
+      # `builtins.fetch*`ed ones
+      sources = lib.attrNames (lib.filterAttrs (n: v: v ? path) context);
+      packages =
+        lib.mapAttrs'
+          (name: value:
+            {
+              inherit value;
+              name = lib.head (builtins.match "${builtins.storeDir}/[${nixHashChars}]+-(.*)\.drv" name);
+            })
+          derivations;
+      # The syntax of output paths differs between outputs named `out`
+      # and other, explicitly named ones. For explicitly named ones,
+      # the output name is suffixed as `-name`, but `out` outputs
+      # aren't suffixed at all, and thus aren't easily distinguished
+      # from named output paths. Therefore, we find all the named ones
+      # first so we can use them to remove false matches when looking
+      # for `out` outputs (see the definition of `outputPaths`).
+      namedOutputPaths =
+        lib.flatten
+          (lib.mapAttrsToList
+            (name: value:
+              (map
+                (output:
+                  lib.filter
+                    lib.isList
+                    (builtins.split "(${builtins.storeDir}/[${nixHashChars}]+-${name}-${output})" string))
+                (lib.remove "out" value.outputs)))
+            packages);
+      # Only `out` outputs
+      outputPaths =
+        lib.flatten
+          (lib.mapAttrsToList
+            (name: value:
+              if lib.elem "out" value.outputs then
+                lib.filter
+                  (x: lib.isList x &&
+                      # If the matched path is in `namedOutputPaths`,
+                      # it's a partial match of an output path where
+                      # the output name isn't `out`
+                      lib.all (o: !lib.hasPrefix (lib.head x) o) namedOutputPaths)
+                  (builtins.split "(${builtins.storeDir}/[${nixHashChars}]+-${name})" string)
+              else
+                [])
+            packages);
+      allPaths = lib.concatStringsSep "\n" (lib.unique (sources ++ namedOutputPaths ++ outputPaths));
+      allPathsWithContext = builtins.appendContext allPaths context;
+    in
+      if builtins ? getContext then
+        writeText "string-references" allPathsWithContext
+      else
+        writeDirectReferencesToFile (writeText "string-file" string);
+
+
   /* Print an error message if the file with the specified name and
    * hash doesn't exist in the Nix store. This function should only
    * be used by non-redistributable software with an unfree license
diff --git a/pkgs/build-support/trivial-builders/test.nix b/pkgs/build-support/trivial-builders/test.nix
index 204fb54fca3db..420a0fd0114d0 100644
--- a/pkgs/build-support/trivial-builders/test.nix
+++ b/pkgs/build-support/trivial-builders/test.nix
@@ -38,11 +38,27 @@ nixosTest {
       DIRECT_REFS = invokeSamples ./test/invoke-writeDirectReferencesToFile.nix;
     };
   };
-  testScript = ''
-    machine.succeed("""
-      ${./test.sh} 2>/dev/console
-    """)
-  '';
+  testScript =
+    let
+      sample = import ./test/sample.nix { inherit pkgs; };
+      samplePaths = lib.unique (lib.attrValues sample);
+      sampleText = pkgs.writeText "sample-text" (lib.concatStringsSep "\n" samplePaths);
+      stringReferencesText =
+        pkgs.writeStringReferencesToFile
+          ((lib.concatMapStringsSep "fillertext"
+            (d: "${d}")
+            (lib.attrValues sample)) + ''
+              STORE=${builtins.storeDir};\nsystemctl start bar-foo.service
+            '');
+    in ''
+      machine.succeed("""
+        ${./test.sh} 2>/dev/console
+      """)
+      machine.succeed("""
+        echo >&2 Testing string references...
+        diff -U3 <(sort ${stringReferencesText}) <(sort ${sampleText})
+      """)
+    '';
   meta = {
     license = lib.licenses.mit; # nixpkgs license
     maintainers = with lib.maintainers; [
diff --git a/pkgs/build-support/trivial-builders/test/sample.nix b/pkgs/build-support/trivial-builders/test/sample.nix
index 27aee6b73dbed..807594d74bb3b 100644
--- a/pkgs/build-support/trivial-builders/test/sample.nix
+++ b/pkgs/build-support/trivial-builders/test/sample.nix
@@ -2,6 +2,7 @@
 let
   inherit (pkgs)
     figlet
+    zlib
     hello
     writeText
     ;
@@ -9,8 +10,13 @@ in
 {
   hello = hello;
   figlet = figlet;
+  zlib = zlib;
+  zlib-dev = zlib.dev;
   norefs = writeText "hi" "hello";
+  norefsDup = writeText "hi" "hello";
   helloRef = writeText "hi" "hello ${hello}";
+  helloRefDup = writeText "hi" "hello ${hello}";
+  path = ./invoke-writeReferencesToFile.nix;
   helloFigletRef = writeText "hi" "hello ${hello} ${figlet}";
   inherit (pkgs)
     emptyFile