Merge pull request #230523 from hercules-ci/fast-nixos-test-eval

Fast nixos test eval

9 files changed, 245 insertions, 19 deletions

diff --git a/flake.nix b/flake.nix
index f9442d8ea2d2c..fa00bffcdf92f 100644
--- a/flake.nix
+++ b/flake.nix
@@ -57,6 +57,19 @@
 
       nixosModules = {
         notDetected = ./nixos/modules/installer/scan/not-detected.nix;
+
+        /*
+          Make the `nixpkgs.*` configuration read-only. Guarantees that `pkgs`
+          is the way you initialize it.
+
+          Example:
+
+              {
+                imports = [ nixpkgs.nixosModules.readOnlyPkgs ];
+                nixpkgs.pkgs = nixpkgs.legacyPackages.x86_64-linux;
+              }
+        */
+        readOnlyPkgs = ./nixos/modules/misc/nixpkgs/read-only.nix;
       };
     };
 }
diff --git a/lib/types.nix b/lib/types.nix
index e0da18a2febb9..373d0ce7876f9 100644
--- a/lib/types.nix
+++ b/lib/types.nix
@@ -476,6 +476,14 @@ rec {
       check = x: isDerivation x && hasAttr "shellPath" x;
     };
 
+    pkgs = addCheck
+      (unique { message = "A Nixpkgs pkgs set can not be merged with another pkgs set."; } attrs // {
+        name = "pkgs";
+        descriptionClass = "noun";
+        description = "Nixpkgs package set";
+      })
+      (x: (x._type or null) == "pkgs");
+
     path = mkOptionType {
       name = "path";
       descriptionClass = "noun";
diff --git a/nixos/doc/manual/development/option-types.section.md b/nixos/doc/manual/development/option-types.section.md
index 9e2ecb8e35626..9e156ebff9d3e 100644
--- a/nixos/doc/manual/development/option-types.section.md
+++ b/nixos/doc/manual/development/option-types.section.md
@@ -99,6 +99,10 @@ merging is handled.
     problems.
     :::
 
+`types.pkgs`
+
+:   A type for the top level Nixpkgs package set.
+
 ### Numeric types {#sec-option-types-numeric}
 
 `types.int`
diff --git a/nixos/lib/eval-config.nix b/nixos/lib/eval-config.nix
index 1e086271e5236..058ab7280ccc3 100644
--- a/nixos/lib/eval-config.nix
+++ b/nixos/lib/eval-config.nix
@@ -38,6 +38,8 @@ let pkgs_ = pkgs;
 in
 
 let
+  inherit (lib) optional;
+
   evalModulesMinimal = (import ./default.nix {
     inherit lib;
     # Implicit use of feature is noted in implementation.
@@ -47,15 +49,19 @@ let
   pkgsModule = rec {
     _file = ./eval-config.nix;
     key = _file;
-    config = {
-      # Explicit `nixpkgs.system` or `nixpkgs.localSystem` should override
-      # this.  Since the latter defaults to the former, the former should
-      # default to the argument. That way this new default could propagate all
-      # they way through, but has the last priority behind everything else.
-      nixpkgs.system = lib.mkIf (system != null) (lib.mkDefault system);
-
-      _module.args.pkgs = lib.mkIf (pkgs_ != null) (lib.mkForce pkgs_);
-    };
+    config = lib.mkMerge (
+      (optional (system != null) {
+        # Explicit `nixpkgs.system` or `nixpkgs.localSystem` should override
+        # this.  Since the latter defaults to the former, the former should
+        # default to the argument. That way this new default could propagate all
+        # they way through, but has the last priority behind everything else.
+        nixpkgs.system = lib.mkDefault system;
+      })
+      ++
+      (optional (pkgs_ != null) {
+        _module.args.pkgs = lib.mkForce pkgs_;
+      })
+    );
   };
 
   withWarnings = x:
diff --git a/nixos/lib/testing/nodes.nix b/nixos/lib/testing/nodes.nix
index c538ab468c526..6e439fd814db7 100644
--- a/nixos/lib/testing/nodes.nix
+++ b/nixos/lib/testing/nodes.nix
@@ -1,13 +1,22 @@
 testModuleArgs@{ config, lib, hostPkgs, nodes, ... }:
 
 let
-  inherit (lib) mkOption mkForce optional types mapAttrs mkDefault mdDoc;
-
-  system = hostPkgs.stdenv.hostPlatform.system;
+  inherit (lib)
+    literalExpression
+    literalMD
+    mapAttrs
+    mdDoc
+    mkDefault
+    mkIf
+    mkOption mkForce
+    optional
+    optionalAttrs
+    types
+    ;
 
   baseOS =
     import ../eval-config.nix {
-      inherit system;
+      system = null; # use modularly defined system
       inherit (config.node) specialArgs;
       modules = [ config.defaults ];
       baseModules = (import ../../modules/module-list.nix) ++
@@ -17,11 +26,17 @@ let
           ({ config, ... }:
             {
               virtualisation.qemu.package = testModuleArgs.config.qemu.package;
-
+            })
+          (optionalAttrs (!config.node.pkgsReadOnly) {
+            key = "nodes.nix-pkgs";
+            config = {
               # Ensure we do not use aliases. Ideally this is only set
               # when the test framework is used by Nixpkgs NixOS tests.
               nixpkgs.config.allowAliases = false;
-            })
+              # TODO: switch to nixpkgs.hostPlatform and make sure containers-imperative test still evaluates.
+              nixpkgs.system = hostPkgs.stdenv.hostPlatform.system;
+            };
+          })
           testModuleArgs.config.extraBaseModules
         ];
     };
@@ -68,6 +83,30 @@ in
       default = { };
     };
 
+    node.pkgs = mkOption {
+      description = mdDoc ''
+        The Nixpkgs to use for the nodes.
+
+        Setting this will make the `nixpkgs.*` options read-only, to avoid mistakenly testing with a Nixpkgs configuration that diverges from regular use.
+      '';
+      type = types.nullOr types.pkgs;
+      default = null;
+      defaultText = literalMD ''
+        `null`, so construct `pkgs` according to the `nixpkgs.*` options as usual.
+      '';
+    };
+
+    node.pkgsReadOnly = mkOption {
+      description = mdDoc ''
+        Whether to make the `nixpkgs.*` options read-only. This is only relevant when [`node.pkgs`](#test-opt-node.pkgs) is set.
+
+        Set this to `false` when any of the [`nodes`](#test-opt-nodes) needs to configure any of the `nixpkgs.*` options. This will slow down evaluation of your test a bit.
+      '';
+      type = types.bool;
+      default = config.node.pkgs != null;
+      defaultText = literalExpression ''node.pkgs != null'';
+    };
+
     node.specialArgs = mkOption {
       type = types.lazyAttrsOf types.raw;
       default = { };
@@ -100,5 +139,11 @@ in
         config.nodes;
 
     passthru.nodes = config.nodesCompat;
+
+    defaults = mkIf config.node.pkgsReadOnly {
+      nixpkgs.pkgs = config.node.pkgs;
+      imports = [ ../../modules/misc/nixpkgs/read-only.nix ];
+    };
+
   };
 }
diff --git a/nixos/modules/misc/nixpkgs.nix b/nixos/modules/misc/nixpkgs.nix
index 7f44c3f6f3f0e..55ec08acf4453 100644
--- a/nixos/modules/misc/nixpkgs.nix
+++ b/nixos/modules/misc/nixpkgs.nix
@@ -49,10 +49,10 @@ let
     merge = lib.mergeOneOption;
   };
 
-  pkgsType = mkOptionType {
-    name = "nixpkgs";
+  pkgsType = types.pkgs // {
+    # This type is only used by itself, so let's elaborate the description a bit
+    # for the purpose of documentation.
     description = "An evaluation of Nixpkgs; the top level attribute set of packages";
-    check = builtins.isAttrs;
   };
 
   # Whether `pkgs` was constructed by this module - not if nixpkgs.pkgs or
diff --git a/nixos/modules/misc/nixpkgs/read-only.nix b/nixos/modules/misc/nixpkgs/read-only.nix
new file mode 100644
index 0000000000000..2a783216a9d54
--- /dev/null
+++ b/nixos/modules/misc/nixpkgs/read-only.nix
@@ -0,0 +1,74 @@
+# A replacement for the traditional nixpkgs module, such that none of the modules
+# can add their own configuration. This ensures that the Nixpkgs configuration is
+# exactly as the user intends.
+# This may also be used as a performance optimization when evaluating multiple
+# configurations at once, with a shared `pkgs`.
+
+# This is a separate module, because merging this logic into the nixpkgs module
+# is too burdensome, considering that it is already burdened with legacy.
+# Moving this logic into a module does not lose any composition benefits, because
+# its purpose is not something that composes anyway.
+
+{ lib, config, ... }:
+
+let
+  cfg = config.nixpkgs;
+  inherit (lib) mkOption types;
+
+in
+{
+  disabledModules = [
+    ../nixpkgs.nix
+  ];
+  options = {
+    nixpkgs = {
+      pkgs = mkOption {
+        type = lib.types.pkgs;
+        description = lib.mdDoc ''The pkgs module argument.'';
+      };
+      config = mkOption {
+        internal = true;
+        type = types.unique { message = "nixpkgs.config is set to read-only"; } types.anything;
+        description = lib.mdDoc ''
+          The Nixpkgs `config` that `pkgs` was initialized with.
+        '';
+      };
+      overlays = mkOption {
+        internal = true;
+        type = types.unique { message = "nixpkgs.overlays is set to read-only"; } types.anything;
+        description = lib.mdDoc ''
+          The Nixpkgs overlays that `pkgs` was initialized with.
+        '';
+      };
+      hostPlatform = mkOption {
+        internal = true;
+        readOnly = true;
+        description = lib.mdDoc ''
+          The platform of the machine that is running the NixOS configuration.
+        '';
+      };
+      buildPlatform = mkOption {
+        internal = true;
+        readOnly = true;
+        description = lib.mdDoc ''
+          The platform of the machine that built the NixOS configuration.
+        '';
+      };
+      # NOTE: do not add the legacy options such as localSystem here. Let's keep
+      #       this module simple and let module authors upgrade their code instead.
+    };
+  };
+  config = {
+    _module.args.pkgs =
+      # find mistaken definitions
+      builtins.seq cfg.config
+      builtins.seq cfg.overlays
+      builtins.seq cfg.hostPlatform
+      builtins.seq cfg.buildPlatform
+      cfg.pkgs;
+    nixpkgs.config = cfg.pkgs.config;
+    nixpkgs.overlays = cfg.pkgs.overlays;
+    nixpkgs.hostPlatform = cfg.pkgs.stdenv.hostPlatform;
+    nixpkgs.buildPlatform = cfg.pkgs.stdenv.buildPlatform;
+  };
+}
diff --git a/nixos/modules/misc/nixpkgs/test.nix b/nixos/modules/misc/nixpkgs/test.nix
index a6d8877ae0700..0536cfc9624a2 100644
--- a/nixos/modules/misc/nixpkgs/test.nix
+++ b/nixos/modules/misc/nixpkgs/test.nix
@@ -1,3 +1,5 @@
+# [nixpkgs]$ nix-build -A nixosTests.nixpkgs --show-trace
+
 { evalMinimalConfig, pkgs, lib, stdenv }:
 let
   eval = mod: evalMinimalConfig {
@@ -27,6 +29,47 @@ let
     let
       uncheckedEval = lib.evalModules { modules = [ ../nixpkgs.nix module ]; };
     in map (ass: ass.message) (lib.filter (ass: !ass.assertion) uncheckedEval.config.assertions);
+
+  readOnlyUndefined = evalMinimalConfig {
+    imports = [ ./read-only.nix ];
+  };
+
+  readOnlyBad = evalMinimalConfig {
+    imports = [ ./read-only.nix ];
+    nixpkgs.pkgs = { };
+  };
+
+  readOnly = evalMinimalConfig {
+    imports = [ ./read-only.nix ];
+    nixpkgs.pkgs = pkgs;
+  };
+
+  readOnlyBadConfig = evalMinimalConfig {
+    imports = [ ./read-only.nix ];
+    nixpkgs.pkgs = pkgs;
+    nixpkgs.config.allowUnfree = true; # do in pkgs instead!
+  };
+
+  readOnlyBadOverlays = evalMinimalConfig {
+    imports = [ ./read-only.nix ];
+    nixpkgs.pkgs = pkgs;
+    nixpkgs.overlays = [ (_: _: {}) ]; # do in pkgs instead!
+  };
+
+  readOnlyBadHostPlatform = evalMinimalConfig {
+    imports = [ ./read-only.nix ];
+    nixpkgs.pkgs = pkgs;
+    nixpkgs.hostPlatform = "foo-linux"; # do in pkgs instead!
+  };
+
+  readOnlyBadBuildPlatform = evalMinimalConfig {
+    imports = [ ./read-only.nix ];
+    nixpkgs.pkgs = pkgs;
+    nixpkgs.buildPlatform = "foo-linux"; # do in pkgs instead!
+  };
+
+  throws = x: ! (builtins.tryEval x).success;
+
 in
 lib.recurseIntoAttrs {
   invokeNixpkgsSimple =
@@ -65,5 +108,21 @@ lib.recurseIntoAttrs {
         nixpkgs.pkgs = pkgs;
       } == [];
 
+
+    # Tests for the read-only.nix module
+    assert readOnly._module.args.pkgs.stdenv.hostPlatform.system == pkgs.stdenv.hostPlatform.system;
+    assert throws readOnlyBad._module.args.pkgs.stdenv;
+    assert throws readOnlyUndefined._module.args.pkgs.stdenv;
+    assert throws readOnlyBadConfig._module.args.pkgs.stdenv;
+    assert throws readOnlyBadOverlays._module.args.pkgs.stdenv;
+    assert throws readOnlyBadHostPlatform._module.args.pkgs.stdenv;
+    assert throws readOnlyBadBuildPlatform._module.args.pkgs.stdenv;
+    # read-only.nix does not provide legacy options, for the sake of simplicity
+    # If you're bothered by this, upgrade your configs to use the new *Platform
+    # options.
+    assert !readOnly.options.nixpkgs?system;
+    assert !readOnly.options.nixpkgs?localSystem;
+    assert !readOnly.options.nixpkgs?crossSystem;
+
     pkgs.emptyFile;
 }
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 95f8fd1c4e043..9f35dca5cc4f1 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -46,7 +46,7 @@ let
   inherit
     (rec {
       doRunTest = arg: ((import ../lib/testing-python.nix { inherit system pkgs; }).evalTest {
-        imports = [ arg ];
+        imports = [ arg readOnlyPkgs ];
       }).config.result;
       findTests = tree:
         if tree?recurseForDerivations && tree.recurseForDerivations
@@ -65,6 +65,23 @@ let
     runTestOn
     ;
 
+  # Using a single instance of nixpkgs makes test evaluation faster.
+  # To make sure we don't accidentally depend on a modified pkgs, we make the
+  # related options read-only. We need to test the right configuration.
+  #
+  # If your service depends on a nixpkgs setting, first try to avoid that, but
+  # otherwise, you can remove the readOnlyPkgs import and test your service as
+  # usual.
+  readOnlyPkgs =
+    # TODO: We currently accept this for nixosTests, so that the `pkgs` argument
+    #       is consistent with `pkgs` in `pkgs.nixosTests`. Can we reinitialize
+    #       it with `allowAliases = false`?
+    # warnIf pkgs.config.allowAliases "nixosTests: pkgs includes aliases."
+    {
+      _class = "nixosTest";
+      node.pkgs = pkgs;
+    };
+
 in {
 
   # Testing the test driver