Merge pull request #191540 from hercules-ci/nixosTest-modular

nixosTest: make modular

17 files changed, 789 insertions, 342 deletions

diff --git a/nixos/lib/build-vms.nix b/nixos/lib/build-vms.nix
deleted file mode 100644
index 18af49db17777..0000000000000
--- a/nixos/lib/build-vms.nix
+++ /dev/null
@@ -1,113 +0,0 @@
-{ system
-, # Use a minimal kernel?
-  minimal ? false
-, # Ignored
-  config ? null
-, # Nixpkgs, for qemu, lib and more
-  pkgs, lib
-, # !!! See comment about args in lib/modules.nix
-  specialArgs ? {}
-, # NixOS configuration to add to the VMs
-  extraConfigurations ? []
-}:
-
-with lib;
-
-rec {
-
-  inherit pkgs;
-
-  # Build a virtual network from an attribute set `{ machine1 =
-  # config1; ... machineN = configN; }', where `machineX' is the
-  # hostname and `configX' is a NixOS system configuration.  Each
-  # machine is given an arbitrary IP address in the virtual network.
-  buildVirtualNetwork =
-    nodes: let nodesOut = mapAttrs (n: buildVM nodesOut) (assignIPAddresses nodes); in nodesOut;
-
-
-  buildVM =
-    nodes: configurations:
-
-    import ./eval-config.nix {
-      inherit system specialArgs;
-      modules = configurations ++ extraConfigurations;
-      baseModules =  (import ../modules/module-list.nix) ++
-        [ ../modules/virtualisation/qemu-vm.nix
-          ../modules/testing/test-instrumentation.nix # !!! should only get added for automated test runs
-          { key = "no-manual"; documentation.nixos.enable = false; }
-          { key = "no-revision";
-            # Make the revision metadata constant, in order to avoid needless retesting.
-            # The human version (e.g. 21.05-pre) is left as is, because it is useful
-            # for external modules that test with e.g. testers.nixosTest and rely on that
-            # version number.
-            config.system.nixos.revision = mkForce "constant-nixos-revision";
-          }
-          { key = "nodes"; _module.args.nodes = nodes; }
-        ] ++ optional minimal ../modules/testing/minimal-kernel.nix;
-    };
-
-
-  # Given an attribute set { machine1 = config1; ... machineN =
-  # configN; }, sequentially assign IP addresses in the 192.168.1.0/24
-  # range to each machine, and set the hostname to the attribute name.
-  assignIPAddresses = nodes:
-
-    let
-
-      machines = attrNames nodes;
-
-      machinesNumbered = zipLists machines (range 1 254);
-
-      nodes_ = forEach machinesNumbered (m: nameValuePair m.fst
-        [ ( { config, nodes, ... }:
-            let
-              interfacesNumbered = zipLists config.virtualisation.vlans (range 1 255);
-              interfaces = forEach interfacesNumbered ({ fst, snd }:
-                nameValuePair "eth${toString snd}" { ipv4.addresses =
-                  [ { address = "192.168.${toString fst}.${toString m.snd}";
-                      prefixLength = 24;
-                  } ];
-                });
-
-              networkConfig =
-                { networking.hostName = mkDefault m.fst;
-
-                  networking.interfaces = listToAttrs interfaces;
-
-                  networking.primaryIPAddress =
-                    optionalString (interfaces != []) (head (head interfaces).value.ipv4.addresses).address;
-
-                  # Put the IP addresses of all VMs in this machine's
-                  # /etc/hosts file.  If a machine has multiple
-                  # interfaces, use the IP address corresponding to
-                  # the first interface (i.e. the first network in its
-                  # virtualisation.vlans option).
-                  networking.extraHosts = flip concatMapStrings machines
-                    (m': let config = (getAttr m' nodes).config; in
-                      optionalString (config.networking.primaryIPAddress != "")
-                        ("${config.networking.primaryIPAddress} " +
-                         optionalString (config.networking.domain != null)
-                           "${config.networking.hostName}.${config.networking.domain} " +
-                         "${config.networking.hostName}\n"));
-
-                  virtualisation.qemu.options =
-                    let qemu-common = import ../lib/qemu-common.nix { inherit lib pkgs; };
-                    in flip concatMap interfacesNumbered
-                      ({ fst, snd }: qemu-common.qemuNICFlags snd fst m.snd);
-                };
-
-              in
-                { key = "ip-address";
-                  config = networkConfig // {
-                    # Expose the networkConfig items for tests like nixops
-                    # that need to recreate the network config.
-                    system.build.networkConfig = networkConfig;
-                  };
-                }
-          )
-          (getAttr m.fst nodes)
-        ] );
-
-    in listToAttrs nodes_;
-
-}
diff --git a/nixos/lib/default.nix b/nixos/lib/default.nix
index 2b3056e01457f..65d91342d4d15 100644
--- a/nixos/lib/default.nix
+++ b/nixos/lib/default.nix
@@ -21,6 +21,8 @@ let
   seqAttrsIf = cond: a: lib.mapAttrs (_: v: seqIf cond a v);
 
   eval-config-minimal = import ./eval-config-minimal.nix { inherit lib; };
+
+  testing-lib = import ./testing/default.nix { inherit lib; };
 in
 /*
   This attribute set appears as lib.nixos in the flake, or can be imported
@@ -30,4 +32,10 @@ in
   inherit (seqAttrsIf (!featureFlags?minimalModules) minimalModulesWarning eval-config-minimal)
     evalModules
     ;
+
+  inherit (testing-lib)
+    evalTest
+    runTest
+    ;
+
 }
diff --git a/nixos/lib/eval-config.nix b/nixos/lib/eval-config.nix
index 791a03a3ba3cc..1e086271e5236 100644
--- a/nixos/lib/eval-config.nix
+++ b/nixos/lib/eval-config.nix
@@ -17,6 +17,8 @@ evalConfigArgs@
   # be set modularly anyway.
   pkgs ? null
 , # !!! what do we gain by making this configurable?
+  #     we can add modules that are included in specialisations, regardless
+  #     of inheritParentConfig.
   baseModules ? import ../modules/module-list.nix
 , # !!! See comment about args in lib/modules.nix
   extraArgs ? {}
diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix
index 4bb1689ffd789..c303b0bf17bc0 100644
--- a/nixos/lib/testing-python.nix
+++ b/nixos/lib/testing-python.nix
@@ -12,159 +12,22 @@
 
 with pkgs;
 
+let
+  nixos-lib = import ./default.nix { inherit (pkgs) lib; };
+in
+
 rec {
 
   inherit pkgs;
 
-  # Run an automated test suite in the given virtual network.
-  runTests = { driver, driverInteractive, pos }:
-    stdenv.mkDerivation {
-      name = "vm-test-run-${driver.testName}";
-
-      requiredSystemFeatures = [ "kvm" "nixos-test" ];
-
-      buildCommand =
-        ''
-          mkdir -p $out
-
-          # effectively mute the XMLLogger
-          export LOGFILE=/dev/null
-
-          ${driver}/bin/nixos-test-driver -o $out
-        '';
+  evalTest = module: nixos-lib.evalTest { imports = [ extraTestModule module ]; };
+  runTest = module: nixos-lib.runTest { imports = [ extraTestModule module ]; };
 
-      passthru = driver.passthru // {
-        inherit driver driverInteractive;
-      };
-
-      inherit pos; # for better debugging
+  extraTestModule = {
+    config = {
+      hostPkgs = pkgs;
     };
-
-  # Generate convenience wrappers for running the test driver
-  # has vlans, vms and test script defaulted through env variables
-  # also instantiates test script with nodes, if it's a function (contract)
-  setupDriverForTest = {
-      testScript
-    , testName
-    , nodes
-    , qemu_pkg ? pkgs.qemu_test
-    , enableOCR ? false
-    , skipLint ? false
-    , skipTypeCheck ? false
-    , passthru ? {}
-    , interactive ? false
-    , extraPythonPackages ? (_ :[])
-  }:
-    let
-      # Reifies and correctly wraps the python test driver for
-      # the respective qemu version and with or without ocr support
-      testDriver = pkgs.callPackage ./test-driver {
-        inherit enableOCR extraPythonPackages;
-        qemu_pkg = qemu_test;
-        imagemagick_light = imagemagick_light.override { inherit libtiff; };
-        tesseract4 = tesseract4.override { enableLanguages = [ "eng" ]; };
-      };
-
-
-      testDriverName =
-        let
-          # A standard store path to the vm monitor is built like this:
-          #   /tmp/nix-build-vm-test-run-$name.drv-0/vm-state-machine/monitor
-          # The max filename length of a unix domain socket is 108 bytes.
-          # This means $name can at most be 50 bytes long.
-          maxTestNameLen = 50;
-          testNameLen = builtins.stringLength testName;
-        in with builtins;
-          if testNameLen > maxTestNameLen then
-            abort
-              ("The name of the test '${testName}' must not be longer than ${toString maxTestNameLen} " +
-                "it's currently ${toString testNameLen} characters long.")
-          else
-            "nixos-test-driver-${testName}";
-
-      vlans = map (m: m.config.virtualisation.vlans) (lib.attrValues nodes);
-      vms = map (m: m.config.system.build.vm) (lib.attrValues nodes);
-
-      nodeHostNames = let
-        nodesList = map (c: c.config.system.name) (lib.attrValues nodes);
-      in nodesList ++ lib.optional (lib.length nodesList == 1 && !lib.elem "machine" nodesList) "machine";
-
-      # TODO: This is an implementation error and needs fixing
-      # the testing famework cannot legitimately restrict hostnames further
-      # beyond RFC1035
-      invalidNodeNames = lib.filter
-        (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null)
-        nodeHostNames;
-
-      testScript' =
-        # Call the test script with the computed nodes.
-        if lib.isFunction testScript
-        then testScript { inherit nodes; }
-        else testScript;
-
-      uniqueVlans = lib.unique (builtins.concatLists vlans);
-      vlanNames = map (i: "vlan${toString i}: VLan;") uniqueVlans;
-      machineNames = map (name: "${name}: Machine;") nodeHostNames;
-    in
-    if lib.length invalidNodeNames > 0 then
-      throw ''
-        Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})!
-        All machines are referenced as python variables in the testing framework which will break the
-        script when special characters are used.
-
-        This is an IMPLEMENTATION ERROR and needs to be fixed. Meanwhile,
-        please stick to alphanumeric chars and underscores as separation.
-      ''
-    else lib.warnIf skipLint "Linting is disabled" (runCommand testDriverName
-      {
-        inherit testName;
-        nativeBuildInputs = [ makeWrapper mypy ];
-        buildInputs = [ testDriver ];
-        testScript = testScript';
-        preferLocalBuild = true;
-        passthru = passthru // {
-          inherit nodes;
-        };
-        meta.mainProgram = "nixos-test-driver";
-      }
-      ''
-        mkdir -p $out/bin
-
-        vmStartScripts=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
-
-        ${lib.optionalString (!skipTypeCheck) ''
-          # prepend type hints so the test script can be type checked with mypy
-          cat "${./test-script-prepend.py}" >> testScriptWithTypes
-          echo "${builtins.toString machineNames}" >> testScriptWithTypes
-          echo "${builtins.toString vlanNames}" >> testScriptWithTypes
-          echo -n "$testScript" >> testScriptWithTypes
-
-          mypy  --no-implicit-optional \
-                --pretty \
-                --no-color-output \
-                testScriptWithTypes
-        ''}
-
-        echo -n "$testScript" >> $out/test-script
-
-        ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver
-
-        ${testDriver}/bin/generate-driver-symbols
-        ${lib.optionalString (!skipLint) ''
-          PYFLAKES_BUILTINS="$(
-            echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)},
-            < ${lib.escapeShellArg "driver-symbols"}
-          )" ${python3Packages.pyflakes}/bin/pyflakes $out/test-script
-        ''}
-
-        # set defaults through environment
-        # see: ./test-driver/test-driver.py argparse implementation
-        wrapProgram $out/bin/nixos-test-driver \
-          --set startScripts "''${vmStartScripts[*]}" \
-          --set testScript "$out/test-script" \
-          --set vlans '${toString vlans}' \
-          ${lib.optionalString (interactive) "--add-flags --interactive"}
-      '');
+  };
 
   # Make a full-blown test
   makeTest =
@@ -184,91 +47,20 @@ rec {
           then builtins.unsafeGetAttrPos "description" meta
           else builtins.unsafeGetAttrPos "testScript" t)
     , extraPythonPackages ? (_ : [])
+    , interactive ? {}
     } @ t:
-    let
-      mkNodes = qemu_pkg:
-        let
-          testScript' =
-            # Call the test script with the computed nodes.
-            if lib.isFunction testScript
-            then testScript { nodes = mkNodes qemu_pkg; }
-            else testScript;
-
-          build-vms = import ./build-vms.nix {
-            inherit system lib pkgs minimal specialArgs;
-            extraConfigurations = extraConfigurations ++ [(
-              { config, ... }:
-              {
-                virtualisation.qemu.package = qemu_pkg;
-
-                # Make sure all derivations referenced by the test
-                # script are available on the nodes. When the store is
-                # accessed through 9p, this isn't important, since
-                # everything in the store is available to the guest,
-                # but when building a root image it is, as all paths
-                # that should be available to the guest has to be
-                # copied to the image.
-                virtualisation.additionalPaths =
-                  lib.optional
-                    # A testScript may evaluate nodes, which has caused
-                    # infinite recursions. The demand cycle involves:
-                    #   testScript -->
-                    #   nodes -->
-                    #   toplevel -->
-                    #   additionalPaths -->
-                    #   hasContext testScript' -->
-                    #   testScript (ad infinitum)
-                    # If we don't need to build an image, we can break this
-                    # cycle by short-circuiting when useNixStoreImage is false.
-                    (config.virtualisation.useNixStoreImage && builtins.hasContext testScript')
-                    (pkgs.writeStringReferencesToFile testScript');
-
-                # Ensure we do not use aliases. Ideally this is only set
-                # when the test framework is used by Nixpkgs NixOS tests.
-                nixpkgs.config.allowAliases = false;
-              }
-            )];
-          };
-        in
-          lib.warnIf (t?machine) "In test `${name}': The `machine' attribute in NixOS tests (pkgs.nixosTest / make-test-python.nix / testing-python.nix / makeTest) is deprecated. Please use the equivalent `nodes.machine'."
-          build-vms.buildVirtualNetwork (
-              nodes // lib.optionalAttrs (machine != null) { inherit machine; }
-          );
-
-      driver = setupDriverForTest {
-        inherit testScript enableOCR skipTypeCheck skipLint passthru extraPythonPackages;
-        testName = name;
-        qemu_pkg = pkgs.qemu_test;
-        nodes = mkNodes pkgs.qemu_test;
-      };
-      driverInteractive = setupDriverForTest {
-        inherit testScript enableOCR skipTypeCheck skipLint passthru extraPythonPackages;
-        testName = name;
-        qemu_pkg = pkgs.qemu;
-        nodes = mkNodes pkgs.qemu;
-        interactive = true;
+      runTest {
+        imports = [
+          { _file = "makeTest parameters"; config = t; }
+          {
+            defaults = {
+              _file = "makeTest: extraConfigurations";
+              imports = extraConfigurations;
+            };
+          }
+        ];
       };
 
-      test = lib.addMetaAttrs meta (runTests { inherit driver pos driverInteractive; });
-
-    in
-      test // {
-        inherit test driver driverInteractive;
-        inherit (driver) nodes;
-      };
-
-  abortForFunction = functionName: abort ''The ${functionName} function was
-    removed because it is not an essential part of the NixOS testing
-    infrastructure. It had no usage in NixOS or Nixpkgs and it had no designated
-    maintainer. You are free to reintroduce it by documenting it in the manual
-    and adding yourself as maintainer. It was removed in
-    https://github.com/NixOS/nixpkgs/pull/137013
-  '';
-
-  runInMachine = abortForFunction "runInMachine";
-
-  runInMachineWithX = abortForFunction "runInMachineWithX";
-
   simpleTest = as: (makeTest as).test;
 
 }
diff --git a/nixos/lib/testing/call-test.nix b/nixos/lib/testing/call-test.nix
new file mode 100644
index 0000000000000..3e137e78cd47f
--- /dev/null
+++ b/nixos/lib/testing/call-test.nix
@@ -0,0 +1,16 @@
+{ config, lib, ... }:
+let
+  inherit (lib) mkOption types;
+in
+{
+  options = {
+    callTest = mkOption {
+      internal = true;
+      type = types.functionTo types.raw;
+    };
+    result = mkOption {
+      internal = true;
+      default = config;
+    };
+  };
+}
diff --git a/nixos/lib/testing/default.nix b/nixos/lib/testing/default.nix
new file mode 100644
index 0000000000000..676d52f5c3fb7
--- /dev/null
+++ b/nixos/lib/testing/default.nix
@@ -0,0 +1,24 @@
+{ lib }:
+let
+
+  evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; };
+  runTest = module: (evalTest module).config.result;
+
+  testModules = [
+    ./call-test.nix
+    ./driver.nix
+    ./interactive.nix
+    ./legacy.nix
+    ./meta.nix
+    ./name.nix
+    ./network.nix
+    ./nodes.nix
+    ./pkgs.nix
+    ./run.nix
+    ./testScript.nix
+  ];
+
+in
+{
+  inherit evalTest runTest testModules;
+}
diff --git a/nixos/lib/testing/driver.nix b/nixos/lib/testing/driver.nix
new file mode 100644
index 0000000000000..04e99f9e21d61
--- /dev/null
+++ b/nixos/lib/testing/driver.nix
@@ -0,0 +1,188 @@
+{ config, lib, hostPkgs, ... }:
+let
+  inherit (lib) mkOption types literalMD mdDoc;
+
+  # Reifies and correctly wraps the python test driver for
+  # the respective qemu version and with or without ocr support
+  testDriver = hostPkgs.callPackage ../test-driver {
+    inherit (config) enableOCR extraPythonPackages;
+    qemu_pkg = config.qemu.package;
+    imagemagick_light = hostPkgs.imagemagick_light.override { inherit (hostPkgs) libtiff; };
+    tesseract4 = hostPkgs.tesseract4.override { enableLanguages = [ "eng" ]; };
+  };
+
+
+  vlans = map (m: m.virtualisation.vlans) (lib.attrValues config.nodes);
+  vms = map (m: m.system.build.vm) (lib.attrValues config.nodes);
+
+  nodeHostNames =
+    let
+      nodesList = map (c: c.system.name) (lib.attrValues config.nodes);
+    in
+    nodesList ++ lib.optional (lib.length nodesList == 1 && !lib.elem "machine" nodesList) "machine";
+
+  # TODO: This is an implementation error and needs fixing
+  # the testing famework cannot legitimately restrict hostnames further
+  # beyond RFC1035
+  invalidNodeNames = lib.filter
+    (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null)
+    nodeHostNames;
+
+  uniqueVlans = lib.unique (builtins.concatLists vlans);
+  vlanNames = map (i: "vlan${toString i}: VLan;") uniqueVlans;
+  machineNames = map (name: "${name}: Machine;") nodeHostNames;
+
+  withChecks =
+    if lib.length invalidNodeNames > 0 then
+      throw ''
+        Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})!
+        All machines are referenced as python variables in the testing framework which will break the
+        script when special characters are used.
+
+        This is an IMPLEMENTATION ERROR and needs to be fixed. Meanwhile,
+        please stick to alphanumeric chars and underscores as separation.
+      ''
+    else
+      lib.warnIf config.skipLint "Linting is disabled";
+
+  driver =
+    hostPkgs.runCommand "nixos-test-driver-${config.name}"
+      {
+        # inherit testName; TODO (roberth): need this?
+        nativeBuildInputs = [
+          hostPkgs.makeWrapper
+        ] ++ lib.optionals (!config.skipTypeCheck) [ hostPkgs.mypy ];
+        buildInputs = [ testDriver ];
+        testScript = config.testScriptString;
+        preferLocalBuild = true;
+        passthru = config.passthru;
+        meta = config.meta // {
+          mainProgram = "nixos-test-driver";
+        };
+      }
+      ''
+        mkdir -p $out/bin
+
+        vmStartScripts=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
+
+        ${lib.optionalString (!config.skipTypeCheck) ''
+          # prepend type hints so the test script can be type checked with mypy
+          cat "${../test-script-prepend.py}" >> testScriptWithTypes
+          echo "${builtins.toString machineNames}" >> testScriptWithTypes
+          echo "${builtins.toString vlanNames}" >> testScriptWithTypes
+          echo -n "$testScript" >> testScriptWithTypes
+
+          cat -n testScriptWithTypes
+
+          mypy  --no-implicit-optional \
+                --pretty \
+                --no-color-output \
+                testScriptWithTypes
+        ''}
+
+        echo -n "$testScript" >> $out/test-script
+
+        ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver
+
+        ${testDriver}/bin/generate-driver-symbols
+        ${lib.optionalString (!config.skipLint) ''
+          PYFLAKES_BUILTINS="$(
+            echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)},
+            < ${lib.escapeShellArg "driver-symbols"}
+          )" ${hostPkgs.python3Packages.pyflakes}/bin/pyflakes $out/test-script
+        ''}
+
+        # set defaults through environment
+        # see: ./test-driver/test-driver.py argparse implementation
+        wrapProgram $out/bin/nixos-test-driver \
+          --set startScripts "''${vmStartScripts[*]}" \
+          --set testScript "$out/test-script" \
+          --set vlans '${toString vlans}' \
+          ${lib.escapeShellArgs (lib.concatMap (arg: ["--add-flags" arg]) config.extraDriverArgs)}
+      '';
+
+in
+{
+  options = {
+
+    driver = mkOption {
+      description = mdDoc "Package containing a script that runs the test.";
+      type = types.package;
+      defaultText = literalMD "set by the test framework";
+    };
+
+    hostPkgs = mkOption {
+      description = mdDoc "Nixpkgs attrset used outside the nodes.";
+      type = types.raw;
+      example = lib.literalExpression ''
+        import nixpkgs { inherit system config overlays; }
+      '';
+    };
+
+    qemu.package = mkOption {
+      description = mdDoc "Which qemu package to use for the virtualisation of [{option}`nodes`](#opt-nodes).";
+      type = types.package;
+      default = hostPkgs.qemu_test;
+      defaultText = "hostPkgs.qemu_test";
+    };
+
+    enableOCR = mkOption {
+      description = mdDoc ''
+        Whether to enable Optical Character Recognition functionality for
+        testing graphical programs. See [Machine objects](`ssec-machine-objects`).
+      '';
+      type = types.bool;
+      default = false;
+    };
+
+    extraPythonPackages = mkOption {
+      description = mdDoc ''
+        Python packages to add to the test driver.
+
+        The argument is a Python package set, similar to `pkgs.pythonPackages`.
+      '';
+      example = lib.literalExpression ''
+        p: [ p.numpy ]
+      '';
+      type = types.functionTo (types.listOf types.package);
+      default = ps: [ ];
+    };
+
+    extraDriverArgs = mkOption {
+      description = mdDoc ''
+        Extra arguments to pass to the test driver.
+
+        They become part of [{option}`driver`](#opt-driver) via `wrapProgram`.
+      '';
+      type = types.listOf types.str;
+      default = [];
+    };
+
+    skipLint = mkOption {
+      type = types.bool;
+      default = false;
+      description = mdDoc ''
+        Do not run the linters. This may speed up your iteration cycle, but it is not something you should commit.
+      '';
+    };
+
+    skipTypeCheck = mkOption {
+      type = types.bool;
+      default = false;
+      description = mdDoc ''
+        Disable type checking. This must not be enabled for new NixOS tests.
+
+        This may speed up your iteration cycle, unless you're working on the [{option}`testScript`](#opt-testScript).
+      '';
+    };
+  };
+
+  config = {
+    _module.args.hostPkgs = config.hostPkgs;
+
+    driver = withChecks driver;
+
+    # make available on the test runner
+    passthru.driver = config.driver;
+  };
+}
diff --git a/nixos/lib/testing/interactive.nix b/nixos/lib/testing/interactive.nix
new file mode 100644
index 0000000000000..317ed4241882b
--- /dev/null
+++ b/nixos/lib/testing/interactive.nix
@@ -0,0 +1,45 @@
+{ config, lib, moduleType, hostPkgs, ... }:
+let
+  inherit (lib) mkOption types mdDoc;
+in
+{
+  options = {
+    interactive = mkOption {
+      description = mdDoc ''
+        Tests [can be run interactively](#sec-running-nixos-tests-interactively)
+        using the program in the test derivation's `.driverInteractive` attribute.
+
+        When they are, the configuration will include anything set in this submodule.
+
+        You can set any top-level test option here.
+
+        Example test module:
+
+        ```nix
+        { config, lib, ... }: {
+
+          nodes.rabbitmq = {
+            services.rabbitmq.enable = true;
+          };
+
+          # When running interactively ...
+          interactive.nodes.rabbitmq = {
+            # ... enable the web ui.
+            services.rabbitmq.managementPlugin.enable = true;
+          };
+        }
+        ```
+
+        For details, see the section about [running tests interactively](#sec-running-nixos-tests-interactively).
+      '';
+      type = moduleType;
+      visible = "shallow";
+    };
+  };
+
+  config = {
+    interactive.qemu.package = hostPkgs.qemu;
+    interactive.extraDriverArgs = [ "--interactive" ];
+    passthru.driverInteractive = config.interactive.driver;
+  };
+}
diff --git a/nixos/lib/testing/legacy.nix b/nixos/lib/testing/legacy.nix
new file mode 100644
index 0000000000000..868b8b65b17d5
--- /dev/null
+++ b/nixos/lib/testing/legacy.nix
@@ -0,0 +1,25 @@
+{ config, options, lib, ... }:
+let
+  inherit (lib) mkIf mkOption types;
+in
+{
+  # This needs options.warnings, which we don't have (yet?).
+  # imports = [
+  #   (lib.mkRenamedOptionModule [ "machine" ] [ "nodes" "machine" ])
+  # ];
+
+  options = {
+    machine = mkOption {
+      internal = true;
+      type = types.raw;
+    };
+  };
+
+  config = {
+    nodes = mkIf options.machine.isDefined (
+      lib.warn
+        "In test `${config.name}': The `machine' attribute in NixOS tests (pkgs.nixosTest / make-test-python.nix / testing-python.nix / makeTest) is deprecated. Please set the equivalent `nodes.machine'."
+        { inherit (config) machine; }
+    );
+  };
+}
diff --git a/nixos/lib/testing/meta.nix b/nixos/lib/testing/meta.nix
new file mode 100644
index 0000000000000..4d8b0e0f1c439
--- /dev/null
+++ b/nixos/lib/testing/meta.nix
@@ -0,0 +1,42 @@
+{ lib, ... }:
+let
+  inherit (lib) types mkOption mdDoc;
+in
+{
+  options = {
+    meta = lib.mkOption {
+      description = mdDoc ''
+        The [`meta`](https://nixos.org/manual/nixpkgs/stable/#chap-meta) attributes that will be set on the returned derivations.
+
+        Not all [`meta`](https://nixos.org/manual/nixpkgs/stable/#chap-meta) attributes are supported, but more can be added as desired.
+      '';
+      apply = lib.filterAttrs (k: v: v != null);
+      type = types.submodule {
+        options = {
+          maintainers = lib.mkOption {
+            type = types.listOf types.raw;
+            default = [];
+            description = mdDoc ''
+              The [list of maintainers](https://nixos.org/manual/nixpkgs/stable/#var-meta-maintainers) for this test.
+            '';
+          };
+          timeout = lib.mkOption {
+            type = types.nullOr types.int;
+            default = null; # NOTE: null values are filtered out by `meta`.
+            description = mdDoc ''
+              The [{option}`test`](#opt-test)'s [`meta.timeout`](https://nixos.org/manual/nixpkgs/stable/#var-meta-timeout) in seconds.
+            '';
+          };
+          broken = lib.mkOption {
+            type = types.bool;
+            default = false;
+            description = mdDoc ''
+              Sets the [`meta.broken`](https://nixos.org/manual/nixpkgs/stable/#var-meta-broken) attribute on the [{option}`test`](#opt-test) derivation.
+            '';
+          };
+        };
+      };
+      default = {};
+    };
+  };
+}
diff --git a/nixos/lib/testing/name.nix b/nixos/lib/testing/name.nix
new file mode 100644
index 0000000000000..a54622e139bf6
--- /dev/null
+++ b/nixos/lib/testing/name.nix
@@ -0,0 +1,14 @@
+{ lib, ... }:
+let
+  inherit (lib) mkOption types mdDoc;
+in
+{
+  options.name = mkOption {
+    description = mdDoc ''
+      The name of the test.
+
+      This is used in the derivation names of the [{option}`driver`](#opt-driver) and [{option}`test`](#opt-test) runner.
+    '';
+    type = types.str;
+  };
+}
diff --git a/nixos/lib/testing/network.nix b/nixos/lib/testing/network.nix
new file mode 100644
index 0000000000000..04ea9a2bc9f7a
--- /dev/null
+++ b/nixos/lib/testing/network.nix
@@ -0,0 +1,117 @@
+{ lib, nodes, ... }:
+
+let
+  inherit (lib)
+    attrNames concatMap concatMapStrings flip forEach head
+    listToAttrs mkDefault mkOption nameValuePair optionalString
+    range types zipListsWith zipLists
+    mdDoc
+    ;
+
+  nodeNumbers =
+    listToAttrs
+      (zipListsWith
+        nameValuePair
+        (attrNames nodes)
+        (range 1 254)
+      );
+
+  networkModule = { config, nodes, pkgs, ... }:
+    let
+      interfacesNumbered = zipLists config.virtualisation.vlans (range 1 255);
+      interfaces = forEach interfacesNumbered ({ fst, snd }:
+        nameValuePair "eth${toString snd}" {
+          ipv4.addresses =
+            [{
+              address = "192.168.${toString fst}.${toString config.virtualisation.test.nodeNumber}";
+              prefixLength = 24;
+            }];
+        });
+
+      networkConfig =
+        {
+          networking.hostName = mkDefault config.virtualisation.test.nodeName;
+
+          networking.interfaces = listToAttrs interfaces;
+
+          networking.primaryIPAddress =
+            optionalString (interfaces != [ ]) (head (head interfaces).value.ipv4.addresses).address;
+
+          # Put the IP addresses of all VMs in this machine's
+          # /etc/hosts file.  If a machine has multiple
+          # interfaces, use the IP address corresponding to
+          # the first interface (i.e. the first network in its
+          # virtualisation.vlans option).
+          networking.extraHosts = flip concatMapStrings (attrNames nodes)
+            (m':
+              let config = nodes.${m'}; in
+              optionalString (config.networking.primaryIPAddress != "")
+                ("${config.networking.primaryIPAddress} " +
+                  optionalString (config.networking.domain != null)
+                    "${config.networking.hostName}.${config.networking.domain} " +
+                  "${config.networking.hostName}\n"));
+
+          virtualisation.qemu.options =
+            let qemu-common = import ../qemu-common.nix { inherit lib pkgs; };
+            in
+            flip concatMap interfacesNumbered
+              ({ fst, snd }: qemu-common.qemuNICFlags snd fst config.virtualisation.test.nodeNumber);
+        };
+
+    in
+    {
+      key = "ip-address";
+      config = networkConfig // {
+        # Expose the networkConfig items for tests like nixops
+        # that need to recreate the network config.
+        system.build.networkConfig = networkConfig;
+      };
+    };
+
+  nodeNumberModule = (regular@{ config, name, ... }: {
+    options = {
+      virtualisation.test.nodeName = mkOption {
+        internal = true;
+        default = name;
+        # We need to force this in specilisations, otherwise it'd be
+        # readOnly = true;
+        description = mdDoc ''
+          The `name` in `nodes.<name>`; stable across `specialisations`.
+        '';
+      };
+      virtualisation.test.nodeNumber = mkOption {
+        internal = true;
+        type = types.int;
+        readOnly = true;
+        default = nodeNumbers.${config.virtualisation.test.nodeName};
+        description = mdDoc ''
+          A unique number assigned for each node in `nodes`.
+        '';
+      };
+
+      # specialisations override the `name` module argument,
+      # so we push the real `virtualisation.test.nodeName`.
+      specialisation = mkOption {
+        type = types.attrsOf (types.submodule {
+          options.configuration = mkOption {
+            type = types.submoduleWith {
+              modules = [
+                {
+                  config.virtualisation.test.nodeName =
+                    # assert regular.config.virtualisation.test.nodeName != "configuration";
+                    regular.config.virtualisation.test.nodeName;
+                }
+              ];
+            };
+          };
+        });
+      };
+    };
+  });
+
+in
+{
+  config = {
+    extraBaseModules = { imports = [ networkModule nodeNumberModule ]; };
+  };
+}
diff --git a/nixos/lib/testing/nixos-test-base.nix b/nixos/lib/testing/nixos-test-base.nix
new file mode 100644
index 0000000000000..59e6e38433679
--- /dev/null
+++ b/nixos/lib/testing/nixos-test-base.nix
@@ -0,0 +1,23 @@
+# A module containing the base imports and overrides that
+# are always applied in NixOS VM tests, unconditionally,
+# even in `inheritParentConfig = false` specialisations.
+{ lib, ... }:
+let
+  inherit (lib) mkForce;
+in
+{
+  imports = [
+    ../../modules/virtualisation/qemu-vm.nix
+    ../../modules/testing/test-instrumentation.nix # !!! should only get added for automated test runs
+    { key = "no-manual"; documentation.nixos.enable = false; }
+    {
+      key = "no-revision";
+      # Make the revision metadata constant, in order to avoid needless retesting.
+      # The human version (e.g. 21.05-pre) is left as is, because it is useful
+      # for external modules that test with e.g. testers.nixosTest and rely on that
+      # version number.
+      config.system.nixos.revision = mkForce "constant-nixos-revision";
+    }
+
+  ];
+}
diff --git a/nixos/lib/testing/nodes.nix b/nixos/lib/testing/nodes.nix
new file mode 100644
index 0000000000000..765af2878dfe8
--- /dev/null
+++ b/nixos/lib/testing/nodes.nix
@@ -0,0 +1,112 @@
+testModuleArgs@{ config, lib, hostPkgs, nodes, ... }:
+
+let
+  inherit (lib) mkOption mkForce optional types mapAttrs mkDefault mdDoc;
+
+  system = hostPkgs.stdenv.hostPlatform.system;
+
+  baseOS =
+    import ../eval-config.nix {
+      inherit system;
+      inherit (config.node) specialArgs;
+      modules = [ config.defaults ];
+      baseModules = (import ../../modules/module-list.nix) ++
+        [
+          ./nixos-test-base.nix
+          { key = "nodes"; _module.args.nodes = config.nodesCompat; }
+          ({ config, ... }:
+            {
+              virtualisation.qemu.package = testModuleArgs.config.qemu.package;
+
+              # Ensure we do not use aliases. Ideally this is only set
+              # when the test framework is used by Nixpkgs NixOS tests.
+              nixpkgs.config.allowAliases = false;
+            })
+          testModuleArgs.config.extraBaseModules
+        ] ++ optional config.minimal ../../modules/testing/minimal-kernel.nix;
+    };
+
+
+in
+
+{
+
+  options = {
+    node.type = mkOption {
+      type = types.raw;
+      default = baseOS.type;
+      internal = true;
+    };
+
+    nodes = mkOption {
+      type = types.lazyAttrsOf config.node.type;
+      visible = "shallow";
+      description = mdDoc ''
+        An attribute set of NixOS configuration modules.
+
+        The configurations are augmented by the [`defaults`](#opt-defaults) option.
+
+        They are assigned network addresses according to the `nixos/lib/testing/network.nix` module.
+
+        A few special options are available, that aren't in a plain NixOS configuration. See [Configuring the nodes](#sec-nixos-test-nodes)
+      '';
+    };
+
+    defaults = mkOption {
+      description = mdDoc ''
+        NixOS configuration that is applied to all [{option}`nodes`](#opt-nodes).
+      '';
+      type = types.deferredModule;
+      default = { };
+    };
+
+    extraBaseModules = mkOption {
+      description = mdDoc ''
+        NixOS configuration that, like [{option}`defaults`](#opt-defaults), is applied to all [{option}`nodes`](#opt-nodes) and can not be undone with [`specialisation.<name>.inheritParentConfig`](https://search.nixos.org/options?show=specialisation.%3Cname%3E.inheritParentConfig&from=0&size=50&sort=relevance&type=packages&query=specialisation).
+      '';
+      type = types.deferredModule;
+      default = { };
+    };
+
+    node.specialArgs = mkOption {
+      type = types.lazyAttrsOf types.raw;
+      default = { };
+      description = mdDoc ''
+        An attribute set of arbitrary values that will be made available as module arguments during the resolution of module `imports`.
+
+        Note that it is not possible to override these from within the NixOS configurations. If you argument is not relevant to `imports`, consider setting {option}`defaults._module.args.<name>` instead.
+      '';
+    };
+
+    minimal = mkOption {
+      type = types.bool;
+      default = false;
+      description = mdDoc ''
+        Enable to configure all [{option}`nodes`](#opt-nodes) to run with a minimal kernel.
+      '';
+    };
+
+    nodesCompat = mkOption {
+      internal = true;
+      description = mdDoc ''
+        Basically `_module.args.nodes`, but with backcompat and warnings added.
+
+        This will go away.
+      '';
+    };
+  };
+
+  config = {
+    _module.args.nodes = config.nodesCompat;
+    nodesCompat =
+      mapAttrs
+        (name: config: config // {
+          config = lib.warn
+            "Module argument `nodes.${name}.config` is deprecated. Use `nodes.${name}` instead."
+            config;
+        })
+        config.nodes;
+
+    passthru.nodes = config.nodesCompat;
+  };
+}
diff --git a/nixos/lib/testing/pkgs.nix b/nixos/lib/testing/pkgs.nix
new file mode 100644
index 0000000000000..22dd586868e30
--- /dev/null
+++ b/nixos/lib/testing/pkgs.nix
@@ -0,0 +1,11 @@
+{ config, lib, hostPkgs, ... }:
+{
+  config = {
+    # default pkgs for use in VMs
+    _module.args.pkgs = hostPkgs;
+
+    defaults = {
+      # TODO: a module to set a shared pkgs, if options.nixpkgs.* is untouched by user (highestPrio) */
+    };
+  };
+}
diff --git a/nixos/lib/testing/run.nix b/nixos/lib/testing/run.nix
new file mode 100644
index 0000000000000..0cd07d8afd21d
--- /dev/null
+++ b/nixos/lib/testing/run.nix
@@ -0,0 +1,57 @@
+{ config, hostPkgs, lib, ... }:
+let
+  inherit (lib) types mkOption mdDoc;
+in
+{
+  options = {
+    passthru = mkOption {
+      type = types.lazyAttrsOf types.raw;
+      description = mdDoc ''
+        Attributes to add to the returned derivations,
+        which are not necessarily part of the build.
+
+        This is a bit like doing `drv // { myAttr = true; }` (which would be lost by `overrideAttrs`).
+        It does not change the actual derivation, but adds the attribute nonetheless, so that
+        consumers of what would be `drv` have more information.
+      '';
+    };
+
+    test = mkOption {
+      type = types.package;
+      # TODO: can the interactive driver be configured to access the network?
+      description = mdDoc ''
+        Derivation that runs the test as its "build" process.
+
+        This implies that NixOS tests run isolated from the network, making them
+        more dependable.
+      '';
+    };
+  };
+
+  config = {
+    test = lib.lazyDerivation { # lazyDerivation improves performance when only passthru items and/or meta are used.
+      derivation = hostPkgs.stdenv.mkDerivation {
+        name = "vm-test-run-${config.name}";
+
+        requiredSystemFeatures = [ "kvm" "nixos-test" ];
+
+        buildCommand = ''
+          mkdir -p $out
+
+          # effectively mute the XMLLogger
+          export LOGFILE=/dev/null
+
+          ${config.driver}/bin/nixos-test-driver -o $out
+        '';
+
+        passthru = config.passthru;
+
+        meta = config.meta;
+      };
+      inherit (config) passthru meta;
+    };
+
+    # useful for inspection (debugging / exploration)
+    passthru.config = config;
+  };
+}
diff --git a/nixos/lib/testing/testScript.nix b/nixos/lib/testing/testScript.nix
new file mode 100644
index 0000000000000..5d4181c5f5dd5
--- /dev/null
+++ b/nixos/lib/testing/testScript.nix
@@ -0,0 +1,84 @@
+testModuleArgs@{ config, lib, hostPkgs, nodes, moduleType, ... }:
+let
+  inherit (lib) mkOption types mdDoc;
+  inherit (types) either str functionTo;
+in
+{
+  options = {
+    testScript = mkOption {
+      type = either str (functionTo str);
+      description = ''
+        A series of python declarations and statements that you write to perform
+        the test.
+      '';
+    };
+    testScriptString = mkOption {
+      type = str;
+      readOnly = true;
+      internal = true;
+    };
+
+    includeTestScriptReferences = mkOption {
+      type = types.bool;
+      default = true;
+      internal = true;
+    };
+    withoutTestScriptReferences = mkOption {
+      type = moduleType;
+      description = mdDoc ''
+        A parallel universe where the testScript is invalid and has no references.
+      '';
+      internal = true;
+      visible = false;
+    };
+  };
+  config = {
+    withoutTestScriptReferences.includeTestScriptReferences = false;
+    withoutTestScriptReferences.testScript = lib.mkForce "testscript omitted";
+
+    testScriptString =
+      if lib.isFunction config.testScript
+      then
+        config.testScript
+          {
+            nodes =
+              lib.mapAttrs
+                (k: v:
+                  if v.virtualisation.useNixStoreImage
+                  then
+                  # prevent infinite recursion when testScript would
+                  # reference v's toplevel
+                    config.withoutTestScriptReferences.nodesCompat.${k}
+                  else
+                  # reuse memoized config
+                    v
+                )
+                config.nodesCompat;
+          }
+      else config.testScript;
+
+    defaults = { config, name, ... }: {
+      # Make sure all derivations referenced by the test
+      # script are available on the nodes. When the store is
+      # accessed through 9p, this isn't important, since
+      # everything in the store is available to the guest,
+      # but when building a root image it is, as all paths
+      # that should be available to the guest has to be
+      # copied to the image.
+      virtualisation.additionalPaths =
+        lib.optional
+          # A testScript may evaluate nodes, which has caused
+          # infinite recursions. The demand cycle involves:
+          #   testScript -->
+          #   nodes -->
+          #   toplevel -->
+          #   additionalPaths -->
+          #   hasContext testScript' -->
+          #   testScript (ad infinitum)
+          # If we don't need to build an image, we can break this
+          # cycle by short-circuiting when useNixStoreImage is false.
+          (config.virtualisation.useNixStoreImage && builtins.hasContext testModuleArgs.config.testScriptString && testModuleArgs.config.includeTestScriptReferences)
+          (hostPkgs.writeStringReferencesToFile testModuleArgs.config.testScriptString);
+    };
+  };
+}