tests.nixpkgs-check-by-name: Test for gradual transition

This implements the ability to test gradual transitions in check
strictness, and adds one such test for the empty non-auto-called
arguments check.

9 files changed, 22 insertions, 1 deletions

diff --git a/pkgs/test/nixpkgs-check-by-name/README.md b/pkgs/test/nixpkgs-check-by-name/README.md
index b098658fce4c2..c906eaffc974b 100644
--- a/pkgs/test/nixpkgs-check-by-name/README.md
+++ b/pkgs/test/nixpkgs-check-by-name/README.md
@@ -81,6 +81,10 @@ Tests are declared in [`./tests`](./tests) as subdirectories imitating Nixpkgs w
   allowing the simulation of package overrides to the real [`pkgs/top-level/all-packages.nix`](../../top-level/all-packages.nix`).
   The default is an empty overlay.
 
+- `base` (optional):
+  Contains another subdirectory imitating Nixpkgs with potentially any of the above structures.
+  This will be used as the `--base` argument, allowing tests of gradual transitions.
+
 - `expected` (optional):
   A file containing the expected standard output.
   The default is expecting an empty standard output.
diff --git a/pkgs/test/nixpkgs-check-by-name/src/main.rs b/pkgs/test/nixpkgs-check-by-name/src/main.rs
index 53c24845cb200..bf3bfb193f181 100644
--- a/pkgs/test/nixpkgs-check-by-name/src/main.rs
+++ b/pkgs/test/nixpkgs-check-by-name/src/main.rs
@@ -212,10 +212,17 @@ mod tests {
     fn test_nixpkgs(name: &str, path: &Path, expected_errors: &str) -> anyhow::Result<()> {
         let extra_nix_path = Path::new("tests/mock-nixpkgs.nix");
 
+        let base_path = path.join("base");
+        let base_nixpkgs = if base_path.exists() {
+            Some(base_path.as_path())
+        } else {
+            None
+        };
+
         // We don't want coloring to mess up the tests
         let writer = temp_env::with_var("NO_COLOR", Some("1"), || -> anyhow::Result<_> {
             let mut writer = vec![];
-            process(None, &path, &vec![&extra_nix_path], &mut writer)
+            process(base_nixpkgs, &path, &vec![&extra_nix_path], &mut writer)
                 .context(format!("Failed test case {name}"))?;
             Ok(writer)
         })?;
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/all-packages.nix
new file mode 100644
index 0000000000000..d369dd7228dca
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/all-packages.nix
@@ -0,0 +1,3 @@
+self: super: {
+  nonDerivation = self.callPackage ./pkgs/by-name/no/nonDerivation/package.nix { };
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/all-packages.nix
new file mode 100644
index 0000000000000..d369dd7228dca
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/all-packages.nix
@@ -0,0 +1,3 @@
+self: super: {
+  nonDerivation = self.callPackage ./pkgs/by-name/no/nonDerivation/package.nix { };
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/default.nix
new file mode 100644
index 0000000000000..2875ea6327ef2
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/default.nix
@@ -0,0 +1 @@
+import ../../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/pkgs/by-name/no/nonDerivation/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/pkgs/by-name/no/nonDerivation/package.nix
new file mode 100644
index 0000000000000..a1b92efbbadb9
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/pkgs/by-name/no/nonDerivation/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/default.nix
new file mode 100644
index 0000000000000..af25d1450122b
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/expected b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/expected
new file mode 100644
index 0000000000000..e69de29bb2d1d
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/expected
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/pkgs/by-name/no/nonDerivation/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/pkgs/by-name/no/nonDerivation/package.nix
new file mode 100644
index 0000000000000..a1b92efbbadb9
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/pkgs/by-name/no/nonDerivation/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv