tests.nixpkgs-check-by-name: Introduce a non-applicable ratchet state

Introduces NonApplicable as a state of a ratchet, to be used when the
ratchet doesn't make sense to have.

This fixes an odd problem where before, changing an attribute to use
e.g. `callPackage` suddenly requires moving it to `pkgs/by-name`, when that
shouldn't have been required.

11 files changed, 34 insertions, 4 deletions

diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.rs b/pkgs/test/nixpkgs-check-by-name/src/eval.rs
index c3be2c5917d62..5aa64f9bb00db 100644
--- a/pkgs/test/nixpkgs-check-by-name/src/eval.rs
+++ b/pkgs/test/nixpkgs-check-by-name/src/eval.rs
@@ -159,8 +159,8 @@ pub fn check_values(
                     let uses_by_name = match attribute_info {
                         // In these cases the package doesn't qualify for being in pkgs/by-name,
                         // so the UsesByName ratchet is already as tight as it can be
-                        NonAttributeSet => Success(Tight),
-                        NonCallPackage => Success(Tight),
+                        NonAttributeSet => Success(NonApplicable),
+                        NonCallPackage => Success(NonApplicable),
                         // This is the case when the `pkgs/by-name`-internal _internalCallByNamePackageFile
                         // is used for a package outside `pkgs/by-name`
                         CallPackage(CallPackageInfo {
@@ -176,14 +176,14 @@ pub fn check_values(
                             // In the future we could kind of abuse this behavior to have better
                             // enforcement of conditional aliases, but for now we just need to not
                             // give an error.
-                            Success(Tight)
+                            Success(NonApplicable)
                         }
                         // Only derivations can be in pkgs/by-name,
                         // so this attribute doesn't qualify
                         CallPackage(CallPackageInfo {
                             is_derivation: false,
                             ..
-                        }) => Success(Tight),
+                        }) => Success(NonApplicable),
 
                         // The case of an attribute that qualifies:
                         // - Uses callPackage
diff --git a/pkgs/test/nixpkgs-check-by-name/src/ratchet.rs b/pkgs/test/nixpkgs-check-by-name/src/ratchet.rs
index dabc8f67316a7..10ecc01d3580c 100644
--- a/pkgs/test/nixpkgs-check-by-name/src/ratchet.rs
+++ b/pkgs/test/nixpkgs-check-by-name/src/ratchet.rs
@@ -69,6 +69,9 @@ pub enum RatchetState<Ratchet: ToNixpkgsProblem> {
     /// This is either because we already use the latest state, or because the ratchet isn't
     /// relevant.
     Tight,
+    /// This ratchet can't be applied.
+    /// State transitions from/to NonApplicable are always allowed
+    NonApplicable,
 }
 
 /// A trait that can convert an attribute-specific error context into a NixpkgsProblem
@@ -102,6 +105,7 @@ impl<Context: ToNixpkgsProblem> RatchetState<Context> {
             // Everything else is allowed, including:
             // - Loose -> Loose (grandfathering policy for a loose ratchet)
             // - -> Tight (always okay to keep or make the ratchet tight)
+            // - Anything involving NotApplicable, where we can't really make any good calls
             _ => Success(()),
         }
     }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/all-packages.nix
new file mode 100644
index 0000000000000..07b2caaab4e71
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/all-packages.nix
@@ -0,0 +1,10 @@
+self: super: {
+  nonAttributeSet = self.callPackage ({ someDrv }: someDrv) { };
+  nonCallPackage = self.callPackage ({ someDrv }: someDrv) { };
+  internalCallByName = self.callPackage ({ someDrv }: someDrv) { };
+  nonDerivation = self.callPackage ({ someDrv }: someDrv) { };
+
+  onlyMove = self.callPackage ./pkgs/by-name/on/onlyMove/package.nix { };
+
+  noEval = self.callPackage ./pkgs/by-name/no/noEval/package.nix { };
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/all-packages.nix
new file mode 100644
index 0000000000000..75efb5952e7a6
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/all-packages.nix
@@ -0,0 +1,9 @@
+self: super: {
+  nonAttributeSet = null;
+  nonCallPackage = self.someDrv;
+  internalCallByName = self._internalCallByNamePackageFile ./some-pkg.nix;
+  nonDerivation = self.callPackage ({ }: { }) { };
+
+  onlyMove = self.callPackage ({ someDrv }: someDrv) { };
+  noEval = throw "foo";
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/default.nix
new file mode 100644
index 0000000000000..861260cdca4b2
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/pkgs/by-name/README.md b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/pkgs/by-name/README.md
new file mode 100644
index 0000000000000..e69de29bb2d1d
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/pkgs/by-name/README.md
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/some-pkg.nix b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/some-pkg.nix
new file mode 100644
index 0000000000000..a1b92efbbadb9
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/some-pkg.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/default.nix
new file mode 100644
index 0000000000000..861260cdca4b2
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/expected b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/expected
new file mode 100644
index 0000000000000..29d33f7dbdc00
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/expected
@@ -0,0 +1,2 @@
+pkgs.noEval: This attribute is manually defined (most likely in pkgs/top-level/all-packages.nix), which is only allowed if the definition is of the form `pkgs.callPackage pkgs/by-name/no/noEval/package.nix { ... }` with a non-empty second argument.
+pkgs.onlyMove: This attribute is manually defined (most likely in pkgs/top-level/all-packages.nix), which is only allowed if the definition is of the form `pkgs.callPackage pkgs/by-name/on/onlyMove/package.nix { ... }` with a non-empty second argument.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/pkgs/by-name/no/noEval/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/pkgs/by-name/no/noEval/package.nix
new file mode 100644
index 0000000000000..a1b92efbbadb9
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/pkgs/by-name/no/noEval/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/pkgs/by-name/on/onlyMove/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/pkgs/by-name/on/onlyMove/package.nix
new file mode 100644
index 0000000000000..a1b92efbbadb9
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/pkgs/by-name/on/onlyMove/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv