tests.nixpkgs-check-by-name: Don't enforce for fixed evals

Stops enforcing that packages whose evaluation gets fixed have to be
moved to `pkgs/by-name`.

This didn't really cause problems before, but I don't think it's great
behavior, and now that NonApplicable is a thing, we can easily make this
work, whereas before it would've been a larger change.

5 files changed, 22 insertions, 11 deletions

diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.rs b/pkgs/test/nixpkgs-check-by-name/src/eval.rs
index 5aa64f9bb00db..dd30cb9045e53 100644
--- a/pkgs/test/nixpkgs-check-by-name/src/eval.rs
+++ b/pkgs/test/nixpkgs-check-by-name/src/eval.rs
@@ -202,19 +202,24 @@ pub fn check_values(
                     })
                 }
                 NonByName(EvalFailure) => {
-                    // This is a bit of an odd case: We don't even _know_ whether this attribute
-                    // would qualify for using pkgs/by-name. We can either:
-                    // - Assume it's not using pkgs/by-name, which has the problem that if a
-                    //   package evaluation gets broken temporarily, the fix can remove it from
-                    //   pkgs/by-name again
-                    // - Assume it's using pkgs/by-name already, which has the problem that if a
-                    //   package evaluation gets broken temporarily, fixing it requires a move to
-                    //   pkgs/by-name
-                    // We choose the latter, since we want to move towards pkgs/by-name, not away
-                    // from it
+                    // We don't know anything about this attribute really
                     Success(ratchet::Package {
+                        // We'll assume that we can't remove any manual definitions, which has the
+                        // minimal drawback that if there was a manual definition that could've
+                        // been removed, fixing the package requires removing the definition, no
+                        // big deal, that's a minor edit.
                         manual_definition: Tight,
-                        uses_by_name: Tight,
+
+                        // Regarding whether this attribute could `pkgs/by-name`, we don't really
+                        // know, so return NonApplicable, which has the effect that if a
+                        // package evaluation gets broken temporarily, the fix can remove it from
+                        // pkgs/by-name again. For now this isn't our problem, but in the future we
+                        // might have another check to enforce that evaluation must not be broken.
+                        // The alternative of assuming that it's using `pkgs/by-name` already
+                        // has the problem that if a package evaluation gets broken temporarily,
+                        // fixing it requires a move to pkgs/by-name, which could happen more
+                        // often and isn't really justified.
+                        uses_by_name: NonApplicable,
                     })
                 }
                 ByName(Missing) => NixpkgsProblem::UndefinedAttr {
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/no-eval/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/no-eval/all-packages.nix
index e2831c2d542e6..38762c6de1cc9 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/no-eval/all-packages.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/no-eval/all-packages.nix
@@ -1,3 +1,5 @@
 self: super: {
   iDontEval = throw "I don't eval";
+
+  futureEval = self.callPackage ({ someDrv }: someDrv) { };
 }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/all-packages.nix
new file mode 100644
index 0000000000000..ac763b454eb08
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/all-packages.nix
@@ -0,0 +1,3 @@
+self: super: {
+  futureEval = throw "foo";
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/default.nix
new file mode 100644
index 0000000000000..861260cdca4b2
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/pkgs/by-name/README.md b/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/pkgs/by-name/README.md
new file mode 100644
index 0000000000000..e69de29bb2d1d
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/pkgs/by-name/README.md