pythonPackages: ensure all derivations provide python modules

This adds a test to ensure no new uses of `buildPythonApplication` can
be added to `python-packages.nix`.

Python packages can be grouped into two groups: 1) applications and 2)
packages providing importable modules. In `python-packages.nix` we only
want to have 2). 1) should be in the top-level package set.

To achieve this, all setup hooks need to be marked as being a setup hook.
For the setup hooks in the Python packages set this is done by creating
a new builder, `makePythonHook`.

Because there were issues with splicing, the file importing all the hooks
is converted to an extension. All non-packages were moved out of `python-packages.nix`
into `python-packages-base.nix`. The `keep` argument to `makeScopeWithSplicing
was cleaned up as well; there is no need to keep this one manually in sync
reducing the risk of breaking cross-compilation.

4 files changed, 182 insertions, 102 deletions

diff --git a/pkgs/development/interpreters/python/default.nix b/pkgs/development/interpreters/python/default.nix
index f770f3fb52d75..1f425d60c63b6 100644
--- a/pkgs/development/interpreters/python/default.nix
+++ b/pkgs/development/interpreters/python/default.nix
@@ -31,13 +31,19 @@
     , pythonAttr ? null
     , self # is pythonOnHostForTarget
     }: let
-      pythonPackages = callPackage
+      pythonPackages = let
+        ensurePythonModules = items: let
+          providesSetupHook = lib.attrByPath [ "provides" "setupHook"] false;
+          notValid = value: (lib.isDerivation value) && !((pythonPackages.hasPythonModule value) || (providesSetupHook value));
+          func = name: value: if !(notValid value) then value else throw "${name} should use `buildPythonPackage` or `toPythonModule` if it is to be part of the Python packages set.";
+        in lib.mapAttrs func items;
+      in ensurePythonModules (callPackage
         # Function that when called
         # - imports python-packages.nix
         # - adds spliced package sets to the package set
         # - applies overrides from `packageOverrides` and `pythonPackagesOverlays`.
         ({ pkgs, stdenv, python, overrides }: let
-          pythonPackagesFun = import ../../../top-level/python-packages.nix {
+          pythonPackagesFun = import ./python-packages-base.nix {
             inherit stdenv pkgs lib;
             python = self;
           };
@@ -48,47 +54,19 @@
             selfHostHost = pythonOnHostForHost.pkgs;
             selfTargetTarget = pythonOnTargetForTarget.pkgs or {}; # There is no Python TargetTarget.
           };
-          keep = self: {
-            # TODO maybe only define these here so nothing is needed to be kept in sync.
-            inherit (self)
-              isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy3k isPyPy pythonAtLeast pythonOlder
-              python bootstrapped-pip buildPythonPackage buildPythonApplication
-              fetchPypi
-              hasPythonModule requiredPythonModules makePythonPath disabledIf
-              toPythonModule toPythonApplication
-              buildSetupcfg
-
-              condaInstallHook
-              condaUnpackHook
-              eggUnpackHook
-              eggBuildHook
-              eggInstallHook
-              flitBuildHook
-              pipBuildHook
-              pipInstallHook
-              pytestCheckHook
-              pythonCatchConflictsHook
-              pythonImportsCheckHook
-              pythonNamespacesHook
-              pythonRecompileBytecodeHook
-              pythonRemoveBinBytecodeHook
-              pythonRemoveTestsDirHook
-              setuptoolsBuildHook
-              setuptoolsCheckHook
-              venvShellHook
-              wheelUnpackHook
-
-              wrapPython
-
-              pythonPackages
-
-              recursivePthLoader
-            ;
-          };
+          hooks = import ./hooks/default.nix;
+          keep = lib.extends hooks pythonPackagesFun;
           extra = _: {};
           optionalExtensions = cond: as: if cond then as else [];
+          pythonExtension = import ../../../top-level/python-packages.nix;
           python2Extension = import ../../../top-level/python2-packages.nix;
-          extensions = lib.composeManyExtensions ((optionalExtensions (!self.isPy3k) [python2Extension]) ++ pythonPackagesExtensions ++ [ overrides ]);
+          extensions = lib.composeManyExtensions ([
+            pythonExtension
+          ] ++ (optionalExtensions (!self.isPy3k) [
+            python2Extension
+          ]) ++ pythonPackagesExtensions ++ [
+            overrides
+          ]);
           aliases = self: super: lib.optionalAttrs config.allowAliases (import ../../../top-level/python-aliases.nix lib self super);
         in lib.makeScopeWithSplicing
           splicePackages
@@ -96,11 +74,11 @@
           otherSplices
           keep
           extra
-          (lib.extends (lib.composeExtensions aliases extensions) pythonPackagesFun))
+          (lib.extends (lib.composeExtensions aliases extensions) keep))
         {
           overrides = packageOverrides;
           python = self;
-        };
+        });
     in rec {
         isPy27 = pythonVersion == "2.7";
         isPy35 = pythonVersion == "3.5";
diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix
index e75d78758f983..d30876aa9fde9 100644
--- a/pkgs/development/interpreters/python/hooks/default.nix
+++ b/pkgs/development/interpreters/python/hooks/default.nix
@@ -1,21 +1,15 @@
-# Hooks for building Python packages.
-{ python
-, lib
-, makeSetupHook
-, disabledIf
-, isPy3k
-}:
+self: super: with self;
 
 let
-  callPackage = python.pythonForBuild.pkgs.callPackage;
-  pythonInterpreter = python.pythonForBuild.interpreter;
-  pythonSitePackages = python.sitePackages;
-  pythonCheckInterpreter = python.interpreter;
+  pythonInterpreter = super.python.pythonForBuild.interpreter;
+  pythonSitePackages = super.python.sitePackages;
+  pythonCheckInterpreter = super.python.interpreter;
   setuppy = ../run_setup.py;
-in rec {
+in {
+  makePythonHook = args: pkgs.makeSetupHook ({passthru.provides.setupHook = true; } // args);
 
-  condaInstallHook = callPackage ({ gnutar, lbzip2 }:
-    makeSetupHook {
+  condaInstallHook = callPackage ({ makePythonHook, gnutar, lbzip2 }:
+    makePythonHook {
       name = "conda-install-hook";
       deps = [ gnutar lbzip2 ];
       substitutions = {
@@ -23,20 +17,20 @@ in rec {
       };
     } ./conda-install-hook.sh) {};
 
-  condaUnpackHook = callPackage ({}:
-    makeSetupHook {
+  condaUnpackHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "conda-unpack-hook";
       deps = [];
     } ./conda-unpack-hook.sh) {};
 
-  eggBuildHook = callPackage ({ }:
-    makeSetupHook {
+  eggBuildHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "egg-build-hook.sh";
       deps = [ ];
     } ./egg-build-hook.sh) {};
 
-  eggInstallHook = callPackage ({ setuptools }:
-    makeSetupHook {
+  eggInstallHook = callPackage ({ makePythonHook, setuptools }:
+    makePythonHook {
       name = "egg-install-hook.sh";
       deps = [ setuptools ];
       substitutions = {
@@ -44,14 +38,14 @@ in rec {
       };
     } ./egg-install-hook.sh) {};
 
-  eggUnpackHook = callPackage ({ }:
-    makeSetupHook {
+  eggUnpackHook = callPackage ({ makePythonHook, }:
+    makePythonHook {
       name = "egg-unpack-hook.sh";
       deps = [ ];
     } ./egg-unpack-hook.sh) {};
 
-  flitBuildHook = callPackage ({ flit }:
-    makeSetupHook {
+  flitBuildHook = callPackage ({ makePythonHook, flit }:
+    makePythonHook {
       name = "flit-build-hook";
       deps = [ flit ];
       substitutions = {
@@ -59,8 +53,8 @@ in rec {
       };
     } ./flit-build-hook.sh) {};
 
-  pipBuildHook = callPackage ({ pip, wheel }:
-    makeSetupHook {
+  pipBuildHook = callPackage ({ makePythonHook, pip, wheel }:
+    makePythonHook {
       name = "pip-build-hook.sh";
       deps = [ pip wheel ];
       substitutions = {
@@ -68,8 +62,8 @@ in rec {
       };
     } ./pip-build-hook.sh) {};
 
-  pipInstallHook = callPackage ({ pip }:
-    makeSetupHook {
+  pipInstallHook = callPackage ({ makePythonHook, pip }:
+    makePythonHook {
       name = "pip-install-hook";
       deps = [ pip ];
       substitutions = {
@@ -77,8 +71,8 @@ in rec {
       };
     } ./pip-install-hook.sh) {};
 
-  pytestCheckHook = callPackage ({ pytest }:
-    makeSetupHook {
+  pytestCheckHook = callPackage ({ makePythonHook, pytest }:
+    makePythonHook {
       name = "pytest-check-hook";
       deps = [ pytest ];
       substitutions = {
@@ -86,8 +80,8 @@ in rec {
       };
     } ./pytest-check-hook.sh) {};
 
-  pythonCatchConflictsHook = callPackage ({ setuptools }:
-    makeSetupHook {
+  pythonCatchConflictsHook = callPackage ({ makePythonHook, setuptools }:
+    makePythonHook {
       name = "python-catch-conflicts-hook";
       substitutions = {
         inherit pythonInterpreter pythonSitePackages setuptools;
@@ -95,29 +89,29 @@ in rec {
       };
     } ./python-catch-conflicts-hook.sh) {};
 
-  pythonImportsCheckHook = callPackage ({}:
-    makeSetupHook {
+  pythonImportsCheckHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-imports-check-hook.sh";
       substitutions = {
         inherit pythonCheckInterpreter;
       };
     } ./python-imports-check-hook.sh) {};
 
-  pythonNamespacesHook = callPackage ({ findutils }:
-    makeSetupHook {
+  pythonNamespacesHook = callPackage ({ makePythonHook, findutils }:
+    makePythonHook {
       name = "python-namespaces-hook.sh";
       substitutions = {
         inherit pythonSitePackages findutils;
       };
     } ./python-namespaces-hook.sh) {};
 
-  pythonOutputDistHook = callPackage ({ }:
-    makeSetupHook {
+  pythonOutputDistHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-output-dist-hook";
   } ./python-output-dist-hook.sh ) {};
 
-  pythonRecompileBytecodeHook = callPackage ({ }:
-    makeSetupHook {
+  pythonRecompileBytecodeHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-recompile-bytecode-hook";
       substitutions = {
         inherit pythonInterpreter pythonSitePackages;
@@ -126,8 +120,8 @@ in rec {
       };
     } ./python-recompile-bytecode-hook.sh ) {};
 
-  pythonRelaxDepsHook = callPackage ({ wheel }:
-    makeSetupHook {
+  pythonRelaxDepsHook = callPackage ({ makePythonHook, wheel }:
+    makePythonHook {
       name = "python-relax-deps-hook";
       deps = [ wheel ];
       substitutions = {
@@ -135,21 +129,21 @@ in rec {
       };
     } ./python-relax-deps-hook.sh) {};
 
-  pythonRemoveBinBytecodeHook = callPackage ({ }:
-    makeSetupHook {
+  pythonRemoveBinBytecodeHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-remove-bin-bytecode-hook";
     } ./python-remove-bin-bytecode-hook.sh) {};
 
-  pythonRemoveTestsDirHook = callPackage ({ }:
-    makeSetupHook {
+  pythonRemoveTestsDirHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-remove-tests-dir-hook";
       substitutions = {
         inherit pythonSitePackages;
       };
     } ./python-remove-tests-dir-hook.sh) {};
 
-  setuptoolsBuildHook = callPackage ({ setuptools, wheel }:
-    makeSetupHook {
+  setuptoolsBuildHook = callPackage ({ makePythonHook, setuptools, wheel }:
+    makePythonHook {
       name = "setuptools-setup-hook";
       deps = [ setuptools wheel ];
       substitutions = {
@@ -157,8 +151,8 @@ in rec {
       };
     } ./setuptools-build-hook.sh) {};
 
-  setuptoolsCheckHook = callPackage ({ setuptools }:
-    makeSetupHook {
+  setuptoolsCheckHook = callPackage ({ makePythonHook, setuptools }:
+    makePythonHook {
       name = "setuptools-check-hook";
       deps = [ setuptools ];
       substitutions = {
@@ -166,16 +160,16 @@ in rec {
       };
     } ./setuptools-check-hook.sh) {};
 
-  unittestCheckHook = callPackage ({ }:
-    makeSetupHook {
+  unittestCheckHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "unittest-check-hook";
       substitutions = {
         inherit pythonCheckInterpreter;
       };
     } ./unittest-check-hook.sh) {};
 
-  venvShellHook = disabledIf (!isPy3k) (callPackage ({ ensureNewerSourcesForZipFilesHook }:
-    makeSetupHook {
+  venvShellHook = disabledIf (!isPy3k) (callPackage ({ makePythonHook, ensureNewerSourcesForZipFilesHook }:
+    makePythonHook {
       name = "venv-shell-hook";
       deps = [ ensureNewerSourcesForZipFilesHook ];
       substitutions = {
@@ -183,14 +177,18 @@ in rec {
       };
     } ./venv-shell-hook.sh) {});
 
-  wheelUnpackHook = callPackage ({ wheel }:
-    makeSetupHook {
+  wheelUnpackHook = callPackage ({ makePythonHook, wheel }:
+    makePythonHook {
       name = "wheel-unpack-hook.sh";
       deps = [ wheel ];
     } ./wheel-unpack-hook.sh) {};
 
-  sphinxHook = callPackage ({ sphinx, installShellFiles }:
-    makeSetupHook {
+  wrapPython = callPackage ../wrap-python.nix {
+    inherit (pkgs.buildPackages) makeWrapper;
+  };
+
+  sphinxHook = callPackage ({ makePythonHook, sphinx, installShellFiles }:
+    makePythonHook {
       name = "python${python.pythonVersion}-sphinx-hook";
       deps = [ sphinx installShellFiles ];
     } ./sphinx-hook.sh) {};
diff --git a/pkgs/development/interpreters/python/python-packages-base.nix b/pkgs/development/interpreters/python/python-packages-base.nix
new file mode 100644
index 0000000000000..a63ece3c23699
--- /dev/null
+++ b/pkgs/development/interpreters/python/python-packages-base.nix
@@ -0,0 +1,104 @@
+{ pkgs
+, stdenv
+, lib
+, python
+}:
+
+self:
+
+let
+  inherit (self) callPackage;
+  inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy311 isPy3k isPyPy pythonAtLeast pythonOlder;
+
+  namePrefix = python.libPrefix + "-";
+
+  # Derivations built with `buildPythonPackage` can already be overriden with `override`, `overrideAttrs`, and `overrideDerivation`.
+  # This function introduces `overridePythonAttrs` and it overrides the call to `buildPythonPackage`.
+  makeOverridablePythonPackage = f: origArgs:
+    let
+      ff = f origArgs;
+      overrideWith = newArgs: origArgs // (if pkgs.lib.isFunction newArgs then newArgs origArgs else newArgs);
+    in
+      if builtins.isAttrs ff then (ff // {
+        overridePythonAttrs = newArgs: makeOverridablePythonPackage f (overrideWith newArgs);
+      })
+      else if builtins.isFunction ff then {
+        overridePythonAttrs = newArgs: makeOverridablePythonPackage f (overrideWith newArgs);
+        __functor = self: ff;
+      }
+      else ff;
+
+  buildPythonPackage = makeOverridablePythonPackage (lib.makeOverridable (callPackage ./mk-python-derivation.nix {
+    inherit namePrefix;     # We want Python libraries to be named like e.g. "python3.6-${name}"
+    inherit toPythonModule; # Libraries provide modules
+  }));
+
+  buildPythonApplication = makeOverridablePythonPackage (lib.makeOverridable (callPackage ./mk-python-derivation.nix {
+    namePrefix = "";        # Python applications should not have any prefix
+    toPythonModule = x: x;  # Application does not provide modules.
+  }));
+
+  # See build-setupcfg/default.nix for documentation.
+  buildSetupcfg = import ../../../build-support/build-setupcfg self;
+
+  fetchPypi = callPackage ./fetchpypi.nix { };
+
+  # Check whether a derivation provides a Python module.
+  hasPythonModule = drv: drv?pythonModule && drv.pythonModule == python;
+
+  # Get list of required Python modules given a list of derivations.
+  requiredPythonModules = drvs: let
+    modules = lib.filter hasPythonModule drvs;
+  in lib.unique ([python] ++ modules ++ lib.concatLists (lib.catAttrs "requiredPythonModules" modules));
+
+  # Create a PYTHONPATH from a list of derivations. This function recurses into the items to find derivations
+  # providing Python modules.
+  makePythonPath = drvs: lib.makeSearchPath python.sitePackages (requiredPythonModules drvs);
+
+  removePythonPrefix = lib.removePrefix namePrefix;
+
+  # Convert derivation to a Python module.
+  toPythonModule = drv:
+    drv.overrideAttrs( oldAttrs: {
+      # Use passthru in order to prevent rebuilds when possible.
+      passthru = (oldAttrs.passthru or {})// {
+        pythonModule = python;
+        pythonPath = [ ]; # Deprecated, for compatibility.
+        requiredPythonModules = requiredPythonModules drv.propagatedBuildInputs;
+      };
+    });
+
+  # Convert a Python library to an application.
+  toPythonApplication = drv:
+    drv.overrideAttrs( oldAttrs: {
+      passthru = (oldAttrs.passthru or {}) // {
+        # Remove Python prefix from name so we have a "normal" name.
+        # While the prefix shows up in the store path, it won't be
+        # used by `nix-env`.
+        name = removePythonPrefix oldAttrs.name;
+        pythonModule = false;
+      };
+    });
+
+  disabled = drv: throw "${removePythonPrefix (drv.pname or drv.name)} not supported for interpreter ${python.executable}";
+
+  disabledIf = x: drv: if x then disabled drv else drv;
+
+in {
+
+  inherit lib pkgs stdenv;
+  inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy311 isPy3k isPyPy pythonAtLeast pythonOlder;
+  inherit buildPythonPackage buildPythonApplication;
+  inherit fetchPypi;
+  inherit hasPythonModule requiredPythonModules makePythonPath disabled disabledIf;
+  inherit toPythonModule toPythonApplication;
+  inherit buildSetupcfg;
+
+  python = toPythonModule python;
+  # Dont take pythonPackages from "global" pkgs scope to avoid mixing python versions
+  pythonPackages = self;
+
+  # Remove?
+  recursivePthLoader = toPythonModule (callPackage ../../../development/python-modules/recursive-pth-loader { });
+
+}
diff --git a/pkgs/development/interpreters/python/wrap-python.nix b/pkgs/development/interpreters/python/wrap-python.nix
index 6a19a21524190..29fc6cf820b20 100644
--- a/pkgs/development/interpreters/python/wrap-python.nix
+++ b/pkgs/development/interpreters/python/wrap-python.nix
@@ -1,11 +1,11 @@
 { lib
 , python
-, makeSetupHook
+, makePythonHook
 , makeWrapper }:
 
 with lib;
 
-makeSetupHook {
+makePythonHook {
       deps = makeWrapper;
       substitutions.sitePackages = python.sitePackages;
       substitutions.executable = python.interpreter;