diff options
author | Frederik Rietdijk <fridh@fridh.nl> | 2022-10-03 12:25:02 +0200 |
---|---|---|
committer | Frederik Rietdijk <freddyrietdijk@fridh.nl> | 2022-10-27 10:03:16 +0200 |
commit | 33d12e5f0bcc61d8e12d07fd73ad981f0d42ab59 (patch) | |
tree | 4256db333001b343f095dc99789cb74b5d290dec /pkgs/development | |
parent | 99bcead8dd4698a7bb118b52aaf986074e66acf5 (diff) |
pythonPackages: ensure all derivations provide python modules
This adds a test to ensure no new uses of `buildPythonApplication` can be added to `python-packages.nix`. Python packages can be grouped into two groups: 1) applications and 2) packages providing importable modules. In `python-packages.nix` we only want to have 2). 1) should be in the top-level package set. To achieve this, all setup hooks need to be marked as being a setup hook. For the setup hooks in the Python packages set this is done by creating a new builder, `makePythonHook`. Because there were issues with splicing, the file importing all the hooks is converted to an extension. All non-packages were moved out of `python-packages.nix` into `python-packages-base.nix`. The `keep` argument to `makeScopeWithSplicing was cleaned up as well; there is no need to keep this one manually in sync reducing the risk of breaking cross-compilation.
Diffstat (limited to 'pkgs/development')
4 files changed, 182 insertions, 102 deletions
diff --git a/pkgs/development/interpreters/python/default.nix b/pkgs/development/interpreters/python/default.nix index f770f3fb52d75..1f425d60c63b6 100644 --- a/pkgs/development/interpreters/python/default.nix +++ b/pkgs/development/interpreters/python/default.nix @@ -31,13 +31,19 @@ , pythonAttr ? null , self # is pythonOnHostForTarget }: let - pythonPackages = callPackage + pythonPackages = let + ensurePythonModules = items: let + providesSetupHook = lib.attrByPath [ "provides" "setupHook"] false; + notValid = value: (lib.isDerivation value) && !((pythonPackages.hasPythonModule value) || (providesSetupHook value)); + func = name: value: if !(notValid value) then value else throw "${name} should use `buildPythonPackage` or `toPythonModule` if it is to be part of the Python packages set."; + in lib.mapAttrs func items; + in ensurePythonModules (callPackage # Function that when called # - imports python-packages.nix # - adds spliced package sets to the package set # - applies overrides from `packageOverrides` and `pythonPackagesOverlays`. ({ pkgs, stdenv, python, overrides }: let - pythonPackagesFun = import ../../../top-level/python-packages.nix { + pythonPackagesFun = import ./python-packages-base.nix { inherit stdenv pkgs lib; python = self; }; @@ -48,47 +54,19 @@ selfHostHost = pythonOnHostForHost.pkgs; selfTargetTarget = pythonOnTargetForTarget.pkgs or {}; # There is no Python TargetTarget. }; - keep = self: { - # TODO maybe only define these here so nothing is needed to be kept in sync. - inherit (self) - isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy3k isPyPy pythonAtLeast pythonOlder - python bootstrapped-pip buildPythonPackage buildPythonApplication - fetchPypi - hasPythonModule requiredPythonModules makePythonPath disabledIf - toPythonModule toPythonApplication - buildSetupcfg - - condaInstallHook - condaUnpackHook - eggUnpackHook - eggBuildHook - eggInstallHook - flitBuildHook - pipBuildHook - pipInstallHook - pytestCheckHook - pythonCatchConflictsHook - pythonImportsCheckHook - pythonNamespacesHook - pythonRecompileBytecodeHook - pythonRemoveBinBytecodeHook - pythonRemoveTestsDirHook - setuptoolsBuildHook - setuptoolsCheckHook - venvShellHook - wheelUnpackHook - - wrapPython - - pythonPackages - - recursivePthLoader - ; - }; + hooks = import ./hooks/default.nix; + keep = lib.extends hooks pythonPackagesFun; extra = _: {}; optionalExtensions = cond: as: if cond then as else []; + pythonExtension = import ../../../top-level/python-packages.nix; python2Extension = import ../../../top-level/python2-packages.nix; - extensions = lib.composeManyExtensions ((optionalExtensions (!self.isPy3k) [python2Extension]) ++ pythonPackagesExtensions ++ [ overrides ]); + extensions = lib.composeManyExtensions ([ + pythonExtension + ] ++ (optionalExtensions (!self.isPy3k) [ + python2Extension + ]) ++ pythonPackagesExtensions ++ [ + overrides + ]); aliases = self: super: lib.optionalAttrs config.allowAliases (import ../../../top-level/python-aliases.nix lib self super); in lib.makeScopeWithSplicing splicePackages @@ -96,11 +74,11 @@ otherSplices keep extra - (lib.extends (lib.composeExtensions aliases extensions) pythonPackagesFun)) + (lib.extends (lib.composeExtensions aliases extensions) keep)) { overrides = packageOverrides; python = self; - }; + }); in rec { isPy27 = pythonVersion == "2.7"; isPy35 = pythonVersion == "3.5"; diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix index e75d78758f983..d30876aa9fde9 100644 --- a/pkgs/development/interpreters/python/hooks/default.nix +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -1,21 +1,15 @@ -# Hooks for building Python packages. -{ python -, lib -, makeSetupHook -, disabledIf -, isPy3k -}: +self: super: with self; let - callPackage = python.pythonForBuild.pkgs.callPackage; - pythonInterpreter = python.pythonForBuild.interpreter; - pythonSitePackages = python.sitePackages; - pythonCheckInterpreter = python.interpreter; + pythonInterpreter = super.python.pythonForBuild.interpreter; + pythonSitePackages = super.python.sitePackages; + pythonCheckInterpreter = super.python.interpreter; setuppy = ../run_setup.py; -in rec { +in { + makePythonHook = args: pkgs.makeSetupHook ({passthru.provides.setupHook = true; } // args); - condaInstallHook = callPackage ({ gnutar, lbzip2 }: - makeSetupHook { + condaInstallHook = callPackage ({ makePythonHook, gnutar, lbzip2 }: + makePythonHook { name = "conda-install-hook"; deps = [ gnutar lbzip2 ]; substitutions = { @@ -23,20 +17,20 @@ in rec { }; } ./conda-install-hook.sh) {}; - condaUnpackHook = callPackage ({}: - makeSetupHook { + condaUnpackHook = callPackage ({ makePythonHook }: + makePythonHook { name = "conda-unpack-hook"; deps = []; } ./conda-unpack-hook.sh) {}; - eggBuildHook = callPackage ({ }: - makeSetupHook { + eggBuildHook = callPackage ({ makePythonHook }: + makePythonHook { name = "egg-build-hook.sh"; deps = [ ]; } ./egg-build-hook.sh) {}; - eggInstallHook = callPackage ({ setuptools }: - makeSetupHook { + eggInstallHook = callPackage ({ makePythonHook, setuptools }: + makePythonHook { name = "egg-install-hook.sh"; deps = [ setuptools ]; substitutions = { @@ -44,14 +38,14 @@ in rec { }; } ./egg-install-hook.sh) {}; - eggUnpackHook = callPackage ({ }: - makeSetupHook { + eggUnpackHook = callPackage ({ makePythonHook, }: + makePythonHook { name = "egg-unpack-hook.sh"; deps = [ ]; } ./egg-unpack-hook.sh) {}; - flitBuildHook = callPackage ({ flit }: - makeSetupHook { + flitBuildHook = callPackage ({ makePythonHook, flit }: + makePythonHook { name = "flit-build-hook"; deps = [ flit ]; substitutions = { @@ -59,8 +53,8 @@ in rec { }; } ./flit-build-hook.sh) {}; - pipBuildHook = callPackage ({ pip, wheel }: - makeSetupHook { + pipBuildHook = callPackage ({ makePythonHook, pip, wheel }: + makePythonHook { name = "pip-build-hook.sh"; deps = [ pip wheel ]; substitutions = { @@ -68,8 +62,8 @@ in rec { }; } ./pip-build-hook.sh) {}; - pipInstallHook = callPackage ({ pip }: - makeSetupHook { + pipInstallHook = callPackage ({ makePythonHook, pip }: + makePythonHook { name = "pip-install-hook"; deps = [ pip ]; substitutions = { @@ -77,8 +71,8 @@ in rec { }; } ./pip-install-hook.sh) {}; - pytestCheckHook = callPackage ({ pytest }: - makeSetupHook { + pytestCheckHook = callPackage ({ makePythonHook, pytest }: + makePythonHook { name = "pytest-check-hook"; deps = [ pytest ]; substitutions = { @@ -86,8 +80,8 @@ in rec { }; } ./pytest-check-hook.sh) {}; - pythonCatchConflictsHook = callPackage ({ setuptools }: - makeSetupHook { + pythonCatchConflictsHook = callPackage ({ makePythonHook, setuptools }: + makePythonHook { name = "python-catch-conflicts-hook"; substitutions = { inherit pythonInterpreter pythonSitePackages setuptools; @@ -95,29 +89,29 @@ in rec { }; } ./python-catch-conflicts-hook.sh) {}; - pythonImportsCheckHook = callPackage ({}: - makeSetupHook { + pythonImportsCheckHook = callPackage ({ makePythonHook }: + makePythonHook { name = "python-imports-check-hook.sh"; substitutions = { inherit pythonCheckInterpreter; }; } ./python-imports-check-hook.sh) {}; - pythonNamespacesHook = callPackage ({ findutils }: - makeSetupHook { + pythonNamespacesHook = callPackage ({ makePythonHook, findutils }: + makePythonHook { name = "python-namespaces-hook.sh"; substitutions = { inherit pythonSitePackages findutils; }; } ./python-namespaces-hook.sh) {}; - pythonOutputDistHook = callPackage ({ }: - makeSetupHook { + pythonOutputDistHook = callPackage ({ makePythonHook }: + makePythonHook { name = "python-output-dist-hook"; } ./python-output-dist-hook.sh ) {}; - pythonRecompileBytecodeHook = callPackage ({ }: - makeSetupHook { + pythonRecompileBytecodeHook = callPackage ({ makePythonHook }: + makePythonHook { name = "python-recompile-bytecode-hook"; substitutions = { inherit pythonInterpreter pythonSitePackages; @@ -126,8 +120,8 @@ in rec { }; } ./python-recompile-bytecode-hook.sh ) {}; - pythonRelaxDepsHook = callPackage ({ wheel }: - makeSetupHook { + pythonRelaxDepsHook = callPackage ({ makePythonHook, wheel }: + makePythonHook { name = "python-relax-deps-hook"; deps = [ wheel ]; substitutions = { @@ -135,21 +129,21 @@ in rec { }; } ./python-relax-deps-hook.sh) {}; - pythonRemoveBinBytecodeHook = callPackage ({ }: - makeSetupHook { + pythonRemoveBinBytecodeHook = callPackage ({ makePythonHook }: + makePythonHook { name = "python-remove-bin-bytecode-hook"; } ./python-remove-bin-bytecode-hook.sh) {}; - pythonRemoveTestsDirHook = callPackage ({ }: - makeSetupHook { + pythonRemoveTestsDirHook = callPackage ({ makePythonHook }: + makePythonHook { name = "python-remove-tests-dir-hook"; substitutions = { inherit pythonSitePackages; }; } ./python-remove-tests-dir-hook.sh) {}; - setuptoolsBuildHook = callPackage ({ setuptools, wheel }: - makeSetupHook { + setuptoolsBuildHook = callPackage ({ makePythonHook, setuptools, wheel }: + makePythonHook { name = "setuptools-setup-hook"; deps = [ setuptools wheel ]; substitutions = { @@ -157,8 +151,8 @@ in rec { }; } ./setuptools-build-hook.sh) {}; - setuptoolsCheckHook = callPackage ({ setuptools }: - makeSetupHook { + setuptoolsCheckHook = callPackage ({ makePythonHook, setuptools }: + makePythonHook { name = "setuptools-check-hook"; deps = [ setuptools ]; substitutions = { @@ -166,16 +160,16 @@ in rec { }; } ./setuptools-check-hook.sh) {}; - unittestCheckHook = callPackage ({ }: - makeSetupHook { + unittestCheckHook = callPackage ({ makePythonHook }: + makePythonHook { name = "unittest-check-hook"; substitutions = { inherit pythonCheckInterpreter; }; } ./unittest-check-hook.sh) {}; - venvShellHook = disabledIf (!isPy3k) (callPackage ({ ensureNewerSourcesForZipFilesHook }: - makeSetupHook { + venvShellHook = disabledIf (!isPy3k) (callPackage ({ makePythonHook, ensureNewerSourcesForZipFilesHook }: + makePythonHook { name = "venv-shell-hook"; deps = [ ensureNewerSourcesForZipFilesHook ]; substitutions = { @@ -183,14 +177,18 @@ in rec { }; } ./venv-shell-hook.sh) {}); - wheelUnpackHook = callPackage ({ wheel }: - makeSetupHook { + wheelUnpackHook = callPackage ({ makePythonHook, wheel }: + makePythonHook { name = "wheel-unpack-hook.sh"; deps = [ wheel ]; } ./wheel-unpack-hook.sh) {}; - sphinxHook = callPackage ({ sphinx, installShellFiles }: - makeSetupHook { + wrapPython = callPackage ../wrap-python.nix { + inherit (pkgs.buildPackages) makeWrapper; + }; + + sphinxHook = callPackage ({ makePythonHook, sphinx, installShellFiles }: + makePythonHook { name = "python${python.pythonVersion}-sphinx-hook"; deps = [ sphinx installShellFiles ]; } ./sphinx-hook.sh) {}; diff --git a/pkgs/development/interpreters/python/python-packages-base.nix b/pkgs/development/interpreters/python/python-packages-base.nix new file mode 100644 index 0000000000000..a63ece3c23699 --- /dev/null +++ b/pkgs/development/interpreters/python/python-packages-base.nix @@ -0,0 +1,104 @@ +{ pkgs +, stdenv +, lib +, python +}: + +self: + +let + inherit (self) callPackage; + inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy311 isPy3k isPyPy pythonAtLeast pythonOlder; + + namePrefix = python.libPrefix + "-"; + + # Derivations built with `buildPythonPackage` can already be overriden with `override`, `overrideAttrs`, and `overrideDerivation`. + # This function introduces `overridePythonAttrs` and it overrides the call to `buildPythonPackage`. + makeOverridablePythonPackage = f: origArgs: + let + ff = f origArgs; + overrideWith = newArgs: origArgs // (if pkgs.lib.isFunction newArgs then newArgs origArgs else newArgs); + in + if builtins.isAttrs ff then (ff // { + overridePythonAttrs = newArgs: makeOverridablePythonPackage f (overrideWith newArgs); + }) + else if builtins.isFunction ff then { + overridePythonAttrs = newArgs: makeOverridablePythonPackage f (overrideWith newArgs); + __functor = self: ff; + } + else ff; + + buildPythonPackage = makeOverridablePythonPackage (lib.makeOverridable (callPackage ./mk-python-derivation.nix { + inherit namePrefix; # We want Python libraries to be named like e.g. "python3.6-${name}" + inherit toPythonModule; # Libraries provide modules + })); + + buildPythonApplication = makeOverridablePythonPackage (lib.makeOverridable (callPackage ./mk-python-derivation.nix { + namePrefix = ""; # Python applications should not have any prefix + toPythonModule = x: x; # Application does not provide modules. + })); + + # See build-setupcfg/default.nix for documentation. + buildSetupcfg = import ../../../build-support/build-setupcfg self; + + fetchPypi = callPackage ./fetchpypi.nix { }; + + # Check whether a derivation provides a Python module. + hasPythonModule = drv: drv?pythonModule && drv.pythonModule == python; + + # Get list of required Python modules given a list of derivations. + requiredPythonModules = drvs: let + modules = lib.filter hasPythonModule drvs; + in lib.unique ([python] ++ modules ++ lib.concatLists (lib.catAttrs "requiredPythonModules" modules)); + + # Create a PYTHONPATH from a list of derivations. This function recurses into the items to find derivations + # providing Python modules. + makePythonPath = drvs: lib.makeSearchPath python.sitePackages (requiredPythonModules drvs); + + removePythonPrefix = lib.removePrefix namePrefix; + + # Convert derivation to a Python module. + toPythonModule = drv: + drv.overrideAttrs( oldAttrs: { + # Use passthru in order to prevent rebuilds when possible. + passthru = (oldAttrs.passthru or {})// { + pythonModule = python; + pythonPath = [ ]; # Deprecated, for compatibility. + requiredPythonModules = requiredPythonModules drv.propagatedBuildInputs; + }; + }); + + # Convert a Python library to an application. + toPythonApplication = drv: + drv.overrideAttrs( oldAttrs: { + passthru = (oldAttrs.passthru or {}) // { + # Remove Python prefix from name so we have a "normal" name. + # While the prefix shows up in the store path, it won't be + # used by `nix-env`. + name = removePythonPrefix oldAttrs.name; + pythonModule = false; + }; + }); + + disabled = drv: throw "${removePythonPrefix (drv.pname or drv.name)} not supported for interpreter ${python.executable}"; + + disabledIf = x: drv: if x then disabled drv else drv; + +in { + + inherit lib pkgs stdenv; + inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy311 isPy3k isPyPy pythonAtLeast pythonOlder; + inherit buildPythonPackage buildPythonApplication; + inherit fetchPypi; + inherit hasPythonModule requiredPythonModules makePythonPath disabled disabledIf; + inherit toPythonModule toPythonApplication; + inherit buildSetupcfg; + + python = toPythonModule python; + # Dont take pythonPackages from "global" pkgs scope to avoid mixing python versions + pythonPackages = self; + + # Remove? + recursivePthLoader = toPythonModule (callPackage ../../../development/python-modules/recursive-pth-loader { }); + +} diff --git a/pkgs/development/interpreters/python/wrap-python.nix b/pkgs/development/interpreters/python/wrap-python.nix index 6a19a21524190..29fc6cf820b20 100644 --- a/pkgs/development/interpreters/python/wrap-python.nix +++ b/pkgs/development/interpreters/python/wrap-python.nix @@ -1,11 +1,11 @@ { lib , python -, makeSetupHook +, makePythonHook , makeWrapper }: with lib; -makeSetupHook { +makePythonHook { deps = makeWrapper; substitutions.sitePackages = python.sitePackages; substitutions.executable = python.interpreter; |