Merge pull request #104201 from obsidiansystems/splice-python

python: Splice packages to better support cross

8 files changed, 124 insertions, 21 deletions

diff --git a/lib/customisation.nix b/lib/customisation.nix
index dc5dd76919765..37a7951896b0e 100644
--- a/lib/customisation.nix
+++ b/lib/customisation.nix
@@ -217,4 +217,31 @@ rec {
         };
     in self;
 
+  /* Like the above, but aims to support cross compilation. It's still ugly, but
+     hopefully it helps a little bit. */
+  makeScopeWithSplicing = splicePackages: newScope: otherSplices: keep: f:
+    let
+      spliced = splicePackages {
+        pkgsBuildBuild = otherSplices.selfBuildBuild;
+        pkgsBuildHost = otherSplices.selfBuildHost;
+        pkgsBuildTarget = otherSplices.selfBuildTarget;
+        pkgsHostHost = otherSplices.selfHostHost;
+        pkgsHostTarget = self; # Not `otherSplices.selfHostTarget`;
+        pkgsTargetTarget = otherSplices.selfTargetTarget;
+      } // keep self;
+      self = f self // {
+        newScope = scope: newScope (spliced // scope);
+        callPackage = newScope spliced; # == self.newScope {};
+        # N.B. the other stages of the package set spliced in are *not*
+        # overridden.
+        overrideScope = g: makeScopeWithSplicing
+          splicePackages
+          newScope
+          otherSplices
+          keep
+          (lib.fixedPoints.extends g f);
+        packages = f;
+      };
+    in self;
+
 }
diff --git a/lib/default.nix b/lib/default.nix
index e3c1ed7134637..f985266ed9385 100644
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -101,7 +101,7 @@ let
       noDepEntry fullDepEntry packEntry stringAfter;
     inherit (self.customisation) overrideDerivation makeOverridable
       callPackageWith callPackagesWith extendDerivation hydraJob
-      makeScope;
+      makeScope makeScopeWithSplicing;
     inherit (self.meta) addMetaAttrs dontDistribute setName updateName
       appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio
       hiPrioSet;
diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix
index 2cfaa69a4c8b4..26bd8a8f3609c 100644
--- a/pkgs/development/interpreters/python/cpython/2.7/default.nix
+++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix
@@ -19,6 +19,10 @@
 # For the Python package set
 , packageOverrides ? (self: super: {})
 , buildPackages
+, pkgsBuildBuild
+, pkgsBuildTarget
+, pkgsHostHost
+, pkgsTargetTarget
 , sourceVersion
 , sha256
 , passthruFun
@@ -35,7 +39,8 @@ with stdenv.lib;
 
 let
 
-  pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
+  pythonAttr = "python${sourceVersion.major}${sourceVersion.minor}";
+  pythonForBuild = buildPackages.${pythonAttr};
 
   passthru = passthruFun rec {
     inherit self sourceVersion packageOverrides;
@@ -45,6 +50,10 @@ let
     pythonVersion = with sourceVersion; "${major}.${minor}";
     sitePackages = "lib/${libPrefix}/site-packages";
     inherit hasDistutilsCxxPatch pythonForBuild;
+    pythonPackagesBuildBuild = pkgsBuildBuild.${pythonAttr};
+    pythonPackagesBuildTarget = pkgsBuildTarget.${pythonAttr};
+    pythonPackagesHostHost = pkgsHostHost.${pythonAttr};
+    pythonPackagesTargetTarget = pkgsTargetTarget.${pythonAttr} or {};
   } // {
     inherit ucsEncoding;
   };
diff --git a/pkgs/development/interpreters/python/cpython/default.nix b/pkgs/development/interpreters/python/cpython/default.nix
index 77512c02d5550..02777063a7722 100644
--- a/pkgs/development/interpreters/python/cpython/default.nix
+++ b/pkgs/development/interpreters/python/cpython/default.nix
@@ -20,7 +20,11 @@
 # For the Python package set
 , packageOverrides ? (self: super: {})
 , buildPackages
-, pythonForBuild ? buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"}
+, pkgsBuildBuild
+, pkgsBuildTarget
+, pkgsHostHost
+, pkgsTargetTarget
+, pythonForBuild ? buildPackages.${pythonAttr}
 , sourceVersion
 , sha256
 , passthruFun
@@ -36,6 +40,7 @@
 # Not using optimizations on Darwin
 # configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.
 , enableOptimizations ? (!stdenv.isDarwin)
+, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
 }:
 
 # Note: this package is used for bootstrapping fetchurl, and thus
@@ -54,6 +59,7 @@ with stdenv.lib;
 
 let
 
+
   passthru = passthruFun rec {
     inherit self sourceVersion packageOverrides;
     implementation = "cpython";
@@ -62,6 +68,10 @@ let
     pythonVersion = with sourceVersion; "${major}.${minor}";
     sitePackages = "lib/${libPrefix}/site-packages";
     inherit hasDistutilsCxxPatch pythonForBuild;
+    pythonPackagesBuildBuild = pkgsBuildBuild.${pythonAttr};
+    pythonPackagesBuildTarget = pkgsBuildTarget.${pythonAttr};
+    pythonPackagesHostHost = pkgsHostHost.${pythonAttr};
+    pythonPackagesTargetTarget = pkgsTargetTarget.${pythonAttr} or {};
   };
 
   version = with sourceVersion; "${major}.${minor}.${patch}${suffix}";
diff --git a/pkgs/development/interpreters/python/default.nix b/pkgs/development/interpreters/python/default.nix
index ae666d44f57b2..19a7f44de369c 100644
--- a/pkgs/development/interpreters/python/default.nix
+++ b/pkgs/development/interpreters/python/default.nix
@@ -14,13 +14,70 @@ with pkgs;
     , packageOverrides
     , sitePackages
     , hasDistutilsCxxPatch
-    , pythonForBuild
-    , self
+    , pythonPackagesBuildBuild
+    , pythonForBuild # provides pythonPackagesBuildHost
+    , pythonPackagesBuildTarget
+    , pythonPackagesHostHost
+    , self # is pythonPackagesHostTarget
+    , pythonPackagesTargetTarget
     }: let
-      pythonPackages = callPackage ../../../top-level/python-packages.nix {
-        python = self;
-        overrides = packageOverrides;
-      };
+      pythonPackages = callPackage
+        ({ pkgs, stdenv, python, overrides }: let
+          pythonPackagesFun = import ../../../top-level/python-packages.nix {
+            inherit stdenv pkgs;
+            python = self;
+          };
+          otherSplices = {
+            selfBuildBuild = pythonPackagesBuildBuild;
+            selfBuildHost = pythonForBuild.pkgs;
+            selfBuildTarget = pythonPackagesBuildTarget;
+            selfHostHost = pythonPackagesHostHost;
+            selfTargetTarget = pythonPackagesTargetTarget;
+          };
+          keep = self: {
+            # TODO maybe only define these here so nothing is needed to be kept in sync.
+            inherit (self)
+              isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder
+              python bootstrapped-pip buildPythonPackage buildPythonApplication
+              fetchPypi
+              hasPythonModule requiredPythonModules makePythonPath disabledIf
+              toPythonModule toPythonApplication
+              buildSetupcfg
+
+              eggUnpackHook
+              eggBuildHook
+              eggInstallHook
+              flitBuildHook
+              pipBuildHook
+              pipInstallHook
+              pytestCheckHook
+              pythonCatchConflictsHook
+              pythonImportsCheckHook
+              pythonNamespacesHook
+              pythonRecompileBytecodeHook
+              pythonRemoveBinBytecodeHook
+              pythonRemoveTestsDirHook
+              setuptoolsBuildHook
+              setuptoolsCheckHook
+              venvShellHook
+              wheelUnpackHook
+
+              wrapPython
+
+              pythonPackages
+
+              recursivePthLoader
+            ;
+          };
+        in lib.makeScopeWithSplicing
+          pkgs.splicePackages
+          pkgs.newScope
+          otherSplices
+          keep
+          (lib.extends overrides pythonPackagesFun))
+        {
+          overrides = packageOverrides;
+        };
     in rec {
         isPy27 = pythonVersion == "2.7";
         isPy35 = pythonVersion == "3.5";
@@ -48,7 +105,6 @@ with pkgs;
           python = self;
         };
   };
-
 in {
 
   python27 = callPackage ./cpython/2.7 {
diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix
index d14eb9cbb09dd..456aea4c5d8c6 100644
--- a/pkgs/development/interpreters/python/hooks/default.nix
+++ b/pkgs/development/interpreters/python/hooks/default.nix
@@ -1,7 +1,6 @@
 # Hooks for building Python packages.
 { python
 , lib
-, callPackage
 , makeSetupHook
 , disabledIf
 , isPy3k
@@ -9,6 +8,7 @@
 }:
 
 let
+  callPackage = python.pythonForBuild.pkgs.callPackage;
   pythonInterpreter = python.pythonForBuild.interpreter;
   pythonSitePackages = python.sitePackages;
   pythonCheckInterpreter = python.interpreter;
diff --git a/pkgs/development/interpreters/python/pypy/default.nix b/pkgs/development/interpreters/python/pypy/default.nix
index 482ba5df54ceb..0647ce878642d 100644
--- a/pkgs/development/interpreters/python/pypy/default.nix
+++ b/pkgs/development/interpreters/python/pypy/default.nix
@@ -22,9 +22,15 @@ let
     implementation = "pypy";
     libPrefix = "pypy${pythonVersion}";
     executable = "pypy${if isPy3k then "3" else ""}";
-    pythonForBuild = self; # No cross-compiling for now.
     sitePackages = "site-packages";
     hasDistutilsCxxPatch = false;
+
+    # No cross-compiling for now.
+    pythonForBuild = self;
+    pythonPackagesBuildBuild = {};
+    pythonPackagesBuildTarget = {};
+    pythonPackagesHostHost = {};
+    pythonPackagesTargetTarget = {};
   };
   pname = passthru.executable;
   version = with sourceVersion; "${major}.${minor}.${patch}";
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 5fca264fd3d19..d74d95f17f4c5 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -9,19 +9,16 @@
 { pkgs
 , stdenv
 , python
-, overrides ? (self: super: {})
 }:
 
 with pkgs.lib;
 
-let
-  packages = ( self:
+self:
 
 let
+  inherit (self) callPackage;
   inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder;
 
-  callPackage = pkgs.newScope self;
-
   namePrefix = python.libPrefix + "-";
 
   bootstrapped-pip = callPackage ../development/python-modules/bootstrapped-pip { };
@@ -102,7 +99,7 @@ in {
 
   inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder;
   inherit python bootstrapped-pip buildPythonPackage buildPythonApplication;
-  inherit fetchPypi callPackage;
+  inherit fetchPypi;
   inherit hasPythonModule requiredPythonModules makePythonPath disabledIf;
   inherit toPythonModule toPythonApplication;
   inherit buildSetupcfg;
@@ -7976,6 +7973,4 @@ in {
 
   zxcvbn = callPackage ../development/python-modules/zxcvbn { };
 
-});
-
-in fix' (extends overrides packages)
+}