diff options
author | Domen Kožar <domen@dev.si> | 2024-03-22 19:17:13 +0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-03-22 19:17:13 +0700 |
commit | fb884172ab67e5afc541fb3b45eccd31de7b0828 (patch) | |
tree | 0a275f3c5a52b5978879f10e9ad9c8d575752e18 /pkgs/development/interpreters | |
parent | 350f0a9a66f0017374621855abd53c41f18c6ec6 (diff) | |
parent | 96118850f323f41c163f1f5a8231128879fb2f2e (diff) |
Merge pull request #297628 from cwp/python-env-venv
Fix venv creation in Python environments
Diffstat (limited to 'pkgs/development/interpreters')
9 files changed, 82 insertions, 72 deletions
diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix index dda254fca3894..86b09fa877685 100644 --- a/pkgs/development/interpreters/python/cpython/2.7/default.nix +++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix @@ -318,8 +318,6 @@ in with passthru; stdenv.mkDerivation ({ inherit passthru; postFixup = '' - # Include a sitecustomize.py file. Note it causes an error when it's in postInstall with 2.7. - cp ${../../sitecustomize.py} $out/${sitePackages}/sitecustomize.py '' + lib.optionalString strip2to3 '' rm -R $out/bin/2to3 $out/lib/python*/lib2to3 '' + lib.optionalString stripConfig '' diff --git a/pkgs/development/interpreters/python/cpython/default.nix b/pkgs/development/interpreters/python/cpython/default.nix index 4a0533536c84a..e96685016b25c 100644 --- a/pkgs/development/interpreters/python/cpython/default.nix +++ b/pkgs/development/interpreters/python/cpython/default.nix @@ -537,8 +537,7 @@ in with passthru; stdenv.mkDerivation (finalAttrs: { # Strip tests rm -R $out/lib/python*/test $out/lib/python*/**/test{,s} '' + optionalString includeSiteCustomize '' - # Include a sitecustomize.py file - cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py + '' + optionalString stripBytecode '' # Determinism: deterministic bytecode # First we delete all old bytecode. diff --git a/pkgs/development/interpreters/python/pypy/default.nix b/pkgs/development/interpreters/python/pypy/default.nix index 9b414944bba5b..5724f4944d9c2 100644 --- a/pkgs/development/interpreters/python/pypy/default.nix +++ b/pkgs/development/interpreters/python/pypy/default.nix @@ -126,9 +126,6 @@ in with passthru; stdenv.mkDerivation rec { ln -s $out/${executable}-c/include $out/include/${libPrefix} ln -s $out/${executable}-c/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix} - # Include a sitecustomize.py file - cp ${../sitecustomize.py} $out/${if isPy38OrNewer then sitePackages else "lib/${libPrefix}/${sitePackages}"}/sitecustomize.py - runHook postInstall ''; diff --git a/pkgs/development/interpreters/python/pypy/prebuilt.nix b/pkgs/development/interpreters/python/pypy/prebuilt.nix index 4b47c642eca4c..70f8711ef0866 100644 --- a/pkgs/development/interpreters/python/pypy/prebuilt.nix +++ b/pkgs/development/interpreters/python/pypy/prebuilt.nix @@ -95,9 +95,6 @@ in with passthru; stdenv.mkDerivation { echo "Removing bytecode" find . -name "__pycache__" -type d -depth -delete - # Include a sitecustomize.py file - cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py - runHook postInstall ''; diff --git a/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix b/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix index 37a06f9f61ed5..f0b60c2333f56 100644 --- a/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix +++ b/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix @@ -96,9 +96,6 @@ in with passthru; stdenv.mkDerivation { echo "Removing bytecode" find . -name "__pycache__" -type d -depth -delete - # Include a sitecustomize.py file - cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py - runHook postInstall ''; diff --git a/pkgs/development/interpreters/python/sitecustomize.py b/pkgs/development/interpreters/python/sitecustomize.py deleted file mode 100644 index d79a4696d8eaa..0000000000000 --- a/pkgs/development/interpreters/python/sitecustomize.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -This is a Nix-specific module for discovering modules built with Nix. - -The module recursively adds paths that are on `NIX_PYTHONPATH` to `sys.path`. In -order to process possible `.pth` files `site.addsitedir` is used. - -The paths listed in `PYTHONPATH` are added to `sys.path` afterwards, but they -will be added before the entries we add here and thus take precedence. - -Note the `NIX_PYTHONPATH` environment variable is unset in order to prevent leakage. - -Similarly, this module listens to the environment variable `NIX_PYTHONEXECUTABLE` -and sets `sys.executable` to its value. -""" -import site -import sys -import os -import functools - -paths = os.environ.pop('NIX_PYTHONPATH', None) -if paths: - functools.reduce(lambda k, p: site.addsitedir(p, k), paths.split(':'), site._init_pathinfo()) - -# Check whether we are in a venv or virtualenv. -# For Python 3 we check whether our `base_prefix` is different from our current `prefix`. -# For Python 2 we check whether the non-standard `real_prefix` is set. -# https://stackoverflow.com/questions/1871549/determine-if-python-is-running-inside-virtualenv -in_venv = (sys.version_info.major == 3 and sys.prefix != sys.base_prefix) or (sys.version_info.major == 2 and hasattr(sys, "real_prefix")) - -if not in_venv: - executable = os.environ.pop('NIX_PYTHONEXECUTABLE', None) - prefix = os.environ.pop('NIX_PYTHONPREFIX', None) - - if 'PYTHONEXECUTABLE' not in os.environ and executable is not None: - sys.executable = executable - if prefix is not None: - # Sysconfig does not like it when sys.prefix is set to None - sys.prefix = sys.exec_prefix = prefix - site.PREFIXES.insert(0, prefix) diff --git a/pkgs/development/interpreters/python/tests.nix b/pkgs/development/interpreters/python/tests.nix index df4484f9ec68e..0251a903a7ae8 100644 --- a/pkgs/development/interpreters/python/tests.nix +++ b/pkgs/development/interpreters/python/tests.nix @@ -39,11 +39,21 @@ let is_virtualenv = "False"; }; } // lib.optionalAttrs (!python.isPyPy) { - # Use virtualenv from a Nix env. - nixenv-virtualenv = rec { - env = runCommand "${python.name}-virtualenv" {} '' - ${pythonVirtualEnv.interpreter} -m virtualenv venv - mv venv $out + # Use virtualenv with symlinks from a Nix env. + nixenv-virtualenv-links = rec { + env = runCommand "${python.name}-virtualenv-links" {} '' + ${pythonVirtualEnv.interpreter} -m virtualenv --system-site-packages --symlinks --no-seed $out + ''; + interpreter = "${env}/bin/${python.executable}"; + is_venv = "False"; + is_nixenv = "True"; + is_virtualenv = "True"; + }; + } // lib.optionalAttrs (!python.isPyPy) { + # Use virtualenv with copies from a Nix env. + nixenv-virtualenv-copies = rec { + env = runCommand "${python.name}-virtualenv-copies" {} '' + ${pythonVirtualEnv.interpreter} -m virtualenv --system-site-packages --copies --no-seed $out ''; interpreter = "${env}/bin/${python.executable}"; is_venv = "False"; @@ -59,27 +69,48 @@ let is_nixenv = "True"; is_virtualenv = "False"; }; - } // lib.optionalAttrs (python.isPy3k && (!python.isPyPy)) { - # Venv built using plain Python + } // lib.optionalAttrs (python.pythonAtLeast "3.8" && (!python.isPyPy)) { + # Venv built using links to plain Python # Python 2 does not support venv # TODO: PyPy executable name is incorrect, it should be pypy-c or pypy-3c instead of pypy and pypy3. - plain-venv = rec { - env = runCommand "${python.name}-venv" {} '' - ${python.interpreter} -m venv $out + plain-venv-links = rec { + env = runCommand "${python.name}-venv-links" {} '' + ${python.interpreter} -m venv --system-site-packages --symlinks --without-pip $out + ''; + interpreter = "${env}/bin/${python.executable}"; + is_venv = "True"; + is_nixenv = "False"; + is_virtualenv = "False"; + }; + } // lib.optionalAttrs (python.pythonAtLeast "3.8" && (!python.isPyPy)) { + # Venv built using copies from plain Python + # Python 2 does not support venv + # TODO: PyPy executable name is incorrect, it should be pypy-c or pypy-3c instead of pypy and pypy3. + plain-venv-copies = rec { + env = runCommand "${python.name}-venv-copies" {} '' + ${python.interpreter} -m venv --system-site-packages --copies --without-pip $out ''; interpreter = "${env}/bin/${python.executable}"; is_venv = "True"; is_nixenv = "False"; is_virtualenv = "False"; }; - } // lib.optionalAttrs (python.pythonAtLeast "3.8") { # Venv built using Python Nix environment (python.buildEnv) - # TODO: Cannot create venv from a nix env - # Error: Command '['/nix/store/ddc8nqx73pda86ibvhzdmvdsqmwnbjf7-python3-3.7.6-venv/bin/python3.7', '-Im', 'ensurepip', '--upgrade', '--default-pip']' returned non-zero exit status 1. - nixenv-venv = rec { - env = runCommand "${python.name}-venv" {} '' - ${pythonEnv.interpreter} -m venv $out + nixenv-venv-links = rec { + env = runCommand "${python.name}-venv-links" {} '' + ${pythonEnv.interpreter} -m venv --system-site-packages --symlinks --without-pip $out + ''; + interpreter = "${env}/bin/${pythonEnv.executable}"; + is_venv = "True"; + is_nixenv = "True"; + is_virtualenv = "False"; + }; + } // lib.optionalAttrs (python.pythonAtLeast "3.8") { + # Venv built using Python Nix environment (python.buildEnv) + nixenv-venv-copies = rec { + env = runCommand "${python.name}-venv-copies" {} '' + ${pythonEnv.interpreter} -m venv --system-site-packages --copies --without-pip $out ''; interpreter = "${env}/bin/${pythonEnv.executable}"; is_venv = "True"; @@ -91,11 +122,33 @@ let testfun = name: attrs: runCommand "${python.name}-tests-${name}" ({ inherit (python) pythonVersion; } // attrs) '' + mkdir $out + + # set up the test files cp -r ${./tests/test_environments} tests chmod -R +w tests substituteAllInPlace tests/test_python.py - ${attrs.interpreter} -m unittest discover --verbose tests #/test_python.py - mkdir $out + + # run the tests by invoking the interpreter via full path + echo "absolute path: ${attrs.interpreter}" + ${attrs.interpreter} -m unittest discover --verbose tests 2>&1 | tee "$out/full.txt" + + # run the tests by invoking the interpreter via $PATH + export PATH="$(dirname ${attrs.interpreter}):$PATH" + echo "PATH: $(basename ${attrs.interpreter})" + "$(basename ${attrs.interpreter})" -m unittest discover --verbose tests 2>&1 | tee "$out/path.txt" + + # make sure we get the right path when invoking through a result link + ln -s "${attrs.env}" result + relative="result/bin/$(basename ${attrs.interpreter})" + expected="$PWD/$relative" + actual="$(./$relative -c "import sys; print(sys.executable)" | tee "$out/result.txt")" + if [ "$actual" != "$expected" ]; then + echo "expected $expected, got $actual" + exit 1 + fi + + # if we got this far, the tests passed touch $out/success ''; diff --git a/pkgs/development/interpreters/python/tests/test_environments/test_python.py b/pkgs/development/interpreters/python/tests/test_environments/test_python.py index 0fc4b8a9e91c1..538273f65dbc7 100644 --- a/pkgs/development/interpreters/python/tests/test_environments/test_python.py +++ b/pkgs/development/interpreters/python/tests/test_environments/test_python.py @@ -38,7 +38,7 @@ class TestCasePython(unittest.TestCase): @unittest.skipIf(IS_PYPY or sys.version_info.major==2, "Python 2 does not have base_prefix") def test_base_prefix(self): - if IS_VENV or IS_NIXENV or IS_VIRTUALENV: + if IS_VENV or IS_VIRTUALENV: self.assertNotEqual(sys.prefix, sys.base_prefix) else: self.assertEqual(sys.prefix, sys.base_prefix) diff --git a/pkgs/development/interpreters/python/wrapper.nix b/pkgs/development/interpreters/python/wrapper.nix index f5f9b03e0fd3c..aa568a01b1b0c 100644 --- a/pkgs/development/interpreters/python/wrapper.nix +++ b/pkgs/development/interpreters/python/wrapper.nix @@ -35,6 +35,8 @@ let fi mkdir -p "$out/bin" + rm -f $out/bin/.*-wrapped + for path in ${lib.concatStringsSep " " paths}; do if [ -d "$path/bin" ]; then cd "$path/bin" @@ -42,7 +44,13 @@ let if [ -f "$prg" ]; then rm -f "$out/bin/$prg" if [ -x "$prg" ]; then - makeWrapper "$path/bin/$prg" "$out/bin/$prg" --set NIX_PYTHONPREFIX "$out" --set NIX_PYTHONEXECUTABLE ${pythonExecutable} --set NIX_PYTHONPATH ${pythonPath} ${lib.optionalString (!permitUserSite) ''--set PYTHONNOUSERSITE "true"''} ${lib.concatStringsSep " " makeWrapperArgs} + if [ -f ".$prg-wrapped" ]; then + echo "#!${pythonExecutable}" > "$out/bin/$prg" + sed -e '1d' -e '3d' ".$prg-wrapped" >> "$out/bin/$prg" + chmod +x "$out/bin/$prg" + else + makeWrapper "$path/bin/$prg" "$out/bin/$prg" --inherit-argv0 --resolve-argv0 ${lib.optionalString (!permitUserSite) ''--set PYTHONNOUSERSITE "true"''} ${lib.concatStringsSep " " makeWrapperArgs} + fi fi fi done |