diff options
18 files changed, 445 insertions, 19 deletions
diff --git a/doc/languages-frameworks/javascript.section.md b/doc/languages-frameworks/javascript.section.md index 4dc207f798479..c148070ad244f 100644 --- a/doc/languages-frameworks/javascript.section.md +++ b/doc/languages-frameworks/javascript.section.md @@ -233,6 +233,37 @@ sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= It returns a derivation with all `package-lock.json` dependencies downloaded into `$out/`, usable as an npm cache. +#### importNpmLock {#javascript-buildNpmPackage-importNpmLock} + +`importNpmLock` is a Nix function that requires the following optional arguments: + +- `npmRoot`: Path to package directory containing the source tree +- `package`: Parsed contents of `package.json` +- `packageLock`: Parsed contents of `package-lock.json` +- `pname`: Package name +- `version`: Package version + +It returns a derivation with a patched `package.json` & `package-lock.json` with all dependencies resolved to Nix store paths. + +This function is analogous to using `fetchNpmDeps`, but instead of specifying `hash` it uses metadata from `package.json` & `package-lock.json`. + +Note that `npmHooks.npmConfigHook` cannot be used with `importNpmLock`. You will instead need to use `importNpmLock.npmConfigHook`: + +```nix +{ buildNpmPackage, importNpmLock }: + +buildNpmPackage { + pname = "hello"; + version = "0.1.0"; + + npmDeps = importNpmLock { + npmRoot = ./.; + }; + + npmConfigHook = importNpmLock.npmConfigHook; +} +``` + ### corepack {#javascript-corepack} This package puts the corepack wrappers for pnpm and yarn in your PATH, and they will honor the `packageManager` setting in the `package.json`. diff --git a/pkgs/build-support/node/build-npm-package/default.nix b/pkgs/build-support/node/build-npm-package/default.nix index 42c6a9c065b2e..1c7bf63e8cd6a 100644 --- a/pkgs/build-support/node/build-npm-package/default.nix +++ b/pkgs/build-support/node/build-npm-package/default.nix @@ -49,6 +49,12 @@ name = "${name}-npm-deps"; hash = npmDepsHash; } + # Custom npmConfigHook +, npmConfigHook ? null + # Custom npmBuildHook +, npmBuildHook ? null + # Custom npmInstallHook +, npmInstallHook ? null , ... } @ args: @@ -57,14 +63,19 @@ let npmHooks = buildPackages.npmHooks.override { inherit nodejs; }; - - inherit (npmHooks) npmConfigHook npmBuildHook npmInstallHook; in stdenv.mkDerivation (args // { inherit npmDeps npmBuildScript; nativeBuildInputs = nativeBuildInputs - ++ [ nodejs npmConfigHook npmBuildHook npmInstallHook nodejs.python ] + ++ [ + nodejs + # Prefer passed hooks + (if npmConfigHook != null then npmConfigHook else npmHooks.npmConfigHook) + (if npmBuildHook != null then npmBuildHook else npmHooks.npmBuildHook) + (if npmInstallHook != null then npmInstallHook else npmHooks.npmInstallHook) + nodejs.python + ] ++ lib.optionals stdenv.isDarwin [ darwin.cctools ]; buildInputs = buildInputs ++ [ nodejs ]; diff --git a/pkgs/build-support/node/import-npm-lock/default.nix b/pkgs/build-support/node/import-npm-lock/default.nix new file mode 100644 index 0000000000000..d530b8ee30ffc --- /dev/null +++ b/pkgs/build-support/node/import-npm-lock/default.nix @@ -0,0 +1,134 @@ +{ lib +, fetchurl +, stdenv +, callPackages +, runCommand +}: + +let + inherit (builtins) match elemAt toJSON removeAttrs; + inherit (lib) importJSON mapAttrs; + + matchGitHubReference = match "github(.com)?:.+"; + getName = package: package.name or "unknown"; + getVersion = package: package.version or "0.0.0"; + + # Fetch a module from package-lock.json -> packages + fetchModule = + { module + , npmRoot ? null + }: ( + if module ? "resolved" then + ( + let + # Parse scheme from URL + mUrl = match "(.+)://(.+)" module.resolved; + scheme = elemAt mUrl 0; + in + ( + if mUrl == null then + ( + assert npmRoot != null; { + outPath = npmRoot + "/${module.resolved}"; + } + ) + else if (scheme == "http" || scheme == "https") then + ( + fetchurl { + url = module.resolved; + hash = module.integrity; + } + ) + else if lib.hasPrefix "git" module.resolved then + ( + builtins.fetchGit { + url = module.resolved; + } + ) + else throw "Unsupported URL scheme: ${scheme}" + ) + ) + else null + ); + + # Manage node_modules outside of the store with hooks + hooks = callPackages ./hooks { }; + +in +{ + importNpmLock = + { npmRoot ? null + , package ? importJSON (npmRoot + "/package.json") + , packageLock ? importJSON (npmRoot + "/package-lock.json") + , pname ? getName package + , version ? getVersion package + }: + let + mapLockDependencies = + mapAttrs + (name: version: ( + # Substitute the constraint with the version of the dependency from the top-level of package-lock. + if ( + # if the version is `latest` + version == "latest" + || + # Or if it's a github reference + matchGitHubReference version != null + ) then packageLock'.packages.${"node_modules/${name}"}.version + # But not a regular version constraint + else version + )); + + packageLock' = packageLock // { + packages = + mapAttrs + (_: module: + let + src = fetchModule { + inherit module npmRoot; + }; + in + (removeAttrs module [ + "link" + "funding" + ]) // lib.optionalAttrs (src != null) { + resolved = "file:${src}"; + } // lib.optionalAttrs (module ? dependencies) { + dependencies = mapLockDependencies module.dependencies; + } // lib.optionalAttrs (module ? optionalDependencies) { + optionalDependencies = mapLockDependencies module.optionalDependencies; + }) + packageLock.packages; + }; + + mapPackageDependencies = mapAttrs (name: _: packageLock'.packages.${"node_modules/${name}"}.resolved); + + # Substitute dependency references in package.json with Nix store paths + packageJSON' = package // lib.optionalAttrs (package ? dependencies) { + dependencies = mapPackageDependencies package.dependencies; + } // lib.optionalAttrs (package ? devDependencies) { + devDependencies = mapPackageDependencies package.devDependencies; + }; + + pname = package.name or "unknown"; + + in + runCommand "${pname}-${version}-sources" + { + inherit pname version; + + passAsFile = [ "package" "packageLock" ]; + + package = toJSON packageJSON'; + packageLock = toJSON packageLock'; + } '' + mkdir $out + cp "$packagePath" $out/package.json + cp "$packageLockPath" $out/package-lock.json + ''; + + inherit hooks; + inherit (hooks) npmConfigHook; + + __functor = self: self.importNpmLock; +} diff --git a/pkgs/build-support/node/import-npm-lock/hooks/canonicalize-symlinks.js b/pkgs/build-support/node/import-npm-lock/hooks/canonicalize-symlinks.js new file mode 100644 index 0000000000000..81cd2593c5b21 --- /dev/null +++ b/pkgs/build-support/node/import-npm-lock/hooks/canonicalize-symlinks.js @@ -0,0 +1,52 @@ +#!/usr/bin/env node +const fs = require("fs"); +const path = require("path"); + +// When installing files rewritten to the Nix store with npm +// npm writes the symlinks relative to the build directory. +// +// This makes relocating node_modules tricky when refering to the store. +// This script walks node_modules and canonicalizes symlinks. + +async function canonicalize(storePrefix, root) { + console.log(storePrefix, root) + const entries = await fs.promises.readdir(root); + const paths = entries.map((entry) => path.join(root, entry)); + + const stats = await Promise.all( + paths.map(async (path) => { + return { + path: path, + stat: await fs.promises.lstat(path), + }; + }) + ); + + const symlinks = stats.filter((stat) => stat.stat.isSymbolicLink()); + const dirs = stats.filter((stat) => stat.stat.isDirectory()); + + // Canonicalize symlinks to their real path + await Promise.all( + symlinks.map(async (stat) => { + const target = await fs.promises.realpath(stat.path); + if (target.startsWith(storePrefix)) { + await fs.promises.unlink(stat.path); + await fs.promises.symlink(target, stat.path); + } + }) + ); + + // Recurse into directories + await Promise.all(dirs.map((dir) => canonicalize(storePrefix, dir.path))); +} + +async function main() { + const args = process.argv.slice(2); + const storePrefix = args[0]; + + if (fs.existsSync("node_modules")) { + await canonicalize(storePrefix, "node_modules"); + } +} + +main(); diff --git a/pkgs/build-support/node/import-npm-lock/hooks/default.nix b/pkgs/build-support/node/import-npm-lock/hooks/default.nix new file mode 100644 index 0000000000000..5990371def912 --- /dev/null +++ b/pkgs/build-support/node/import-npm-lock/hooks/default.nix @@ -0,0 +1,13 @@ +{ callPackage, lib, makeSetupHook, srcOnly, nodejs }: +{ + npmConfigHook = makeSetupHook + { + name = "npm-config-hook"; + substitutions = { + nodeSrc = srcOnly nodejs; + nodeGyp = "${nodejs}/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js"; + canonicalizeSymlinksScript = ./canonicalize-symlinks.js; + storePrefix = builtins.storeDir; + }; + } ./npm-config-hook.sh; +} diff --git a/pkgs/build-support/node/import-npm-lock/hooks/npm-config-hook.sh b/pkgs/build-support/node/import-npm-lock/hooks/npm-config-hook.sh new file mode 100644 index 0000000000000..35c3a2061d4b1 --- /dev/null +++ b/pkgs/build-support/node/import-npm-lock/hooks/npm-config-hook.sh @@ -0,0 +1,70 @@ +# shellcheck shell=bash + +npmConfigHook() { + echo "Executing npmConfigHook" + + if [ -n "${npmRoot-}" ]; then + pushd "$npmRoot" + fi + + if [ -z "${npmDeps-}" ]; then + echo "Error: 'npmDeps' should be set when using npmConfigHook." + exit 1 + fi + + echo "Configuring npm" + + export HOME="$TMPDIR" + export npm_config_nodedir="@nodeSrc@" + export npm_config_node_gyp="@nodeGyp@" + npm config set offline true + npm config set progress false + npm config set fund false + + echo "Installing patched package.json/package-lock.json" + + # Save original package.json/package-lock.json for closure size reductions. + # The patched one contains store paths we don't want at runtime. + mv package.json .package.json.orig + if test -f package-lock.json; then # Not all packages have package-lock.json. + mv package-lock.json .package-lock.json.orig + fi + cp --no-preserve=mode "${npmDeps}/package.json" package.json + cp --no-preserve=mode "${npmDeps}/package-lock.json" package-lock.json + + echo "Installing dependencies" + + if ! npm install --ignore-scripts $npmInstallFlags "${npmInstallFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then + echo + echo "ERROR: npm failed to install dependencies" + echo + echo "Here are a few things you can try, depending on the error:" + echo '1. Set `npmFlags = [ "--legacy-peer-deps" ]`' + echo + + exit 1 + fi + + patchShebangs node_modules + + npm rebuild $npmRebuildFlags "${npmRebuildFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}" + + patchShebangs node_modules + + # Canonicalize symlinks from relative paths to the Nix store. + node @canonicalizeSymlinksScript@ @storePrefix@ + + # Revert to pre-patched package.json/package-lock.json for closure size reductions + mv .package.json.orig package.json + if test -f ".package-lock.json.orig"; then + mv .package-lock.json.orig package-lock.json + fi + + if [ -n "${npmRoot-}" ]; then + popd + fi + + echo "Finished npmConfigHook" +} + +postConfigureHooks+=(npmConfigHook) diff --git a/pkgs/development/python-modules/acquire/default.nix b/pkgs/development/python-modules/acquire/default.nix index c2c4e125e3287..555a60412bf45 100644 --- a/pkgs/development/python-modules/acquire/default.nix +++ b/pkgs/development/python-modules/acquire/default.nix @@ -17,7 +17,7 @@ buildPythonPackage rec { pname = "acquire"; - version = "3.12"; + version = "3.13"; pyproject = true; disabled = pythonOlder "3.9"; @@ -26,7 +26,7 @@ buildPythonPackage rec { owner = "fox-it"; repo = "acquire"; rev = "refs/tags/${version}"; - hash = "sha256-anRNCnLDKHAEfOWi6m1n4R9lvFTlZgw5xxh39exvzH0="; + hash = "sha256-Z85bHM3MtS2MLX9BaKi8VqA13QjO9KdrgqhuyBzjILQ="; }; nativeBuildInputs = [ diff --git a/pkgs/development/python-modules/ago/default.nix b/pkgs/development/python-modules/ago/default.nix new file mode 100644 index 0000000000000..ad095c441db26 --- /dev/null +++ b/pkgs/development/python-modules/ago/default.nix @@ -0,0 +1,40 @@ +{ lib +, buildPythonPackage +, fetchPypi + +, pythonOlder + +, pytestCheckHook + +, setuptools +}: + +buildPythonPackage rec { + pname = "ago"; + version = "0.0.95"; + pyproject = true; + + disabled = pythonOlder "3.3"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-0gEPXqw99UTsSOwRYQLgaFkaNFsaWA8ylz24pQX8p0Q="; + }; + + build-system = [ + setuptools + ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + pythonImportsCheck = [ "ago" ]; + + meta = with lib; { + description = "Human Readable Time Deltas"; + homepage = "https://git.unturf.com/python/ago"; + license = licenses.publicDomain; + maintainers = with maintainers; [ vizid ]; + }; +} diff --git a/pkgs/development/python-modules/autopep8/default.nix b/pkgs/development/python-modules/autopep8/default.nix index 4bfffd8a780b3..34dd0baa4c99d 100644 --- a/pkgs/development/python-modules/autopep8/default.nix +++ b/pkgs/development/python-modules/autopep8/default.nix @@ -1,27 +1,33 @@ { lib , buildPythonPackage , fetchFromGitHub +, fetchpatch , glibcLocales , pycodestyle , pytestCheckHook , pythonOlder +, setuptools , tomli }: buildPythonPackage rec { pname = "autopep8"; - version = "2.0.4"; - format = "setuptools"; + version = "2.0.4-unstable-2023-10-27"; + pyproject = true; disabled = pythonOlder "3.7"; src = fetchFromGitHub { owner = "hhatto"; repo = "autopep8"; - rev = "refs/tags/v${version}"; - hash = "sha256-TuaDnZqn9mRUsoDJkj9JK4ztvzl9JTwAk8nghIkZBvw="; + rev = "af7399d90926f2fe99a71f15197a08fa197f73a1"; + hash = "sha256-psGl9rXxTQGHyXf1VskJ/I/goVH5hRRP5bUXQdaT/8M="; }; + build-system = [ + setuptools + ]; + propagatedBuildInputs = [ pycodestyle ] ++ lib.optionals (pythonOlder "3.11") [ diff --git a/pkgs/development/python-modules/cohere/default.nix b/pkgs/development/python-modules/cohere/default.nix index 86b7e7b1b739a..323efb118503f 100644 --- a/pkgs/development/python-modules/cohere/default.nix +++ b/pkgs/development/python-modules/cohere/default.nix @@ -14,14 +14,14 @@ buildPythonPackage rec { pname = "cohere"; - version = "4.51"; + version = "4.52"; pyproject = true; disabled = pythonOlder "3.7"; src = fetchPypi { inherit pname version; - hash = "sha256-AfsJLqkDjdT7Ng77NQb60kUe0jHLZ3TjJLmTyTdKVQo="; + hash = "sha256-C2w447+hxsFrjLpV3wrSzptuVb1JTyppq+NUzMCU+Iw="; }; nativeBuildInputs = [ diff --git a/pkgs/development/python-modules/ezyrb/default.nix b/pkgs/development/python-modules/ezyrb/default.nix index 024509dad06b8..44d49ee16edd8 100644 --- a/pkgs/development/python-modules/ezyrb/default.nix +++ b/pkgs/development/python-modules/ezyrb/default.nix @@ -14,7 +14,7 @@ buildPythonPackage rec { pname = "ezyrb"; - version = "1.3.0.post2402"; + version = "1.3.0.post2403"; format = "setuptools"; disabled = pythonOlder "3.7"; @@ -23,7 +23,7 @@ buildPythonPackage rec { owner = "mathLab"; repo = "EZyRB"; rev = "refs/tags/v${version}"; - hash = "sha256-MiFNTz3vrN4rMHK7e4ntE35wzgnPt6yczCv7XDcUlO8="; + hash = "sha256-t0Mv8Kae6N+jHeQx57ljDR5lmmbW2mqrlqygtrwGWhY="; }; propagatedBuildInputs = [ diff --git a/pkgs/development/python-modules/hurry-filesize/default.nix b/pkgs/development/python-modules/hurry-filesize/default.nix new file mode 100644 index 0000000000000..62bd1f810f9d5 --- /dev/null +++ b/pkgs/development/python-modules/hurry-filesize/default.nix @@ -0,0 +1,39 @@ +{ lib +, buildPythonPackage +, fetchPypi + +, pythonOlder + +, setuptools +}: + +buildPythonPackage rec { + pname = "hurry-filesize"; + version = "0.9"; + pyproject = true; + + disabled = pythonOlder "3.3"; + + src = fetchPypi { + pname = "hurry.filesize"; + inherit version; + hash = "sha256-9TaDKa2++GrM07yUkFIjQLt5JgRVromxpCwQ9jgBuaY="; + }; + + # project has no repo... + # fix implicit namespaces (PEP 420) warning + patches = [ ./use-pep-420-implicit-namespace-package.patch ]; + + build-system = [ + setuptools + ]; + + pythonImportsCheck = [ "hurry.filesize" ]; + + meta = with lib; { + description = "A simple Python library for human readable file sizes (or anything sized in bytes)"; + homepage = "https://pypi.org/project/hurry.filesize/"; + license = licenses.zpl21; + maintainers = with maintainers; [ vizid ]; + }; +} diff --git a/pkgs/development/python-modules/hurry-filesize/use-pep-420-implicit-namespace-package.patch b/pkgs/development/python-modules/hurry-filesize/use-pep-420-implicit-namespace-package.patch new file mode 100644 index 0000000000000..788c603ac5303 --- /dev/null +++ b/pkgs/development/python-modules/hurry-filesize/use-pep-420-implicit-namespace-package.patch @@ -0,0 +1,24 @@ +diff --git a/setup.py b/setup.py +index 9ec6f2e..607b680 100644 +--- a/setup.py ++++ b/setup.py +@@ -29,7 +29,6 @@ setup( + license='ZPL 2.1', + packages=find_packages('src'), + package_dir= {'':'src'}, +- namespace_packages=['hurry'], + include_package_data=True, + zip_safe=False, + install_requires=[ +diff --git a/src/hurry/__init__.py b/src/hurry/__init__.py +index 2e2033b..e69de29 100644 +--- a/src/hurry/__init__.py ++++ b/src/hurry/__init__.py +@@ -1,7 +0,0 @@ +-# this is a namespace package +-try: +- import pkg_resources +- pkg_resources.declare_namespace(__name__) +-except ImportError: +- import pkgutil +- __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/pkgs/development/python-modules/langchain-core/default.nix b/pkgs/development/python-modules/langchain-core/default.nix index 76dcc2cd66b1b..6e7b52a16b702 100644 --- a/pkgs/development/python-modules/langchain-core/default.nix +++ b/pkgs/development/python-modules/langchain-core/default.nix @@ -16,7 +16,7 @@ buildPythonPackage rec { pname = "langchain-core"; - version = "0.1.29"; + version = "0.1.30"; pyproject = true; disabled = pythonOlder "3.8"; @@ -24,7 +24,7 @@ buildPythonPackage rec { src = fetchPypi { pname = "langchain_core"; inherit version; - hash = "sha256-ZzHav/rQO5ITraJkDVTtf072uZ/Oh63jxxR0rhVN08w="; + hash = "sha256-4ToBblXn8IL/Pu7aLQy1BbiaiDDjojwdE00KideHGJQ="; }; pythonRelaxDeps = [ diff --git a/pkgs/development/python-modules/python-matter-server/default.nix b/pkgs/development/python-modules/python-matter-server/default.nix index 57761e5dcac1e..a451aed79092c 100644 --- a/pkgs/development/python-modules/python-matter-server/default.nix +++ b/pkgs/development/python-modules/python-matter-server/default.nix @@ -55,7 +55,7 @@ in buildPythonPackage rec { pname = "python-matter-server"; - version = "5.8.0"; + version = "5.8.1"; format = "pyproject"; disabled = pythonOlder "3.10"; @@ -64,7 +64,7 @@ buildPythonPackage rec { owner = "home-assistant-libs"; repo = "python-matter-server"; rev = "refs/tags/${version}"; - hash = "sha256-bpXRay4JUujqdnscGldW732e8FTkcmfShbtwp2YJC60="; + hash = "sha256-iisDEopaKklLvvGDTo2fv0Fpkhuhd+7KoNnQA+QmjB8="; }; patches = [ diff --git a/pkgs/development/python-modules/streamlit/default.nix b/pkgs/development/python-modules/streamlit/default.nix index 3e1027913bd3f..a0f6111c6eab8 100644 --- a/pkgs/development/python-modules/streamlit/default.nix +++ b/pkgs/development/python-modules/streamlit/default.nix @@ -32,14 +32,14 @@ buildPythonPackage rec { pname = "streamlit"; - version = "1.31.1"; + version = "1.32.0"; pyproject = true; disabled = pythonOlder "3.8"; src = fetchPypi { inherit pname version; - hash = "sha256-38Q8qFtLTDHQl8J7mDuMzJYCIq2QeGKysvtN3wTFD9w="; + hash = "sha256-Zb8i4ZDelzuRDAsSezPYDTHFJ17Ykcb950DeRuXgUyM="; }; nativeBuildInputs = [ diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index bbe62d82f23e6..97a4da725518d 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -10262,6 +10262,8 @@ with pkgs; inherit (callPackages ../build-support/node/fetch-npm-deps { }) fetchNpmDeps prefetch-npm-deps; + importNpmLock = callPackages ../build-support/node/import-npm-lock { }; + nodePackages_latest = dontRecurseIntoAttrs nodejs_latest.pkgs // { __attrsFailEvaluation = true; }; nodePackages = dontRecurseIntoAttrs nodejs.pkgs // { __attrsFailEvaluation = true; }; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index b73c25df3b31c..7e9292ae50101 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -123,6 +123,8 @@ self: super: with self; { agent-py = callPackage ../development/python-modules/agent-py { }; + ago = callPackage ../development/python-modules/ago { }; + aggdraw = callPackage ../development/python-modules/aggdraw { }; aigpy = callPackage ../development/python-modules/aigpy { }; @@ -5418,6 +5420,8 @@ self: super: with self; { hupper = callPackage ../development/python-modules/hupper { }; + hurry-filesize = callPackage ../development/python-modules/hurry-filesize { }; + huum = callPackage ../development/python-modules/huum { }; hvac = callPackage ../development/python-modules/hvac { }; |