diff options
Diffstat (limited to 'pkgs/build-support')
163 files changed, 3420 insertions, 2517 deletions
diff --git a/pkgs/build-support/add-driver-runpath/default.nix b/pkgs/build-support/add-driver-runpath/default.nix new file mode 100644 index 0000000000000..08547a4453c5f --- /dev/null +++ b/pkgs/build-support/add-driver-runpath/default.nix @@ -0,0 +1,14 @@ +{ lib, stdenv }: + +stdenv.mkDerivation { + name = "add-driver-runpath"; + + # Named "opengl-driver" for legacy reasons, but it is the path to + # hardware drivers installed by NixOS + driverLink = "/run/opengl-driver" + lib.optionalString stdenv.isi686 "-32"; + + buildCommand = '' + mkdir -p $out/nix-support + substituteAll ${./setup-hook.sh} $out/nix-support/setup-hook + ''; +} diff --git a/pkgs/build-support/add-driver-runpath/setup-hook.sh b/pkgs/build-support/add-driver-runpath/setup-hook.sh new file mode 100644 index 0000000000000..86a41a7e25192 --- /dev/null +++ b/pkgs/build-support/add-driver-runpath/setup-hook.sh @@ -0,0 +1,29 @@ +# Set RUNPATH so that driver libraries in /run/opengl-driver(-32)/lib can be found. +# This is needed to not rely on LD_LIBRARY_PATH which does not work with setuid +# executables. Fixes https://github.com/NixOS/nixpkgs/issues/22760. It must be run +# in postFixup because RUNPATH stripping in fixup would undo it. Note that patchelf +# actually sets RUNPATH not RPATH, which applies only to dependencies of the binary +# it set on (including for dlopen), so the RUNPATH must indeed be set on these +# libraries and would not work if set only on executables. +addDriverRunpath() { + local forceRpath= + + while [ $# -gt 0 ]; do + case "$1" in + --) shift; break;; + --force-rpath) shift; forceRpath=1;; + --*) + echo "addDriverRunpath: ERROR: Invalid command line" \ + "argument: $1" >&2 + return 1;; + *) break;; + esac + done + + for file in "$@"; do + if ! isELF "$file"; then continue; fi + local origRpath="$(patchelf --print-rpath "$file")" + patchelf --set-rpath "@driverLink@/lib:$origRpath" ${forceRpath:+--force-rpath} "$file" + done +} + diff --git a/pkgs/build-support/agda/default.nix b/pkgs/build-support/agda/default.nix index 63adf2a276516..893383a759aec 100644 --- a/pkgs/build-support/agda/default.nix +++ b/pkgs/build-support/agda/default.nix @@ -47,6 +47,7 @@ let "lagda.org" "lagda.rst" "lagda.tex" + "lagda.typ" ]; defaults = diff --git a/pkgs/build-support/agda/lib.nix b/pkgs/build-support/agda/lib.nix index 80a0974192bb3..e6e391b2ceddf 100644 --- a/pkgs/build-support/agda/lib.nix +++ b/pkgs/build-support/agda/lib.nix @@ -6,7 +6,7 @@ * interfaceFile "Everything.agda" == "Everything.agdai" * interfaceFile "src/Everything.lagda.tex" == "src/Everything.agdai" */ - interfaceFile = agdaFile: lib.head (builtins.match ''(.*\.)l?agda(\.(md|org|rst|tex))?'' agdaFile) + "agdai"; + interfaceFile = agdaFile: lib.head (builtins.match ''(.*\.)l?agda(\.(md|org|rst|tex|typ))?'' agdaFile) + "agdai"; /* Takes an arbitrary derivation and says whether it is an agda library package * that is not marked as broken. diff --git a/pkgs/build-support/alternatives/blas/default.nix b/pkgs/build-support/alternatives/blas/default.nix index fec2d0526bb32..91001bc85377a 100644 --- a/pkgs/build-support/alternatives/blas/default.nix +++ b/pkgs/build-support/alternatives/blas/default.nix @@ -80,7 +80,7 @@ stdenv.mkDerivation { cp -L "$libblas" $out/lib/libblas${canonicalExtension} chmod +w $out/lib/libblas${canonicalExtension} -'' + (if stdenv.hostPlatform.parsed.kernel.execFormat.name == "elf" then '' +'' + (if stdenv.hostPlatform.isElf then '' patchelf --set-soname libblas${canonicalExtension} $out/lib/libblas${canonicalExtension} patchelf --set-rpath "$(patchelf --print-rpath $out/lib/libblas${canonicalExtension}):${lib.getLib blasProvider'}/lib" $out/lib/libblas${canonicalExtension} '' else lib.optionalString (stdenv.hostPlatform.isDarwin) '' @@ -112,7 +112,7 @@ EOF cp -L "$libcblas" $out/lib/libcblas${canonicalExtension} chmod +w $out/lib/libcblas${canonicalExtension} -'' + (if stdenv.hostPlatform.parsed.kernel.execFormat.name == "elf" then '' +'' + (if stdenv.hostPlatform.isElf then '' patchelf --set-soname libcblas${canonicalExtension} $out/lib/libcblas${canonicalExtension} patchelf --set-rpath "$(patchelf --print-rpath $out/lib/libcblas${canonicalExtension}):${lib.getLib blasProvider'}/lib" $out/lib/libcblas${canonicalExtension} '' else lib.optionalString stdenv.hostPlatform.isDarwin '' diff --git a/pkgs/build-support/alternatives/lapack/default.nix b/pkgs/build-support/alternatives/lapack/default.nix index cbc7bf25c7978..2d62855b258a4 100644 --- a/pkgs/build-support/alternatives/lapack/default.nix +++ b/pkgs/build-support/alternatives/lapack/default.nix @@ -57,7 +57,7 @@ stdenv.mkDerivation { cp -L "$liblapack" $out/lib/liblapack${canonicalExtension} chmod +w $out/lib/liblapack${canonicalExtension} -'' + (lib.optionalString (stdenv.hostPlatform.parsed.kernel.execFormat.name == "elf") '' +'' + (lib.optionalString stdenv.hostPlatform.isElf '' patchelf --set-soname liblapack${canonicalExtension} $out/lib/liblapack${canonicalExtension} patchelf --set-rpath "$(patchelf --print-rpath $out/lib/liblapack${canonicalExtension}):${lapackProvider'}/lib" $out/lib/liblapack${canonicalExtension} '') + '' @@ -86,7 +86,7 @@ EOF cp -L "$liblapacke" $out/lib/liblapacke${canonicalExtension} chmod +w $out/lib/liblapacke${canonicalExtension} -'' + (lib.optionalString (stdenv.hostPlatform.parsed.kernel.execFormat.name == "elf") '' +'' + (lib.optionalString stdenv.hostPlatform.isElf '' patchelf --set-soname liblapacke${canonicalExtension} $out/lib/liblapacke${canonicalExtension} patchelf --set-rpath "$(patchelf --print-rpath $out/lib/liblapacke${canonicalExtension}):${lib.getLib lapackProvider'}/lib" $out/lib/liblapacke${canonicalExtension} '') + '' diff --git a/pkgs/build-support/appimage/default.nix b/pkgs/build-support/appimage/default.nix index 0637964ca62fb..078570ec7aea3 100644 --- a/pkgs/build-support/appimage/default.nix +++ b/pkgs/build-support/appimage/default.nix @@ -26,10 +26,11 @@ rec { ]; }; - extract = args@{ name ? "${args.pname}-${args.version}", src, ... }: pkgs.runCommand "${name}-extracted" { + extract = args@{ name ? "${args.pname}-${args.version}", postExtract ? "", src, ... }: pkgs.runCommand "${name}-extracted" { buildInputs = [ appimage-exec ]; } '' appimage-exec.sh -x $out ${src} + ${postExtract} ''; # for compatibility, deprecated @@ -172,6 +173,7 @@ rec { libpulseaudio libsamplerate libmikmod + libthai libtheora libtiff pixman @@ -206,6 +208,7 @@ rec { libtool.lib # for Synfigstudio xorg.libxshmfence # for apple-music-electron at-spi2-core + pciutils # for FreeCAD ]; }; } diff --git a/pkgs/build-support/binary-cache/default.nix b/pkgs/build-support/binary-cache/default.nix index 27f9ad9628998..8c610c512276e 100644 --- a/pkgs/build-support/binary-cache/default.nix +++ b/pkgs/build-support/binary-cache/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, buildPackages }: +{ lib, stdenv, coreutils, jq, python3, nix, xz }: # This function is for creating a flat-file binary cache, i.e. the kind created by # nix copy --to file:///some/path and usable as a substituter (with the file:// prefix). @@ -19,15 +19,10 @@ stdenv.mkDerivation { preferLocalBuild = true; - PATH = lib.makeBinPath (with buildPackages; [ coreutils jq python3 nix xz ]); + nativeBuildInputs = [ coreutils jq python3 nix xz ]; - builder = builtins.toFile "builder" '' - . .attrs.sh - - export out=''${outputs[out]} - - mkdir $out - mkdir $out/nar + buildCommand = '' + mkdir -p $out/nar python ${./make-binary-cache.py} diff --git a/pkgs/build-support/binary-cache/make-binary-cache.py b/pkgs/build-support/binary-cache/make-binary-cache.py index 16dd8a7e96bcf..589d005562b1f 100644 --- a/pkgs/build-support/binary-cache/make-binary-cache.py +++ b/pkgs/build-support/binary-cache/make-binary-cache.py @@ -3,7 +3,7 @@ import json import os import subprocess -with open(".attrs.json", "r") as f: +with open(os.environ["NIX_ATTRS_JSON_FILE"], "r") as f: closures = json.load(f)["closure"] os.chdir(os.environ["out"]) diff --git a/pkgs/build-support/bintools-wrapper/default.nix b/pkgs/build-support/bintools-wrapper/default.nix index ccd342eaa0d19..66027485ee714 100644 --- a/pkgs/build-support/bintools-wrapper/default.nix +++ b/pkgs/build-support/bintools-wrapper/default.nix @@ -33,6 +33,28 @@ , useMacosReexportHack ? false , wrapGas ? false +# Note: the hardening flags are part of the bintools-wrapper, rather than +# the cc-wrapper, because a few of them are handled by the linker. +, defaultHardeningFlags ? with stdenvNoCC; [ + "bindnow" + "format" + "fortify" + "fortify3" + "pic" + "relro" + "stackprotector" + "strictoverflow" + ] ++ lib.optional ( + # Musl-based platforms will keep "pie", other platforms will not. + # If you change this, make sure to update section `{#sec-hardening-in-nixpkgs}` + # in the nixpkgs manual to inform users about the defaults. + targetPlatform.libc == "musl" + # Except when: + # - static aarch64, where compilation works, but produces segfaulting dynamically linked binaries. + # - static armv7l, where compilation fails. + && !(targetPlatform.isAarch && targetPlatform.isStatic) + ) "pie" + # Darwin code signing support utilities , postLinkSignHook ? null, signingUtils ? null }: @@ -124,6 +146,8 @@ stdenv.mkDerivation { (setenv "NIX_LDFLAGS_${suffixSalt}" (concat (getenv "NIX_LDFLAGS_${suffixSalt}") " -L" arg "/lib64")))) '(${concatStringsSep " " (map (pkg: "\"${pkg}\"") pkgs)})) ''; + + inherit defaultHardeningFlags; }; dontBuild = true; @@ -297,16 +321,10 @@ stdenv.mkDerivation { hardening_unsupported_flags+=" pic" '' - + optionalString targetPlatform.isAvr '' + + optionalString (targetPlatform.isAvr || targetPlatform.isWindows) '' hardening_unsupported_flags+=" relro bindnow" '' - + optionalString (libc != null && targetPlatform.isAvr) '' - for isa in avr5 avr3 avr4 avr6 avr25 avr31 avr35 avr51 avrxmega2 avrxmega4 avrxmega5 avrxmega6 avrxmega7 tiny-stack; do - echo "-L${getLib libc}/avr/lib/$isa" >> $out/nix-support/libc-cflags - done - '' - + optionalString stdenv.targetPlatform.isDarwin '' echo "-arch ${targetPlatform.darwinArch}" >> $out/nix-support/libc-ldflags '' @@ -322,10 +340,10 @@ stdenv.mkDerivation { '' ### - ### Remove LC_UUID + ### Remove certain timestamps from final binaries ### + optionalString (stdenv.targetPlatform.isDarwin && !(bintools.isGNU or false)) '' - echo "-no_uuid" >> $out/nix-support/libc-ldflags-before + echo "export ZERO_AR_DATE=1" >> $out/nix-support/setup-hook '' + '' @@ -386,6 +404,7 @@ stdenv.mkDerivation { wrapperName = "BINTOOLS_WRAPPER"; inherit dynamicLinker targetPrefix suffixSalt coreutils_bin; inherit bintools_bin libc_bin libc_dev libc_lib; + default_hardening_flags_str = builtins.toString defaultHardeningFlags; }; meta = diff --git a/pkgs/build-support/bintools-wrapper/setup-hook.sh b/pkgs/build-support/bintools-wrapper/setup-hook.sh index 7e9547b96c259..c146cbea80e4f 100644 --- a/pkgs/build-support/bintools-wrapper/setup-hook.sh +++ b/pkgs/build-support/bintools-wrapper/setup-hook.sh @@ -65,7 +65,7 @@ do done # If unset, assume the default hardening flags. -: ${NIX_HARDENING_ENABLE="fortify stackprotector pic strictoverflow format relro bindnow"} +: ${NIX_HARDENING_ENABLE="@default_hardening_flags_str@"} export NIX_HARDENING_ENABLE # No local scope in sourced file diff --git a/pkgs/build-support/build-fhsenv-bubblewrap/buildFHSEnv.nix b/pkgs/build-support/build-fhsenv-bubblewrap/buildFHSEnv.nix index 2c0c4a3d513a9..b5e03164ac269 100644 --- a/pkgs/build-support/build-fhsenv-bubblewrap/buildFHSEnv.nix +++ b/pkgs/build-support/build-fhsenv-bubblewrap/buildFHSEnv.nix @@ -52,7 +52,7 @@ let # these match the host's architecture, glibc_multi is used for multilib # builds. glibcLocales must be before glibc or glibc_multi as otherwiese # the wrong LOCALE_ARCHIVE will be used where only C.UTF-8 is available. - basePkgs = with pkgs; [ + baseTargetPaths = with pkgs; [ glibcLocales (if isMultiBuild then glibc_multi else glibc) (toString gcc.cc.lib) @@ -71,7 +71,7 @@ let bzip2 xz ]; - baseMultiPkgs = with pkgsi686Linux; [ + baseMultiPaths = with pkgsi686Linux; [ (toString gcc.cc.lib) ]; @@ -83,7 +83,7 @@ let etcProfile = writeText "profile" '' export PS1='${name}-chrootenv:\u@\h:\w\$ ' export LOCALE_ARCHIVE='/usr/lib/locale/locale-archive' - export LD_LIBRARY_PATH="/run/opengl-driver/lib:/run/opengl-driver-32/lib:/usr/lib:/usr/lib32''${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH" + export LD_LIBRARY_PATH="/run/opengl-driver/lib:/run/opengl-driver-32/lib''${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH" export PATH="/run/wrappers/bin:/usr/bin:/usr/sbin:$PATH" export TZDIR='/etc/zoneinfo' @@ -132,7 +132,7 @@ let staticUsrProfileTarget = buildEnv { name = "${name}-usr-target"; # ldconfig wrapper must come first so it overrides the original ldconfig - paths = [ etcPkg ldconfig ] ++ basePkgs ++ targetPaths; + paths = [ etcPkg ldconfig ] ++ baseTargetPaths ++ targetPaths; extraOutputsToInstall = [ "out" "lib" "bin" ] ++ extraOutputsToInstall; ignoreCollisions = true; postBuild = '' @@ -168,7 +168,7 @@ let staticUsrProfileMulti = buildEnv { name = "${name}-usr-multi"; - paths = baseMultiPkgs ++ multiPaths; + paths = baseMultiPaths ++ multiPaths; extraOutputsToInstall = [ "out" "lib" ] ++ extraOutputsToInstall; ignoreCollisions = true; }; @@ -251,7 +251,7 @@ let in runCommandLocal "${name}-fhs" { passthru = { - inherit args multiPaths targetPaths ldconfig; + inherit args baseTargetPaths targetPaths baseMultiPaths multiPaths ldconfig; }; } '' mkdir -p $out diff --git a/pkgs/build-support/build-fhsenv-bubblewrap/default.nix b/pkgs/build-support/build-fhsenv-bubblewrap/default.nix index 6c9b71624c2bb..3292f4039a637 100644 --- a/pkgs/build-support/build-fhsenv-bubblewrap/default.nix +++ b/pkgs/build-support/build-fhsenv-bubblewrap/default.nix @@ -16,13 +16,15 @@ , extraInstallCommands ? "" , meta ? {} , passthru ? {} +, extraPreBwrapCmds ? "" , extraBwrapArgs ? [] -, unshareUser ? true -, unshareIpc ? true -, unsharePid ? true +, unshareUser ? false +, unshareIpc ? false +, unsharePid ? false , unshareNet ? false -, unshareUts ? true -, unshareCgroup ? true +, unshareUts ? false +, unshareCgroup ? false +, privateTmp ? false , dieWithParent ? true , ... } @ args: @@ -31,15 +33,15 @@ assert (pname != null || version != null) -> (name == null && pname != null); # with builtins; let - pname = if args.name != null then args.name else args.pname; + pname = if args ? name && args.name != null then args.name else args.pname; versionStr = lib.optionalString (version != null) ("-" + version); name = pname + versionStr; buildFHSEnv = callPackage ./buildFHSEnv.nix { }; fhsenv = buildFHSEnv (removeAttrs (args // { inherit name; }) [ - "runScript" "extraInstallCommands" "meta" "passthru" "extraBwrapArgs" "dieWithParent" - "unshareUser" "unshareCgroup" "unshareUts" "unshareNet" "unsharePid" "unshareIpc" + "runScript" "extraInstallCommands" "meta" "passthru" "extraPreBwrapCmds" "extraBwrapArgs" "dieWithParent" + "unshareUser" "unshareCgroup" "unshareUts" "unshareNet" "unsharePid" "unshareIpc" "privateTmp" "pname" "version" ]); @@ -116,7 +118,8 @@ let indentLines = str: lib.concatLines (map (s: " " + s) (filter (s: s != "") (lib.splitString "\n" str))); bwrapCmd = { initArgs ? "" }: '' - ignored=(/nix /dev /proc /etc) + ${extraPreBwrapCmds} + ignored=(/nix /dev /proc /etc ${lib.optionalString privateTmp "/tmp"}) ro_mounts=() symlinks=() etc_ignored=() @@ -146,14 +149,19 @@ let done fi + # propagate /etc from the actual host if nested + if [[ -e /.host-etc ]]; then + ro_mounts+=(--ro-bind /.host-etc /.host-etc) + else + ro_mounts+=(--ro-bind /etc /.host-etc) + fi + for i in ${lib.escapeShellArgs etcBindEntries}; do if [[ "''${etc_ignored[@]}" =~ "$i" ]]; then continue fi - if [[ -L $i ]]; then - symlinks+=(--symlink "$(${coreutils}/bin/readlink "$i")" "$i") - else - ro_mounts+=(--ro-bind-try "$i" "$i") + if [[ -e $i ]]; then + symlinks+=(--symlink "/.host-etc/''${i#/etc/}" "$i") fi done @@ -179,6 +187,26 @@ let x11_args+=(--ro-bind-try "$local_socket" "$local_socket") fi + ${lib.optionalString privateTmp '' + # sddm places XAUTHORITY in /tmp + if [[ "$XAUTHORITY" == /tmp/* ]]; then + x11_args+=(--ro-bind-try "$XAUTHORITY" "$XAUTHORITY") + fi + + # dbus-run-session puts the socket in /tmp + IFS=";" read -ra addrs <<<"$DBUS_SESSION_BUS_ADDRESS" + for addr in "''${addrs[@]}"; do + [[ "$addr" == unix:* ]] || continue + IFS="," read -ra parts <<<"''${addr#unix:}" + for part in "''${parts[@]}"; do + printf -v part '%s' "''${part//\\/\\\\}" + printf -v part '%b' "''${part//%/\\x}" + [[ "$part" == path=/tmp/* ]] || continue + x11_args+=(--ro-bind-try "''${part#path=}" "''${part#path=}") + done + done + ''} + cmd=( ${bubblewrap}/bin/bwrap --dev-bind /dev /dev @@ -192,6 +220,7 @@ let ${lib.optionalString unshareCgroup "--unshare-cgroup"} ${lib.optionalString dieWithParent "--die-with-parent"} --ro-bind /nix /nix + ${lib.optionalString privateTmp "--tmpfs /tmp"} # Our glibc will look for the cache in its own path in `/nix/store`. # As such, we need a cache to exist there, because pressure-vessel # depends on the existence of an ld cache. However, adding one @@ -200,6 +229,7 @@ let # Also, the cache needs to go to both 32 and 64 bit glibcs, for games # of both architectures to work. --tmpfs ${glibc}/etc \ + --tmpfs /etc \ --symlink /etc/ld.so.conf ${glibc}/etc/ld.so.conf \ --symlink /etc/ld.so.cache ${glibc}/etc/ld.so.cache \ --ro-bind ${glibc}/etc/rpc ${glibc}/etc/rpc \ @@ -223,6 +253,7 @@ let bin = writeShellScript "${name}-bwrap" (bwrapCmd { initArgs = ''"$@"''; }); in runCommandLocal name { + inherit pname version; inherit meta; passthru = passthru // { diff --git a/pkgs/build-support/build-fhsenv-chroot/env.nix b/pkgs/build-support/build-fhsenv-chroot/env.nix index a1a26472373f5..6a82435d7067c 100644 --- a/pkgs/build-support/build-fhsenv-chroot/env.nix +++ b/pkgs/build-support/build-fhsenv-chroot/env.nix @@ -135,6 +135,7 @@ let # symlink ALSA stuff ln -s /host/etc/asound.conf asound.conf + ln -s /host/etc/alsa alsa # symlink SSL certs mkdir -p ssl diff --git a/pkgs/build-support/build-graalvm-native-image/default.nix b/pkgs/build-support/build-graalvm-native-image/default.nix index c3c4a7b1e57b2..f0e73901c8db0 100644 --- a/pkgs/build-support/build-graalvm-native-image/default.nix +++ b/pkgs/build-support/build-graalvm-native-image/default.nix @@ -3,7 +3,7 @@ , glibcLocales # The GraalVM derivation to use , graalvmDrv -, name ? "${args.pname}-${args.version}" +, removeReferencesTo , executable ? args.pname # JAR used as input for GraalVM derivation, defaults to src , jar ? args.src @@ -12,16 +12,17 @@ # except in special cases. In most cases, use extraNativeBuildArgs instead , nativeImageBuildArgs ? [ (lib.optionalString stdenv.isDarwin "-H:-CheckToolchain") + (lib.optionalString (stdenv.isLinux && stdenv.isAarch64) "-H:PageSize=64K") "-H:Name=${executable}" + "-march=compatibility" "--verbose" ] # Extra arguments to be passed to the native-image , extraNativeImageBuildArgs ? [ ] # XMX size of GraalVM during build , graalvmXmx ? "-J-Xmx6g" - # Locale to be used by GraalVM compiler -, LC_ALL ? "en_US.UTF-8" , meta ? { } +, LC_ALL ? "en_US.UTF-8" , ... } @ args: @@ -37,19 +38,22 @@ let "buildPhase" "nativeBuildInputs" "installPhase" + "postInstall" ]; in stdenv.mkDerivation ({ - inherit dontUnpack LC_ALL jar; + inherit dontUnpack jar; + + env = { inherit LC_ALL; }; - nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ graalvmDrv glibcLocales ]; + nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ graalvmDrv glibcLocales removeReferencesTo ]; nativeImageBuildArgs = nativeImageBuildArgs ++ extraNativeImageBuildArgs ++ [ graalvmXmx ]; buildPhase = args.buildPhase or '' runHook preBuild - native-image -jar "$jar" ''${nativeImageBuildArgs[@]} + native-image -jar "$jar" $(export -p | sed -n 's/^declare -x \([^=]\+\)=.*$/ -E\1/p' | tr -d \\n) ''${nativeImageBuildArgs[@]} runHook postBuild ''; @@ -62,6 +66,11 @@ stdenv.mkDerivation ({ runHook postInstall ''; + postInstall = '' + remove-references-to -t ${graalvmDrv} $out/bin/${executable} + ${args.postInstall or ""} + ''; + disallowedReferences = [ graalvmDrv ]; passthru = { inherit graalvmDrv; }; @@ -71,7 +80,5 @@ stdenv.mkDerivation ({ platforms = graalvmDrv.meta.platforms; # default to executable name mainProgram = executable; - # need to have native-image-installable-svm available - broken = !(builtins.any (p: (p.product or "") == "native-image-installable-svm") graalvmDrv.products); } // meta; } // extraArgs) diff --git a/pkgs/build-support/build-setupcfg/default.nix b/pkgs/build-support/build-setupcfg/default.nix deleted file mode 100644 index 5737989249af4..0000000000000 --- a/pkgs/build-support/build-setupcfg/default.nix +++ /dev/null @@ -1,26 +0,0 @@ -# Build a python package from info made available by setupcfg2nix. -# -# * src: The source of the package. -# * info: The package information generated by setupcfg2nix. -# * meta: Standard nixpkgs metadata. -# * application: Whether this package is a python library or an -# application which happens to be written in python. -# * doCheck: Whether to run the test suites. -lib: pythonPackages: -{ src, info, meta ? {}, application ? false, doCheck ? true}: let - build = if application - then pythonPackages.buildPythonApplication - else pythonPackages.buildPythonPackage; -in build { - inherit (info) pname version; - - inherit src meta doCheck; - - nativeBuildInputs = map (p: pythonPackages.${p}) ( - (info.setup_requires or []) ++ - (lib.optionals doCheck (info.tests_require or [])) - ); - - propagatedBuildInputs = map (p: pythonPackages.${p}) - (info.install_requires or []); -} diff --git a/pkgs/build-support/buildenv/default.nix b/pkgs/build-support/buildenv/default.nix index 786a2ad5da02c..560f59bcce7da 100644 --- a/pkgs/build-support/buildenv/default.nix +++ b/pkgs/build-support/buildenv/default.nix @@ -3,6 +3,13 @@ { buildPackages, runCommand, lib, substituteAll }: +let + builder = substituteAll { + src = ./builder.pl; + inherit (builtins) storeDir; + }; +in + lib.makeOverridable ({ name @@ -43,13 +50,6 @@ lib.makeOverridable , meta ? {} }: -let - builder = substituteAll { - src = ./builder.pl; - inherit (builtins) storeDir; - }; -in - runCommand name rec { inherit manifest ignoreCollisions checkCollisionContents passthru diff --git a/pkgs/build-support/cc-wrapper/add-hardening.sh b/pkgs/build-support/cc-wrapper/add-hardening.sh index 8d02b4e5124d8..e884f8388b58b 100644 --- a/pkgs/build-support/cc-wrapper/add-hardening.sh +++ b/pkgs/build-support/cc-wrapper/add-hardening.sh @@ -10,6 +10,13 @@ for flag in ${NIX_HARDENING_ENABLE_@suffixSalt@-}; do hardeningEnableMap["$flag"]=1 done +# fortify3 implies fortify enablement - make explicit before +# we filter unsupported flags because unsupporting fortify3 +# doesn't mean we should unsupport fortify too +if [[ -n "${hardeningEnableMap[fortify3]-}" ]]; then + hardeningEnableMap["fortify"]=1 +fi + # Remove unsupported flags. for flag in @hardening_unsupported_flags@; do unset -v "hardeningEnableMap[$flag]" @@ -19,13 +26,13 @@ for flag in @hardening_unsupported_flags@; do fi done -# make fortify and fortify3 mutually exclusive +# now make fortify and fortify3 mutually exclusive if [[ -n "${hardeningEnableMap[fortify3]-}" ]]; then unset -v "hardeningEnableMap['fortify']" fi if (( "${NIX_DEBUG:-0}" >= 1 )); then - declare -a allHardeningFlags=(fortify stackprotector pie pic strictoverflow format) + declare -a allHardeningFlags=(fortify fortify3 stackprotector pie pic strictoverflow format zerocallusedregs) declare -A hardeningDisableMap=() # Determine which flags were effectively disabled so we can report below. @@ -103,6 +110,10 @@ for flag in "${!hardeningEnableMap[@]}"; do if (( "${NIX_DEBUG:-0}" >= 1 )); then echo HARDENING: enabling format >&2; fi hardeningCFlagsBefore+=('-Wformat' '-Wformat-security' '-Werror=format-security') ;; + zerocallusedregs) + if (( "${NIX_DEBUG:-0}" >= 1 )); then echo HARDENING: enabling zerocallusedregs >&2; fi + hardeningCFlagsBefore+=('-fzero-call-used-regs=used-gpr') + ;; *) # Ignore unsupported. Checked in Nix that at least *some* # tool supports each flag. diff --git a/pkgs/build-support/cc-wrapper/cc-wrapper.sh b/pkgs/build-support/cc-wrapper/cc-wrapper.sh index 9dcd29c644316..b8d170df01b73 100644 --- a/pkgs/build-support/cc-wrapper/cc-wrapper.sh +++ b/pkgs/build-support/cc-wrapper/cc-wrapper.sh @@ -31,7 +31,6 @@ cxxLibrary=1 cInclude=1 expandResponseParams "$@" -linkType=$(checkLinkType "${params[@]}") declare -ag positionalArgs=() declare -i n=0 @@ -175,6 +174,7 @@ extraAfter=(${hardeningCFlagsAfter[@]+"${hardeningCFlagsAfter[@]}"} $NIX_CFLAGS_ extraBefore=(${hardeningCFlagsBefore[@]+"${hardeningCFlagsBefore[@]}"} $NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@) if [ "$dontLink" != 1 ]; then + linkType=$(checkLinkType $NIX_LDFLAGS_BEFORE_@suffixSalt@ "${params[@]}" ${NIX_CFLAGS_LINK_@suffixSalt@:-} $NIX_LDFLAGS_@suffixSalt@) # Add the flags that should only be passed to the compiler when # linking. @@ -246,7 +246,7 @@ if [[ -e @out@/nix-support/cc-wrapper-hook ]]; then fi if (( "${NIX_CC_USE_RESPONSE_FILE:-@use_response_file_by_default@}" >= 1 )); then - responseFile=$(mktemp --tmpdir cc-params.XXXXXX) + responseFile=$(mktemp "${TMPDIR:-/tmp}/cc-params.XXXXXX") trap 'rm -f -- "$responseFile"' EXIT printf "%q\n" \ ${extraBefore+"${extraBefore[@]}"} \ diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix index e1da3ceb5bc9e..693c6e6fcfd49 100644 --- a/pkgs/build-support/cc-wrapper/default.nix +++ b/pkgs/build-support/cc-wrapper/default.nix @@ -110,7 +110,23 @@ let gccForLibs_solib = getLib gccForLibs + optionalString (targetPlatform != hostPlatform) "/${targetPlatform.config}"; - # older compilers (for example bootstrap's GCC 5) fail with -march=too-modern-cpu + # Analogously to cc_solib and gccForLibs_solib + libcxx_solib = "${lib.getLib libcxx}/lib"; + + # The following two functions, `isGccArchSupported` and + # `isGccTuneSupported`, only handle those situations where a flag + # (`-march` or `-mtune`) is accepted by one compiler but rejected + # by another, and both compilers are relevant to nixpkgs. We are + # not trying to maintain a complete list of all flags accepted by + # all versions of all compilers ever in nixpkgs. + # + # The two main cases of interest are: + # + # - One compiler is gcc and the other is clang + # - One compiler is pkgs.gcc and the other is bootstrap-files.gcc + # -- older compilers (for example bootstrap's GCC 5) fail with + # -march=too-modern-cpu + isGccArchSupported = arch: if targetPlatform.isPower then false else # powerpc does not allow -march= if isGNU then @@ -159,6 +175,62 @@ let else false; + isGccTuneSupported = tune: + # for x86 -mtune= takes the same values as -march, plus two more: + if targetPlatform.isx86 then + { + generic = true; + intel = true; + }.${tune} or (isGccArchSupported tune) + # on arm64, the -mtune= values are specific processors + else if targetPlatform.isAarch64 then + (if isGNU then + { + cortex-a53 = versionAtLeast ccVersion "4.8"; # gcc 8c075f + cortex-a72 = versionAtLeast ccVersion "5.1"; # gcc d8f70d + "cortex-a72.cortex-a53" = versionAtLeast ccVersion "5.1"; # gcc d8f70d + }.${tune} or false + else if isClang then + { + cortex-a53 = versionAtLeast ccVersion "3.9"; # llvm dfc5d1 + }.${tune} or false + else false) + else if targetPlatform.isPower then + # powerpc does not support -march + true + else if targetPlatform.isMips then + # for mips -mtune= takes the same values as -march + isGccArchSupported tune + else + false; + + # Clang does not support as many `-mtune=` values as gcc does; + # this function will return the best possible approximation of the + # provided `-mtune=` value, or `null` if none exists. + # + # Note: this function can make use of ccVersion; for example, `if + # versionOlder ccVersion "12" then ...` + findBestTuneApproximation = tune: + let guess = if isClang + then { + # clang does not tune for big.LITTLE chips + "cortex-a72.cortex-a53" = "cortex-a72"; + }.${tune} or tune + else tune; + in if isGccTuneSupported guess + then guess + else null; + + defaultHardeningFlags = bintools.defaultHardeningFlags or []; + + # if cc.hardeningUnsupportedFlagsByTargetPlatform exists, this is + # called with the targetPlatform as an argument and + # cc.hardeningUnsupportedFlags is completely ignored - the function + # is responsible for including the constant hardeningUnsupportedFlags + # list however it sees fit. + ccHardeningUnsupportedFlags = if cc ? hardeningUnsupportedFlagsByTargetPlatform + then cc.hardeningUnsupportedFlagsByTargetPlatform targetPlatform + else (cc.hardeningUnsupportedFlags or []); darwinPlatformForCC = optionalString stdenv.targetPlatform.isDarwin ( if (targetPlatform.darwinPlatform == "macos" && isGNU) then "macosx" @@ -213,6 +285,8 @@ stdenv.mkDerivation { inherit expand-response-params; inherit nixSupport; + + inherit defaultHardeningFlags; }; dontBuild = true; @@ -460,15 +534,16 @@ stdenv.mkDerivation { # additional -isystem flags will confuse gfortran (see # https://github.com/NixOS/nixpkgs/pull/209870#issuecomment-1500550903) + optionalString (libcxx == null && isClang && (useGccForLibs && gccForLibs.langCC or false)) '' - for dir in ${gccForLibs}${lib.optionalString (hostPlatform != targetPlatform) "/${targetPlatform.config}"}/include/c++/*; do + for dir in ${gccForLibs}/include/c++/*; do echo "-isystem $dir" >> $out/nix-support/libcxx-cxxflags done - for dir in ${gccForLibs}${lib.optionalString (hostPlatform != targetPlatform) "/${targetPlatform.config}"}/include/c++/*/${targetPlatform.config}; do + for dir in ${gccForLibs}/include/c++/*/${targetPlatform.config}; do echo "-isystem $dir" >> $out/nix-support/libcxx-cxxflags done '' + optionalString (libcxx.isLLVM or false) '' echo "-isystem ${lib.getDev libcxx}/include/c++/v1" >> $out/nix-support/libcxx-cxxflags + echo "-isystem ${lib.getDev libcxx.cxxabi}/include/c++/v1" >> $out/nix-support/libcxx-cxxflags echo "-stdlib=libc++" >> $out/nix-support/libcxx-ldflags echo "-l${libcxx.cxxabi.libName}" >> $out/nix-support/libcxx-ldflags '' @@ -501,7 +576,7 @@ stdenv.mkDerivation { echo "$ccLDFlags" >> $out/nix-support/cc-ldflags echo "$ccCFlags" >> $out/nix-support/cc-cflags '' + optionalString (targetPlatform.isDarwin && (libcxx != null) && (cc.isClang or false)) '' - echo " -L${lib.getLib libcxx}/lib" >> $out/nix-support/cc-ldflags + echo " -L${libcxx_solib}" >> $out/nix-support/cc-ldflags '' ## @@ -518,7 +593,7 @@ stdenv.mkDerivation { ## Hardening support ## + '' - export hardening_unsupported_flags="${builtins.concatStringsSep " " (cc.hardeningUnsupportedFlags or [])}" + export hardening_unsupported_flags="${builtins.concatStringsSep " " ccHardeningUnsupportedFlags}" '' # Machine flags. These are necessary to support @@ -558,10 +633,12 @@ stdenv.mkDerivation { + optionalString (targetPlatform ? gcc.thumb) '' echo "-m${if targetPlatform.gcc.thumb then "thumb" else "arm"}" >> $out/nix-support/cc-cflags-before '' - + optionalString (targetPlatform ? gcc.tune && - isGccArchSupported targetPlatform.gcc.tune) '' - echo "-mtune=${targetPlatform.gcc.tune}" >> $out/nix-support/cc-cflags-before - '' + + (let tune = if targetPlatform ? gcc.tune + then findBestTuneApproximation targetPlatform.gcc.tune + else null; + in optionalString (tune != null) '' + echo "-mtune=${tune}" >> $out/nix-support/cc-cflags-before + '') # TODO: categorize these and figure out a better place for them + optionalString targetPlatform.isWindows '' @@ -588,12 +665,6 @@ stdenv.mkDerivation { hardening_unsupported_flags+=" stackprotector" '' - + optionalString (libc != null && targetPlatform.isAvr) '' - for isa in avr5 avr3 avr4 avr6 avr25 avr31 avr35 avr51 avrxmega2 avrxmega4 avrxmega5 avrxmega6 avrxmega7 tiny-stack; do - echo "-B${getLib libc}/avr/lib/$isa" >> $out/nix-support/libc-crt1-cflags - done - '' - + optionalString stdenv.targetPlatform.isDarwin '' echo "-arch ${targetPlatform.darwinArch}" >> $out/nix-support/cc-cflags '' @@ -651,6 +722,7 @@ stdenv.mkDerivation { inherit suffixSalt coreutils_bin bintools; inherit libc_bin libc_dev libc_lib; inherit darwinPlatformForCC darwinMinVersion darwinMinVersionVariable; + default_hardening_flags_str = builtins.toString defaultHardeningFlags; }; meta = diff --git a/pkgs/build-support/cc-wrapper/fortran-hook.sh b/pkgs/build-support/cc-wrapper/fortran-hook.sh index d72f314c01ce6..02da7fd0dc34c 100644 --- a/pkgs/build-support/cc-wrapper/fortran-hook.sh +++ b/pkgs/build-support/cc-wrapper/fortran-hook.sh @@ -4,8 +4,7 @@ getTargetRoleWrapper export FC${role_post}=@named_fc@ # If unset, assume the default hardening flags. -# These are different for fortran. -: ${NIX_HARDENING_ENABLE="stackprotector pic strictoverflow relro bindnow"} +: ${NIX_HARDENING_ENABLE="@default_hardening_flags_str@"} export NIX_HARDENING_ENABLE unset -v role_post diff --git a/pkgs/build-support/cc-wrapper/setup-hook.sh b/pkgs/build-support/cc-wrapper/setup-hook.sh index 9326d76e2a8ff..33a2b62a49b00 100644 --- a/pkgs/build-support/cc-wrapper/setup-hook.sh +++ b/pkgs/build-support/cc-wrapper/setup-hook.sh @@ -111,7 +111,7 @@ export CC${role_post}=@named_cc@ export CXX${role_post}=@named_cxx@ # If unset, assume the default hardening flags. -: ${NIX_HARDENING_ENABLE="fortify fortify3 stackprotector pic strictoverflow format relro bindnow"} +: ${NIX_HARDENING_ENABLE="@default_hardening_flags_str@"} export NIX_HARDENING_ENABLE # No local scope in sourced file diff --git a/pkgs/build-support/checkpoint-build.nix b/pkgs/build-support/checkpoint-build.nix new file mode 100644 index 0000000000000..c9bee45005a13 --- /dev/null +++ b/pkgs/build-support/checkpoint-build.nix @@ -0,0 +1,90 @@ +{ lib +, buildPackages +}: + +let + # rudimentary support for cross-compiling + # see: https://github.com/NixOS/nixpkgs/pull/279487#discussion_r1444449726 + inherit (buildPackages) + mktemp + rsync + ; +in + +rec { + /* Prepare a derivation for local builds. + * + * This function prepares checkpoint builds by storing + * the build output and the sources for cross checking. + * The build output can be used later to allow checkpoint builds + * by passing the derivation output to the `mkCheckpointBuild` function. + * + * To build a project with checkpoints, follow these steps: + * - run `prepareCheckpointBuild` on the desired derivation, e.g. + * checkpointArtifacts = prepareCheckpointBuild virtualbox; + * - change something you want in the sources of the package, + * e.g. using source override: + * changedVBox = pkgs.virtuabox.overrideAttrs (old: { + * src = path/to/vbox/sources; + * }; + * - use `mkCheckpointBuild changedVBox checkpointArtifacts` + * - enjoy shorter build times + */ + prepareCheckpointBuild = drv: drv.overrideAttrs (old: { + outputs = [ "out" ]; + name = drv.name + "-checkpointArtifacts"; + # To determine differences between the state of the build directory + # from an earlier build and a later one we store the state of the build + # directory before build, but after patch phases. + # This way, the same derivation can be used multiple times and only changes are detected. + # Additionally, removed files are handled correctly in later builds. + preBuild = (old.preBuild or "") + '' + mkdir -p $out/sources + cp -r ./* $out/sources/ + ''; + + # After the build, the build directory is copied again + # to get the output files. + # We copy the complete build folder, to take care of + # build tools that build in the source directory, instead of + # having a separate build directory such as the Linux kernel. + installPhase = '' + runHook preCheckpointInstall + mkdir -p $out/outputs + cp -r ./* $out/outputs/ + runHook postCheckpointInstall + ''; + }); + + /* Build a derivation based on the checkpoint output generated by + * the `prepareCheckpointBuild` function. + * + * Usage: + * let + * checkpointArtifacts = prepareCheckpointBuild drv; + * in mkCheckpointBuild drv checkpointArtifacts + */ + mkCheckpointBuild = drv: checkpointArtifacts: drv.overrideAttrs (old: { + # The actual checkpoint build phase. + # We compare the changed sources from a previous build with the current and create a patch. + # Afterwards we clean the build directory and copy the previous output files (including the sources). + # The source difference patch is then applied to get the latest changes again to allow short build times. + preBuild = (old.preBuild or "") + '' + set +e + sourceDifferencePatchFile=$(${mktemp}/bin/mktemp) + diff -ur ${checkpointArtifacts}/sources ./ > "$sourceDifferencePatchFile" + set -e + shopt -s dotglob + rm -r * + ${rsync}/bin/rsync \ + --checksum --times --atimes --chown=$USER:$USER --chmod=+w \ + -r ${checkpointArtifacts}/outputs/ . + patch -p 1 -i "$sourceDifferencePatchFile" + rm "$sourceDifferencePatchFile" + ''; + }); + + mkCheckpointedBuild = lib.warn + "`mkCheckpointedBuild` is deprecated, use `mkCheckpointBuild` instead!" + mkCheckpointBuild; +} diff --git a/pkgs/build-support/closure-info.nix b/pkgs/build-support/closure-info.nix index 6b3ff6fd62b05..f2aa4964d9a1d 100644 --- a/pkgs/build-support/closure-info.nix +++ b/pkgs/build-support/closure-info.nix @@ -4,7 +4,7 @@ # "nix-store --load-db" and "nix-store --register-validity # --hash-given". -{ stdenv, buildPackages }: +{ stdenv, coreutils, jq }: { rootPaths }: @@ -19,18 +19,24 @@ stdenv.mkDerivation { preferLocalBuild = true; - PATH = "${buildPackages.coreutils}/bin:${buildPackages.jq}/bin"; + nativeBuildInputs = [ coreutils jq ]; - builder = builtins.toFile "builder" - '' - . .attrs.sh + empty = rootPaths == []; + buildCommand = + '' out=''${outputs[out]} mkdir $out - jq -r ".closure | map(.narSize) | add" < .attrs.json > $out/total-nar-size - jq -r '.closure | map([.path, .narHash, .narSize, "", (.references | length)] + .references) | add | map("\(.)\n") | add' < .attrs.json | head -n -1 > $out/registration - jq -r .closure[].path < .attrs.json > $out/store-paths + if [[ -n "$empty" ]]; then + echo 0 > $out/total-nar-size + touch $out/registration $out/store-paths + else + jq -r ".closure | map(.narSize) | add" < "$NIX_ATTRS_JSON_FILE" > $out/total-nar-size + jq -r '.closure | map([.path, .narHash, .narSize, "", (.references | length)] + .references) | add | map("\(.)\n") | add' < "$NIX_ATTRS_JSON_FILE" | head -n -1 > $out/registration + jq -r '.closure[].path' < "$NIX_ATTRS_JSON_FILE" > $out/store-paths + fi + ''; } diff --git a/pkgs/build-support/dart/build-dart-application/default.nix b/pkgs/build-support/dart/build-dart-application/default.nix index be1fd72776711..c99b8bbf325e9 100644 --- a/pkgs/build-support/dart/build-dart-application/default.nix +++ b/pkgs/build-support/dart/build-dart-application/default.nix @@ -1,6 +1,27 @@ -{ lib, stdenv, fetchDartDeps, runCommand, writeText, dartHooks, makeWrapper, dart, cacert, nodejs, darwin }: +{ lib +, stdenv +, callPackage +, runCommand +, writeText +, pub2nix +, dartHooks +, makeWrapper +, dart +, nodejs +, darwin +, jq +, yq +}: -{ pubGetScript ? "dart pub get" +{ src +, sourceRoot ? "source" +, packageRoot ? (lib.removePrefix "/" (lib.removePrefix "source" sourceRoot)) +, gitHashes ? { } +, sdkSourceBuilders ? { } +, customSourceBuilders ? { } + +, sdkSetupScript ? "" +, extraPackageConfigSetup ? "" # Output type to produce. Can be any kind supported by dart # https://dart.dev/tools/dart-compile#types-of-output @@ -18,57 +39,94 @@ , dartEntryPoints ? null # Used when wrapping aot, jit, kernel, and js builds. # Set to null to disable wrapping. -, dartRuntimeCommand ? - if dartOutputType == "aot-snapshot" then "${dart}/bin/dartaotruntime" - else if (dartOutputType == "jit-snapshot" || dartOutputType == "kernel") then "${dart}/bin/dart" - else if dartOutputType == "js" then "${nodejs}/bin/node" - else null - -, pubspecLockFile ? null -, vendorHash ? "" +, dartRuntimeCommand ? if dartOutputType == "aot-snapshot" then "${dart}/bin/dartaotruntime" + else if (dartOutputType == "jit-snapshot" || dartOutputType == "kernel") then "${dart}/bin/dart" + else if dartOutputType == "js" then "${nodejs}/bin/node" + else null + +, runtimeDependencies ? [ ] +, extraWrapProgramArgs ? "" + +, autoPubspecLock ? null +, pubspecLock ? if autoPubspecLock == null then + throw "The pubspecLock argument is required. If import-from-derivation is allowed (it isn't in Nixpkgs), you can set autoPubspecLock to the path to a pubspec.lock instead." + else + assert lib.assertMsg (builtins.pathExists autoPubspecLock) "The pubspec.lock file could not be found!"; + lib.importJSON (runCommand "${lib.getName args}-pubspec-lock-json" { nativeBuildInputs = [ yq ]; } ''yq . '${autoPubspecLock}' > "$out"'') , ... }@args: let - dartDeps = (fetchDartDeps.override { - dart = runCommand "dart-fod" { nativeBuildInputs = [ makeWrapper ]; } '' - mkdir -p "$out/bin" - makeWrapper "${dart}/bin/dart" "$out/bin/dart" \ - --add-flags "--root-certs-file=${cacert}/etc/ssl/certs/ca-bundle.crt" - ''; - }) { - buildDrvArgs = args; - inherit pubGetScript vendorHash pubspecLockFile; + generators = callPackage ./generators.nix { inherit dart; } { buildDrvArgs = args; }; + + pubspecLockFile = builtins.toJSON pubspecLock; + pubspecLockData = pub2nix.readPubspecLock { inherit src packageRoot pubspecLock gitHashes sdkSourceBuilders customSourceBuilders; }; + packageConfig = generators.linkPackageConfig { + packageConfig = pub2nix.generatePackageConfig { + pname = if args.pname != null then "${args.pname}-${args.version}" else null; + + dependencies = + # Ideally, we'd only include the main dependencies and their transitive + # dependencies. + # + # The pubspec.lock file does not contain information about where + # transitive dependencies come from, though, and it would be weird to + # include the transitive dependencies of dev and override dependencies + # without including the dev and override dependencies themselves. + builtins.concatLists (builtins.attrValues pubspecLockData.dependencies); + + inherit (pubspecLockData) dependencySources; + }; + extraSetupCommands = extraPackageConfigSetup; }; - inherit (dartHooks.override { inherit dart; }) dartConfigHook dartBuildHook dartInstallHook; -in -assert !(builtins.isString dartOutputType && dartOutputType != "") -> - throw "dartOutputType must be a non-empty string"; -stdenv.mkDerivation (args // { - inherit pubGetScript dartCompileCommand dartOutputType dartRuntimeCommand - dartCompileFlags dartJitFlags; + + inherit (dartHooks.override { inherit dart; }) dartConfigHook dartBuildHook dartInstallHook dartFixupHook; + + baseDerivation = stdenv.mkDerivation (finalAttrs: (builtins.removeAttrs args [ "gitHashes" "sdkSourceBuilders" "pubspecLock" "customSourceBuilders" ]) // { + inherit pubspecLockFile packageConfig sdkSetupScript + dartCompileCommand dartOutputType dartRuntimeCommand dartCompileFlags + dartJitFlags; + + outputs = [ "out" "pubcache" ] ++ args.outputs or [ ]; dartEntryPoints = if (dartEntryPoints != null) then writeText "entrypoints.json" (builtins.toJSON dartEntryPoints) else null; - nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ - dart - dartDeps - dartConfigHook - dartBuildHook - dartInstallHook - makeWrapper - ] ++ lib.optionals stdenv.isDarwin [ - darwin.sigtool - ]; - - # When stripping, it seems some ELF information is lost and the dart VM cli - # runs instead of the expected program. Don't strip if it's an exe output. - dontStrip = args.dontStrip or (dartOutputType == "exe"); - - passthru = { inherit dartDeps; } // (args.passthru or { }); - - meta = (args.meta or { }) // { platforms = args.meta.platforms or dart.meta.platforms; }; -}) + runtimeDependencies = map lib.getLib runtimeDependencies; + + nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ + dart + dartConfigHook + dartBuildHook + dartInstallHook + dartFixupHook + makeWrapper + jq + ] ++ lib.optionals stdenv.isDarwin [ + darwin.sigtool + ] ++ + # Ensure that we inherit the propagated build inputs from the dependencies. + builtins.attrValues pubspecLockData.dependencySources; + + preConfigure = args.preConfigure or "" + '' + ln -sf "$pubspecLockFilePath" pubspec.lock + ''; + + # When stripping, it seems some ELF information is lost and the dart VM cli + # runs instead of the expected program. Don't strip if it's an exe output. + dontStrip = args.dontStrip or (dartOutputType == "exe"); + + passAsFile = [ "pubspecLockFile" ]; + + passthru = { + pubspecLock = pubspecLockData; + } // (args.passthru or { }); + + meta = (args.meta or { }) // { platforms = args.meta.platforms or dart.meta.platforms; }; + }); +in +assert !(builtins.isString dartOutputType && dartOutputType != "") -> +throw "dartOutputType must be a non-empty string"; +baseDerivation diff --git a/pkgs/build-support/dart/build-dart-application/generators.nix b/pkgs/build-support/dart/build-dart-application/generators.nix new file mode 100644 index 0000000000000..f01a09305dbaa --- /dev/null +++ b/pkgs/build-support/dart/build-dart-application/generators.nix @@ -0,0 +1,74 @@ +{ lib +, stdenvNoCC +, dart +, dartHooks +, jq +, yq +, cacert +}: + +{ + # Arguments used in the derivation that builds the Dart package. + # Passing these is recommended to ensure that the same steps are made to + # prepare the sources in both this derivation and the one that builds the Dart + # package. + buildDrvArgs ? { } +, ... +}@args: + +# This is a derivation and setup hook that can be used to fetch dependencies for Dart projects. +# It is designed to be placed in the nativeBuildInputs of a derivation that builds a Dart package. +# Providing the buildDrvArgs argument is highly recommended. +let + buildDrvInheritArgNames = [ + "name" + "pname" + "version" + "src" + "sourceRoot" + "setSourceRoot" + "preUnpack" + "unpackPhase" + "unpackCmd" + "postUnpack" + "prePatch" + "patchPhase" + "patches" + "patchFlags" + "postPatch" + ]; + + buildDrvInheritArgs = builtins.foldl' + (attrs: arg: + if buildDrvArgs ? ${arg} + then attrs // { ${arg} = buildDrvArgs.${arg}; } + else attrs) + { } + buildDrvInheritArgNames; + + drvArgs = buildDrvInheritArgs // (removeAttrs args [ "buildDrvArgs" ]); + name = (if drvArgs ? name then drvArgs.name else "${drvArgs.pname}-${drvArgs.version}"); + + # Adds the root package to a dependency package_config.json file from pub2nix. + linkPackageConfig = { packageConfig, extraSetupCommands ? "" }: stdenvNoCC.mkDerivation (drvArgs // { + name = "${name}-package-config-with-root.json"; + + nativeBuildInputs = drvArgs.nativeBuildInputs or [ ] ++ args.nativeBuildInputs or [ ] ++ [ jq yq ]; + + dontBuild = true; + + installPhase = '' + runHook preInstall + + packageName="$(yq --raw-output .name pubspec.yaml)" + jq --arg name "$packageName" '.packages |= . + [{ name: $name, rootUri: "../", packageUri: "lib/" }]' '${packageConfig}' > "$out" + ${extraSetupCommands} + + runHook postInstall + ''; + }); +in +{ + inherit + linkPackageConfig; +} diff --git a/pkgs/build-support/dart/build-dart-application/hooks/dart-config-hook.sh b/pkgs/build-support/dart/build-dart-application/hooks/dart-config-hook.sh index 3e901995237d9..50754a7b56d45 100644 --- a/pkgs/build-support/dart/build-dart-application/hooks/dart-config-hook.sh +++ b/pkgs/build-support/dart/build-dart-application/hooks/dart-config-hook.sh @@ -3,8 +3,66 @@ dartConfigHook() { echo "Executing dartConfigHook" + echo "Setting up SDK" + eval "$sdkSetupScript" + echo "Installing dependencies" - eval doPubGet "$pubGetScript" --offline + mkdir -p .dart_tool + cp "$packageConfig" .dart_tool/package_config.json + + packagePath() { + jq --raw-output --arg name "$1" '.packages.[] | select(.name == $name) .rootUri | sub("file://"; "")' .dart_tool/package_config.json + } + + # Runs a Dart executable from a package with a custom path. + # + # Usage: + # packageRunCustom <package> [executable] [bin_dir] + # + # By default, [bin_dir] is "bin", and [executable] is <package>. + # i.e. `packageRunCustom build_runner` is equivalent to `packageRunCustom build_runner build_runner bin`, which runs `bin/build_runner.dart` from the build_runner package. + packageRunCustom() { + local args=() + local passthrough=() + + while [ $# -gt 0 ]; do + if [ "$1" != "--" ]; then + args+=("$1") + shift + else + shift + passthrough=("$@") + break + fi + done + + local name="${args[0]}" + local path="${args[1]:-$name}" + local prefix="${args[2]:-bin}" + + dart --packages=.dart_tool/package_config.json "$(packagePath "$name")/$prefix/$path.dart" "${passthrough[@]}" + } + + # Runs a Dart executable from a package. + # + # Usage: + # packageRun <package> [-e executable] [...] + # + # To run an executable from an unconventional location, use packageRunCustom. + packageRun() { + local name="$1" + shift + + local executableName="$name" + if [ "$1" = "-e" ]; then + shift + executableName="$1" + shift + fi + + fileName="$(@yq@ --raw-output --arg name "$executableName" '.executables.[$name] // $name' "$(packagePath "$name")/pubspec.yaml")" + packageRunCustom "$name" "$fileName" -- "$@" + } echo "Finished dartConfigHook" } diff --git a/pkgs/build-support/dart/build-dart-application/hooks/dart-fixup-hook.sh b/pkgs/build-support/dart/build-dart-application/hooks/dart-fixup-hook.sh new file mode 100644 index 0000000000000..60bd74871c92f --- /dev/null +++ b/pkgs/build-support/dart/build-dart-application/hooks/dart-fixup-hook.sh @@ -0,0 +1,35 @@ +# shellcheck shell=bash + +dartFixupHook() { + echo "Executing dartFixupHook" + + declare -a wrapProgramArgs + + # Add runtime library dependencies to the LD_LIBRARY_PATH. + # For some reason, the RUNPATH of the executable is not used to load dynamic libraries in dart:ffi with DynamicLibrary.open(). + # + # This could alternatively be fixed with patchelf --add-needed, but this would cause all the libraries to be opened immediately, + # which is not what application authors expect. + APPLICATION_LD_LIBRARY_PATH="" + for runtimeDependency in "${runtimeDependencies[@]}"; do + addToSearchPath APPLICATION_LD_LIBRARY_PATH "${runtimeDependency}/lib" + done + if [[ ! -z "$APPLICATION_LD_LIBRARY_PATH" ]]; then + wrapProgramArgs+=(--suffix LD_LIBRARY_PATH : \"$APPLICATION_LD_LIBRARY_PATH\") + fi + + if [[ ! -z "$extraWrapProgramArgs" ]]; then + wrapProgramArgs+=("$extraWrapProgramArgs") + fi + + if [ ${#wrapProgramArgs[@]} -ne 0 ]; then + for f in "$out"/bin/*; do + echo "Wrapping $f..." + eval "wrapProgram \"$f\" ${wrapProgramArgs[@]}" + done + fi + + echo "Finished dartFixupHook" +} + +postFixupHooks+=(dartFixupHook) diff --git a/pkgs/build-support/dart/build-dart-application/hooks/dart-install-hook.sh b/pkgs/build-support/dart/build-dart-application/hooks/dart-install-hook.sh index 1906bcfbca4cd..349a0dfdef0e0 100644 --- a/pkgs/build-support/dart/build-dart-application/hooks/dart-install-hook.sh +++ b/pkgs/build-support/dart/build-dart-application/hooks/dart-install-hook.sh @@ -5,8 +5,8 @@ dartInstallHook() { runHook preInstall + # Install snapshots and executables. mkdir -p "$out" - while IFS=$'\t' read -ra target; do dest="${target[0]}" # Wrap with runtime command, if it's defined @@ -24,6 +24,20 @@ dartInstallHook() { echo "Finished dartInstallHook" } +dartInstallCacheHook() { + echo "Executing dartInstallCacheHook" + + # Install the package_config.json file. + mkdir -p "$pubcache" + cp .dart_tool/package_config.json "$pubcache/package_config.json" + + echo "Finished dartInstallCacheHook" +} + if [ -z "${dontDartInstall-}" ] && [ -z "${installPhase-}" ]; then installPhase=dartInstallHook fi + +if [ -z "${dontDartInstallCache-}" ]; then + postInstallHooks+=(dartInstallCacheHook) +fi diff --git a/pkgs/build-support/dart/build-dart-application/hooks/default.nix b/pkgs/build-support/dart/build-dart-application/hooks/default.nix index 463061c54a8dd..253d3132ad02e 100644 --- a/pkgs/build-support/dart/build-dart-application/hooks/default.nix +++ b/pkgs/build-support/dart/build-dart-application/hooks/default.nix @@ -3,6 +3,8 @@ { dartConfigHook = makeSetupHook { name = "dart-config-hook"; + substitutions.yq = "${yq}/bin/yq"; + substitutions.jq = "${jq}/bin/jq"; } ./dart-config-hook.sh; dartBuildHook = makeSetupHook { name = "dart-build-hook"; @@ -12,4 +14,7 @@ dartInstallHook = makeSetupHook { name = "dart-install-hook"; } ./dart-install-hook.sh; + dartFixupHook = makeSetupHook { + name = "dart-fixup-hook"; + } ./dart-fixup-hook.sh; } diff --git a/pkgs/build-support/dart/fetch-dart-deps/default.nix b/pkgs/build-support/dart/fetch-dart-deps/default.nix deleted file mode 100644 index e523b60797eb1..0000000000000 --- a/pkgs/build-support/dart/fetch-dart-deps/default.nix +++ /dev/null @@ -1,201 +0,0 @@ -{ stdenvNoCC -, lib -, makeSetupHook -, writeShellScriptBin -, dart -, git -, cacert -, jq -}: - -{ - # The output hash of the dependencies for this project. - vendorHash ? "" - # Commands to run once before using Dart or pub. -, sdkSetupScript ? "" - # Commands to run to populate the pub cache. -, pubGetScript ? "dart pub get" - # A path to a pubspec.lock file to use instead of the one in the source directory. -, pubspecLockFile ? null - # Arguments used in the derivation that builds the Dart package. - # Passing these is recommended to ensure that the same steps are made to prepare the sources in both this - # derivation and the one that builds the Dart package. -, buildDrvArgs ? { } -, ... -}@args: - -# This is a fixed-output derivation and setup hook that can be used to fetch dependencies for Dart projects. -# It is designed to be placed in the nativeBuildInputs of a derivation that builds a Dart package. -# Providing the buildDrvArgs argument is highly recommended. -let - buildDrvInheritArgNames = [ - "name" - "pname" - "version" - "src" - "sourceRoot" - "setSourceRoot" - "preUnpack" - "unpackPhase" - "unpackCmd" - "postUnpack" - "prePatch" - "patchPhase" - "patches" - "patchFlags" - "postPatch" - ]; - - buildDrvInheritArgs = builtins.foldl' - (attrs: arg: - if buildDrvArgs ? ${arg} - then attrs // { ${arg} = buildDrvArgs.${arg}; } - else attrs) - { } - buildDrvInheritArgNames; - - drvArgs = buildDrvInheritArgs // (removeAttrs args [ "buildDrvArgs" ]); - name = (if drvArgs ? name then drvArgs.name else "${drvArgs.pname}-${drvArgs.version}"); - - deps = - stdenvNoCC.mkDerivation ({ - name = "${name}-dart-deps"; - - nativeBuildInputs = [ - dart - git - ]; - - # avoid pub phase - dontBuild = true; - - configurePhase = '' - # Configure the package cache - export PUB_CACHE="$out/cache/.pub-cache" - mkdir -p "$PUB_CACHE" - - ${sdkSetupScript} - ''; - - installPhase = '' - _pub_get() { - ${pubGetScript} - } - - # so we can use lock, diff yaml - mkdir -p "$out/pubspec" - cp "pubspec.yaml" "$out/pubspec" - ${lib.optionalString (pubspecLockFile != null) "install -m644 ${pubspecLockFile} pubspec.lock"} - if ! cp "pubspec.lock" "$out/pubspec"; then - echo 1>&2 -e '\nThe pubspec.lock file is missing. This is a requirement for reproducible builds.' \ - '\nThe following steps should be taken to fix this issue:' \ - '\n 1. If you are building an application, contact the developer(s).' \ - '\n The pubspec.lock file should be provided with the source code.' \ - '\n https://dart.dev/guides/libraries/private-files#pubspeclock' \ - '\n 2. An attempt to generate and print a compressed pubspec.lock file will be made now.' \ - '\n It is compressed with gzip and base64 encoded.' \ - '\n Paste it to a file and extract it with `base64 -d pubspec.lock.in | gzip -d > pubspec.lock`.' \ - '\n Provide the path to the pubspec.lock file in the pubspecLockFile argument.' \ - '\n This must be updated whenever the application is updated.' \ - '\n' - _pub_get - echo "" - gzip --to-stdout --best pubspec.lock | base64 1>&2 - echo 1>&2 -e '\nA gzipped pubspec.lock file has been printed. Please see the informational message above.' - exit 1 - fi - - _pub_get - - # nuke nondeterminism - - # Remove Git directories in the Git package cache - these are rarely used by Pub, - # which instead maintains a corresponsing mirror and clones cached packages through it. - # - # An exception is made to keep .git/pub-packages files, which are important. - # https://github.com/dart-lang/pub/blob/c890afa1d65b340fa59308172029680c2f8b0fc6/lib/src/source/git.dart#L621 - if [ -d "$PUB_CACHE"/git ]; then - find "$PUB_CACHE"/git -maxdepth 4 -path "*/.git/*" ! -name "pub-packages" -prune -exec rm -rf {} + - fi - - # Remove continuously updated package metadata caches - rm -rf "$PUB_CACHE"/hosted/*/.cache # Not pinned by pubspec.lock - rm -rf "$PUB_CACHE"/git/cache/*/* # Recreate this on the other end. See: https://github.com/dart-lang/pub/blob/c890afa1d65b340fa59308172029680c2f8b0fc6/lib/src/source/git.dart#L531 - - # Miscelaneous transient package cache files - rm -f "$PUB_CACHE"/README.md # May change with different Dart versions - rm -rf "$PUB_CACHE"/_temp # https://github.com/dart-lang/pub/blob/c890afa1d65b340fa59308172029680c2f8b0fc6/lib/src/system_cache.dart#L131 - rm -rf "$PUB_CACHE"/log # https://github.com/dart-lang/pub/blob/c890afa1d65b340fa59308172029680c2f8b0fc6/lib/src/command.dart#L348 - ''; - - GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt"; - SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; - - impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ - "GIT_PROXY_COMMAND" - "NIX_GIT_SSL_CAINFO" - "SOCKS_SERVER" - ]; - - # Patching shebangs introduces input references to this fixed-output derivation. - # This triggers a bug in Nix, causing the output path to change unexpectedly. - # https://github.com/NixOS/nix/issues/6660 - dontPatchShebangs = true; - - # The following operations are not generally useful for this derivation. - # If a package does contain some native components used at build time, - # please file an issue. - dontStrip = true; - dontMoveSbin = true; - dontPatchELF = true; - - outputHashAlgo = "sha256"; - outputHashMode = "recursive"; - outputHash = if vendorHash != "" then vendorHash else lib.fakeSha256; - } // (removeAttrs drvArgs [ "name" "pname" ])); - - depsListDrv = stdenvNoCC.mkDerivation ({ - name = "${name}-dart-deps-list.json"; - nativeBuildInputs = [ hook dart jq ]; - - configurePhase = '' - runHook preConfigure - doPubGet dart pub get --offline - runHook postConfigure - ''; - - buildPhase = '' - runHook preBuild - dart pub deps --json | jq .packages > $out - runHook postBuild - ''; - } // (removeAttrs buildDrvInheritArgs [ "name" "pname" ])); - - # As of Dart 3.0.0, Pub checks the revision of cached Git-sourced packages. - # Git must be wrapped to return a positive result, as the real .git directory is wiped - # to produce a deteministic dependency derivation output. - # https://github.com/dart-lang/pub/pull/3791/files#diff-1639c4669c428c26e68cfebd5039a33f87ba568795f2c058c303ca8528f62b77R631 - gitSourceWrapper = writeShellScriptBin "git" '' - args=("$@") - if [[ "''${args[0]}" == "rev-list" && "''${args[1]}" == "--max-count=1" ]]; then - revision="''${args[''${#args[@]}-1]}" - echo "$revision" - else - ${git}/bin/git "''${args[@]}" - fi - ''; - - hook = (makeSetupHook { - # The setup hook should not be part of the fixed-output derivation. - # Updates to the hook script should not change vendor hashes, and it won't - # work at all anyway due to https://github.com/NixOS/nix/issues/6660. - name = "${name}-dart-deps-setup-hook"; - substitutions = { inherit gitSourceWrapper deps; }; - propagatedBuildInputs = [ dart git ]; - passthru = { - files = deps.outPath; - depsListFile = depsListDrv.outPath; - }; - }) ./setup-hook.sh; -in -hook diff --git a/pkgs/build-support/dart/fetch-dart-deps/setup-hook.sh b/pkgs/build-support/dart/fetch-dart-deps/setup-hook.sh deleted file mode 100644 index 689e0e8c5b5fc..0000000000000 --- a/pkgs/build-support/dart/fetch-dart-deps/setup-hook.sh +++ /dev/null @@ -1,46 +0,0 @@ -preConfigureHooks+=(_setupPubCache) - -_setupPubCache() { - deps="@deps@" - - # Configure the package cache. - export PUB_CACHE="$(mktemp -d)" - mkdir -p "$PUB_CACHE" - - if [ -d "$deps/cache/.pub-cache/git" ]; then - # Link the Git package cache. - mkdir -p "$PUB_CACHE/git" - ln -s "$deps/cache/.pub-cache/git"/* "$PUB_CACHE/git" - - # Recreate the internal Git cache subdirectory. - # See: https://github.com/dart-lang/pub/blob/c890afa1d65b340fa59308172029680c2f8b0fc6/lib/src/source/git.dart#L339) - # Blank repositories are created instead of attempting to match the cache mirrors to checkouts. - # This is not an issue, as pub does not need the mirrors in the Flutter build process. - rm "$PUB_CACHE/git/cache" && mkdir "$PUB_CACHE/git/cache" - for mirror in $(ls -A "$deps/cache/.pub-cache/git/cache"); do - git --git-dir="$PUB_CACHE/git/cache/$mirror" init --bare --quiet - done - fi - - # Link the remaining package cache directories. - # At this point, any subdirectories that must be writable must have been taken care of. - for file in $(comm -23 <(ls -A "$deps/cache/.pub-cache") <(ls -A "$PUB_CACHE")); do - ln -s "$deps/cache/.pub-cache/$file" "$PUB_CACHE/$file" - done - - # ensure we're using a lockfile for the right package version - if [ ! -e pubspec.lock ]; then - cp -v "$deps/pubspec/pubspec.lock" . - # Sometimes the pubspec.lock will get opened in write mode, even when offline. - chmod u+w pubspec.lock - elif ! { diff -u pubspec.lock "$deps/pubspec/pubspec.lock" && diff -u pubspec.yaml "$deps/pubspec/pubspec.yaml"; }; then - echo 1>&2 -e 'The pubspec.lock or pubspec.yaml of the project derivation differs from the one in the dependency derivation.' \ - '\nYou most likely forgot to update the vendorHash while updating the sources.' - exit 1 - fi -} - -# Performs the given pub get command with an appropriate environment. -doPubGet() { - PATH="@gitSourceWrapper@/bin:$PATH" "$@" -} diff --git a/pkgs/build-support/dart/pub2nix/default.nix b/pkgs/build-support/dart/pub2nix/default.nix new file mode 100644 index 0000000000000..ace2cc5a1e0c7 --- /dev/null +++ b/pkgs/build-support/dart/pub2nix/default.nix @@ -0,0 +1,6 @@ +{ callPackage }: + +{ + readPubspecLock = callPackage ./pubspec-lock.nix { }; + generatePackageConfig = callPackage ./package-config.nix { }; +} diff --git a/pkgs/build-support/dart/pub2nix/package-config.nix b/pkgs/build-support/dart/pub2nix/package-config.nix new file mode 100644 index 0000000000000..70abb0a76cef5 --- /dev/null +++ b/pkgs/build-support/dart/pub2nix/package-config.nix @@ -0,0 +1,68 @@ +{ lib +, runCommand +, jq +, yq +}: + +{ pname ? null + + # A list of dependency package names. +, dependencies + + # An attribute set of package names to sources. +, dependencySources +}: + +let + packages = lib.genAttrs dependencies (dependency: rec { + src = dependencySources.${dependency}; + inherit (src) packageRoot; + }); +in +(runCommand "${lib.optionalString (pname != null) "${pname}-"}package-config.json" { + inherit packages; + + nativeBuildInputs = [ jq yq ]; + + __structuredAttrs = true; +}) '' + declare -A packageSources + declare -A packageRoots + while IFS=',' read -r name src packageRoot; do + packageSources["$name"]="$src" + packageRoots["$name"]="$packageRoot" + done < <(jq -r '.packages | to_entries | map("\(.key),\(.value.src),\(.value.packageRoot)") | .[]' "$NIX_ATTRS_JSON_FILE") + + for package in "''${!packageSources[@]}"; do + if [ ! -e "''${packageSources["$package"]}/''${packageRoots["$package"]}/pubspec.yaml" ]; then + echo >&2 "The package sources for $package are missing. Is the following path inside the source derivation?" + echo >&2 "Source path: ''${packageSources["$package"]}/''${packageRoots["$package"]}/pubspec.yaml" + exit 1 + fi + + languageConstraint="$(yq -r .environment.sdk "''${packageSources["$package"]}/''${packageRoots["$package"]}/pubspec.yaml")" + if [[ "$languageConstraint" =~ ^[[:space:]]*(\^|>=|>)?[[:space:]]*([[:digit:]]+\.[[:digit:]]+)\.[[:digit:]]+.*$ ]]; then + languageVersionJson="\"''${BASH_REMATCH[2]}\"" + elif [ "$languageConstraint" = 'any' ]; then + languageVersionJson='null' + else + # https://github.com/dart-lang/pub/blob/68dc2f547d0a264955c1fa551fa0a0e158046494/lib/src/language_version.dart#L106C35-L106C35 + languageVersionJson='"2.7"' + fi + + jq --null-input \ + --arg name "$package" \ + --arg path "''${packageSources["$package"]}/''${packageRoots["$package"]}" \ + --argjson languageVersion "$languageVersionJson" \ + '{ + name: $name, + rootUri: "file://\($path)", + packageUri: "lib/", + languageVersion: $languageVersion, + }' + done | jq > "$out" --slurp '{ + configVersion: 2, + generator: "nixpkgs", + packages: ., + }' +'' diff --git a/pkgs/build-support/dart/pub2nix/pubspec-lock.nix b/pkgs/build-support/dart/pub2nix/pubspec-lock.nix new file mode 100644 index 0000000000000..e1ab4d7d23590 --- /dev/null +++ b/pkgs/build-support/dart/pub2nix/pubspec-lock.nix @@ -0,0 +1,119 @@ +{ lib +, callPackage +, fetchurl +, fetchgit +, runCommand +}: + +{ + # The source directory of the package. + src + + # The package subdirectory within src. + # Useful if the package references sibling packages with relative paths. +, packageRoot ? "." + + # The pubspec.lock file, in attribute set form. +, pubspecLock + + # Hashes for Git dependencies. + # Pub does not record these itself, so they must be manually provided. +, gitHashes ? { } + + # Functions to generate SDK package sources. + # The function names should match the SDK names, and the package name is given as an argument. +, sdkSourceBuilders ? { } + + # Functions that create custom package source derivations. + # + # The function names should match the package names, and the package version, + # source, and source files are given in an attribute set argument. + # + # The passthru of the source derivation should be propagated. +, customSourceBuilders ? { } +}: + +let + dependencyVersions = builtins.mapAttrs (name: details: details.version) pubspecLock.packages; + + dependencyTypes = { + "direct main" = "main"; + "direct dev" = "dev"; + "direct overridden" = "overridden"; + "transitive" = "transitive"; + }; + + dependencies = lib.foldlAttrs + (dependencies: name: details: dependencies // { ${dependencyTypes.${details.dependency}} = dependencies.${dependencyTypes.${details.dependency}} ++ [ name ]; }) + (lib.genAttrs (builtins.attrValues dependencyTypes) (dependencyType: [ ])) + pubspecLock.packages; + + # fetchTarball fails with "tarball contains an unexpected number of top-level files". This is a workaround. + # https://discourse.nixos.org/t/fetchtarball-with-multiple-top-level-directories-fails/20556 + mkHostedDependencySource = name: details: + let + archive = fetchurl { + name = "pub-${name}-${details.version}.tar.gz"; + url = "${details.description.url}/packages/${details.description.name}/versions/${details.version}.tar.gz"; + sha256 = details.description.sha256; + }; + in + runCommand "pub-${name}-${details.version}" { passthru.packageRoot = "."; } '' + mkdir -p "$out" + tar xf '${archive}' -C "$out" + ''; + + mkGitDependencySource = name: details: (fetchgit { + name = "pub-${name}-${details.version}"; + url = details.description.url; + rev = details.description.resolved-ref; + hash = gitHashes.${name} or (throw "A Git hash is required for ${name}! Set to an empty string to obtain it."); + }).overrideAttrs ({ passthru ? { }, ... }: { + passthru = passthru // { + packageRoot = details.description.path; + }; + }); + + mkPathDependencySource = name: details: + assert lib.assertMsg details.description.relative "Only relative paths are supported - ${name} has an absolue path!"; + (if lib.isDerivation src then src else (runCommand "pub-${name}-${details.version}" { } ''cp -r '${src}' "$out"'')).overrideAttrs ({ passthru ? { }, ... }: { + passthru = passthru // { + packageRoot = "${packageRoot}/${details.description.path}"; + }; + }); + + mkSdkDependencySource = name: details: + (sdkSourceBuilders.${details.description} or (throw "No SDK source builder has been given for ${details.description}!")) name; + + addDependencySourceUtils = dependencySource: details: dependencySource.overrideAttrs ({ passthru, ... }: { + passthru = passthru // { + inherit (details) version; + }; + }); + + sourceBuilders = callPackage ../../../development/compilers/dart/package-source-builders { } // customSourceBuilders; + + dependencySources = lib.filterAttrs (name: src: src != null) (builtins.mapAttrs + (name: details: + (sourceBuilders.${name} or ({ src, ... }: src)) { + inherit (details) version source; + src = ((addDependencySourceUtils (({ + "hosted" = mkHostedDependencySource; + "git" = mkGitDependencySource; + "path" = mkPathDependencySource; + "sdk" = mkSdkDependencySource; + }.${details.source} name) details)) details); + }) + pubspecLock.packages); +in +{ + inherit + # An attribute set of dependency categories to package name lists. + dependencies + + # An attribute set of package names to their versions. + dependencyVersions + + # An attribute set of package names to their sources. + dependencySources; +} diff --git a/pkgs/build-support/deterministic-uname/deterministic-uname.sh b/pkgs/build-support/deterministic-uname/deterministic-uname.sh index 5272bb5b3fe17..31772aeee3cc1 100644 --- a/pkgs/build-support/deterministic-uname/deterministic-uname.sh +++ b/pkgs/build-support/deterministic-uname/deterministic-uname.sh @@ -38,6 +38,10 @@ processor=0 hardware_platform=0 operating_system=0 +# With no OPTION, same as -s. +if [[ $# -eq 0 ]]; then + kernel_name=1 +fi @getopt@/bin/getopt --test > /dev/null && rc=$? || rc=$? if [[ $rc -ne 4 ]]; then @@ -54,11 +58,6 @@ else eval set -- "$PARSED" fi -# With no OPTION, same as -s. -if [[ $# -eq 0 ]]; then - kernel_name=1 -fi - # Process each argument, and set the appropriate flag if we recognize it. while [[ $# -ge 1 ]]; do case "$1" in @@ -132,44 +131,44 @@ fi # Darwin *nodename* 22.1.0 Darwin Kernel Version 22.1.0: Sun Oct 9 20:14:30 PDT 2022; root:xnu-8792.41.9~2/RELEASE_ARM64_T8103 arm64 arm Darwin # NixOS: # Linux *nodename* 6.0.13 #1-NixOS SMP PREEMPT_DYNAMIC Wed Dec 14 10:41:06 UTC 2022 x86_64 GNU/Linux +output=() if [[ "$all" = "1" ]]; then - echo -n "$KERNEL_NAME_VAL $NODENAME_VAL $KERNEL_RELEASE_VAL $KERNEL_VERSION_VAL $MACHINE_VAL " + output+=("$KERNEL_NAME_VAL" "$NODENAME_VAL" "$KERNEL_RELEASE_VAL" "$KERNEL_VERSION_VAL" "$MACHINE_VAL") # in help: except omit -p and -i if unknown. - #echo -n "$PROCESSOR_VAL $HARDWARE_PLATFORM_VAL\n" - echo -n "$OPERATING_SYSTEM_VAL" + # output+=($PROCESSOR_VAL $HARDWARE_PLATFORM_VAL) + output+=("$OPERATING_SYSTEM_VAL") fi if [[ "$kernel_name" = "1" ]]; then - echo -n "$KERNEL_NAME_VAL" + output+=("$KERNEL_NAME_VAL") fi if [[ "$nodename" = "1" ]]; then - echo -n "$NODENAME_VAL" + output+=("$NODENAME_VAL") fi if [[ "$kernel_release" = "1" ]]; then - echo -n "$KERNEL_RELEASE_VAL" + output+=("$KERNEL_RELEASE_VAL") fi if [[ "$kernel_version" = "1" ]]; then - echo -n "$KERNEL_VERSION_VAL" + output+=("$KERNEL_VERSION_VAL") fi if [[ "$machine" = "1" ]]; then - echo -n "$MACHINE_VAL" + output+=("$MACHINE_VAL") fi if [[ "$processor" = "1" ]]; then - echo -n "$PROCESSOR_VAL" + output+=("$PROCESSOR_VAL") fi if [[ "$hardware_platform" = "1" ]]; then - echo -n "$HARDWARE_PLATFORM_VAL" + output+=("$HARDWARE_PLATFORM_VAL") fi if [[ "$operating_system" = "1" ]]; then - echo -n "$OPERATING_SYSTEM_VAL" + output+=("$OPERATING_SYSTEM_VAL") fi -# for newline. -echo +echo "${output[@]}" diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 1ac0a69f74512..3f61ecdb2a461 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -446,7 +446,7 @@ rec { ''; postMount = '' - mkdir -p mnt/{dev,proc,sys} mnt${storeDir} + mkdir -p mnt/{dev,proc,sys,tmp} mnt${storeDir} # Mount /dev, /sys and the nix store as shared folders. mount --rbind /dev mnt/dev @@ -805,6 +805,7 @@ rec { ''; # This provides /bin/sh, pointing to bashInteractive. + # The use of bashInteractive here is intentional to support cases like `docker run -it <image_name>`, so keep these use cases in mind if making any changes to how this works. binSh = runCommand "bin-sh" { } '' mkdir -p $out/bin ln -s ${bashInteractive}/bin/bash $out/bin/sh @@ -914,17 +915,31 @@ rec { (cd old_out; eval "$extraCommands" ) mkdir $out - ${optionalString enableFakechroot ''proot -r $PWD/old_out ${bind-paths} --pwd=/ ''}fakeroot bash -c ' - source $stdenv/setup - ${optionalString (!enableFakechroot) ''cd old_out''} - eval "$fakeRootCommands" - tar \ - --sort name \ - --numeric-owner --mtime "@$SOURCE_DATE_EPOCH" \ - --hard-dereference \ - -cf $out/layer.tar . - ' - + ${if enableFakechroot then '' + proot -r $PWD/old_out ${bind-paths} --pwd=/ fakeroot bash -c ' + source $stdenv/setup + eval "$fakeRootCommands" + tar \ + --sort name \ + --exclude=./proc \ + --exclude=./sys \ + --exclude=.${builtins.storeDir} \ + --numeric-owner --mtime "@$SOURCE_DATE_EPOCH" \ + --hard-dereference \ + -cf $out/layer.tar . + ' + '' else '' + fakeroot bash -c ' + source $stdenv/setup + cd old_out + eval "$fakeRootCommands" + tar \ + --sort name \ + --numeric-owner --mtime "@$SOURCE_DATE_EPOCH" \ + --hard-dereference \ + -cf $out/layer.tar . + ' + ''} sha256sum $out/layer.tar \ | cut -f 1 -d ' ' \ > $out/checksum diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix index 5784e650dc2e4..88f36d337f253 100644 --- a/pkgs/build-support/docker/examples.nix +++ b/pkgs/build-support/docker/examples.nix @@ -639,6 +639,20 @@ rec { includeStorePaths = false; }; + helloOnRootNoStoreFakechroot = pkgs.dockerTools.streamLayeredImage { + name = "hello"; + tag = "latest"; + contents = [ + (pkgs.buildEnv { + name = "hello-root"; + paths = [ pkgs.hello ]; + }) + ]; + config.Cmd = [ "hello" ]; + includeStorePaths = false; + enableFakechroot = true; + }; + etc = let inherit (pkgs) lib; diff --git a/pkgs/build-support/dotnet/build-dotnet-module/default.nix b/pkgs/build-support/dotnet/build-dotnet-module/default.nix index 686d89f8c11cb..af960fb1d617d 100644 --- a/pkgs/build-support/dotnet/build-dotnet-module/default.nix +++ b/pkgs/build-support/dotnet/build-dotnet-module/default.nix @@ -37,7 +37,7 @@ # The path to publish the project to. When unset, the directory "$out/lib/$pname" is used. , installPath ? null - # The binaries that should get installed to `$out/bin`, relative to `$out/lib/$pname/`. These get wrapped accordingly. + # The binaries that should get installed to `$out/bin`, relative to `$installPath/`. These get wrapped accordingly. # Unfortunately, dotnet has no method for doing this automatically. # If unset, all executables in the projects root will get installed. This may cause bloat! , executables ? null @@ -158,6 +158,22 @@ stdenvNoCC.mkDerivation (args // { dotnet-sdk ]; + # Parse the version attr into a format acceptable for the Version msbuild property + # The actual version attr is saved in InformationalVersion, which accepts an arbitrary string + versionForDotnet = if !(lib.hasAttr "version" args) || args.version == null + then null else let + components = lib.pipe args.version [ + lib.splitVersion + (lib.filter (x: (lib.strings.match "[0-9]+" x) != null)) + (lib.filter (x: (lib.toIntBase10 x) < 65535)) # one version component in dotnet has to fit in 16 bits + ]; + in if (lib.length components) == 0 + then null + else lib.concatStringsSep "." ((lib.take 4 components) + ++ (if (lib.length components) < 4 + then lib.replicate (4 - (lib.length components)) "0" + else [ ])); + makeWrapperArgs = args.makeWrapperArgs or [ ] ++ [ "--prefix LD_LIBRARY_PATH : ${dotnet-sdk.icu}/lib" ]; @@ -172,7 +188,7 @@ stdenvNoCC.mkDerivation (args // { passthru = { inherit nuget-source; - } // lib.optionalAttrs (nugetDepsFile != null) { + } // lib.optionalAttrs (!lib.isDerivation nugetDeps) { fetch-deps = let flags = dotnetFlags ++ dotnetRestoreFlags; diff --git a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-build-hook.sh b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-build-hook.sh index e9567b64cf2c9..0acfeced9b169 100644 --- a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-build-hook.sh +++ b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-build-hook.sh @@ -24,8 +24,13 @@ dotnetBuildHook() { dotnetBuildFlags+=("-p:UseAppHost=true") fi + local versionFlags=() if [ "${version-}" ]; then - local -r versionFlag="-p:Version=${version-}" + versionFlags+=("-p:InformationalVersion=${version-}") + fi + + if [ "${versionForDotnet-}" ]; then + versionFlags+=("-p:Version=${versionForDotnet-}") fi dotnetBuild() { @@ -43,7 +48,7 @@ dotnetBuildHook() { -p:Deterministic=true \ --configuration "@buildType@" \ --no-restore \ - ${versionFlag-} \ + ${versionFlags[@]} \ ${runtimeIdFlags[@]} \ ${dotnetBuildFlags[@]} \ ${dotnetFlags[@]} diff --git a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-fixup-hook.sh b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-fixup-hook.sh index 4fec939bed33f..97dd15c17dcf7 100644 --- a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-fixup-hook.sh +++ b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-fixup-hook.sh @@ -32,7 +32,7 @@ dotnetFixupHook() { if [ "${executables-}" ]; then for executable in ${executables[@]}; do - path="$out/lib/$pname/$executable" + path="${installPath-$out/lib/$pname}/$executable" if test -x "$path"; then wrapDotnetProgram "$path" "$out/bin/$(basename "$executable")" @@ -45,7 +45,7 @@ dotnetFixupHook() { else while IFS= read -d '' executable; do wrapDotnetProgram "$executable" "$out/bin/$(basename "$executable")" \; - done < <(find "$out/lib/$pname" ! -name "*.dll" -executable -type f -print0) + done < <(find "${installPath-$out/lib/$pname}" ! -name "*.dll" -executable -type f -print0) fi echo "Finished dotnetFixupPhase" diff --git a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-install-hook.sh b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-install-hook.sh index 3f2a89c414044..d832eac288096 100644 --- a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-install-hook.sh +++ b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-install-hook.sh @@ -30,7 +30,7 @@ dotnetInstallHook() { env dotnet publish ${project-} \ -p:ContinuousIntegrationBuild=true \ -p:Deterministic=true \ - --output "$out/lib/${pname}" \ + --output "${installPath-$out/lib/$pname}" \ --configuration "@buildType@" \ --no-build \ ${runtimeIdFlags[@]} \ diff --git a/pkgs/build-support/dotnet/make-nuget-source/default.nix b/pkgs/build-support/dotnet/make-nuget-source/default.nix index a23a143ab2467..4cf9c1a7412af 100644 --- a/pkgs/build-support/dotnet/make-nuget-source/default.nix +++ b/pkgs/build-support/dotnet/make-nuget-source/default.nix @@ -3,32 +3,26 @@ { name , description ? "" , deps ? [] -}: +, ... +}@args: -let - nuget-source = stdenvNoCC.mkDerivation rec { - inherit name; +stdenvNoCC.mkDerivation (lib.recursiveUpdate { + inherit name; - meta.description = description; - nativeBuildInputs = [ python3 ]; + nativeBuildInputs = [ python3 ]; - buildCommand = '' - mkdir -p $out/{lib,share} + buildCommand = '' + mkdir -p $out/{lib,share} - # use -L to follow symbolic links. When `projectReferences` is used in - # buildDotnetModule, one of the deps will be a symlink farm. - find -L ${lib.concatStringsSep " " deps} -type f -name '*.nupkg' -exec \ - cp --no-clobber '{}' $out/lib ';' + # use -L to follow symbolic links. When `projectReferences` is used in + # buildDotnetModule, one of the deps will be a symlink farm. + find -L ${lib.concatStringsSep " " deps} -type f -name '*.nupkg' -exec \ + ln -s '{}' -t $out/lib ';' - # Generates a list of all licenses' spdx ids, if available. - # Note that this currently ignores any license provided in plain text (e.g. "LICENSE.txt") - python ${./extract-licenses-from-nupkgs.py} $out/lib > $out/share/licenses - ''; - } // { # We need data from `$out` for `meta`, so we have to use overrides as to not hit infinite recursion. - meta.licence = let - depLicenses = lib.splitString "\n" (builtins.readFile "${nuget-source}/share/licenses"); - in (lib.flatten (lib.forEach depLicenses (spdx: - lib.optionals (spdx != "") (lib.getLicenseFromSpdxId spdx) - ))); - }; -in nuget-source + # Generates a list of all licenses' spdx ids, if available. + # Note that this currently ignores any license provided in plain text (e.g. "LICENSE.txt") + python ${./extract-licenses-from-nupkgs.py} $out/lib > $out/share/licenses + ''; + + meta.description = description; +} (removeAttrs args [ "name" "description" "deps" ])) diff --git a/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh b/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh index 86bc4482088b1..34c42929857d1 100755 --- a/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh +++ b/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh @@ -37,7 +37,7 @@ cd "$pkgs" for package in *; do cd "$package" for version in *; do - id=$(xq -r .package.metadata.id "$version/$package".nuspec) + id=$(xq -r .package.metadata.id "$version"/*.nuspec) if grep -qxF "$id.$version.nupkg" "$excluded_list"; then continue diff --git a/pkgs/build-support/emacs/elpa.nix b/pkgs/build-support/emacs/elpa.nix index f7027dc499d84..a43578fd3936b 100644 --- a/pkgs/build-support/emacs/elpa.nix +++ b/pkgs/build-support/emacs/elpa.nix @@ -2,7 +2,11 @@ { lib, stdenv, emacs, texinfo, writeText, gcc }: -with lib; +let + handledArgs = [ "files" "fileSpecs" "meta" ]; + genericBuild = import ./generic.nix { inherit lib stdenv emacs texinfo writeText gcc; }; + +in { pname , version @@ -11,15 +15,7 @@ with lib; , ... }@args: -let - - defaultMeta = { - homepage = args.src.meta.homepage or "https://elpa.gnu.org/packages/${pname}.html"; - }; - -in - -import ./generic.nix { inherit lib stdenv emacs texinfo writeText gcc; } ({ +genericBuild ({ dontUnpack = true; @@ -33,9 +29,9 @@ import ./generic.nix { inherit lib stdenv emacs texinfo writeText gcc; } ({ runHook postInstall ''; - meta = defaultMeta // meta; + meta = { + homepage = args.src.meta.homepage or "https://elpa.gnu.org/packages/${pname}.html"; + } // meta; } -// removeAttrs args [ "files" "fileSpecs" - "meta" - ]) +// removeAttrs args handledArgs) diff --git a/pkgs/build-support/emacs/generic.nix b/pkgs/build-support/emacs/generic.nix index 291f45d513b7f..bdf1cd4e50f31 100644 --- a/pkgs/build-support/emacs/generic.nix +++ b/pkgs/build-support/emacs/generic.nix @@ -2,6 +2,26 @@ { lib, stdenv, emacs, texinfo, writeText, gcc, ... }: +let + inherit (lib) optionalAttrs getLib; + handledArgs = [ "buildInputs" "packageRequires" "meta" ]; + + setupHook = writeText "setup-hook.sh" '' + source ${./emacs-funcs.sh} + + if [[ ! -v emacsHookDone ]]; then + emacsHookDone=1 + + # If this is for a wrapper derivation, emacs and the dependencies are all + # run-time dependencies. If this is for precompiling packages into bytecode, + # emacs is a compile-time dependency of the package. + addEnvHooks "$hostOffset" addEmacsVars + addEnvHooks "$targetOffset" addEmacsVars + fi + ''; + +in + { pname , version , buildInputs ? [] @@ -10,15 +30,6 @@ , ... }@args: -let - defaultMeta = { - broken = false; - platforms = emacs.meta.platforms; - } // lib.optionalAttrs ((args.src.meta.homepage or "") != "") { - homepage = args.src.meta.homepage; - }; -in - stdenv.mkDerivation (finalAttrs: ({ name = "emacs-${pname}-${finalAttrs.version}"; @@ -42,28 +53,21 @@ stdenv.mkDerivation (finalAttrs: ({ propagatedBuildInputs = packageRequires; propagatedUserEnvPkgs = packageRequires; - setupHook = writeText "setup-hook.sh" '' - source ${./emacs-funcs.sh} - - if [[ ! -v emacsHookDone ]]; then - emacsHookDone=1 - - # If this is for a wrapper derivation, emacs and the dependencies are all - # run-time dependencies. If this is for precompiling packages into bytecode, - # emacs is a compile-time dependency of the package. - addEnvHooks "$hostOffset" addEmacsVars - addEnvHooks "$targetOffset" addEmacsVars - fi - ''; + inherit setupHook; doCheck = false; - meta = defaultMeta // meta; + meta = { + broken = false; + platforms = emacs.meta.platforms; + } // optionalAttrs ((args.src.meta.homepage or "") != "") { + homepage = args.src.meta.homepage; + } // meta; } -// lib.optionalAttrs (emacs.withNativeCompilation or false) { +// optionalAttrs (emacs.withNativeCompilation or false) { - LIBRARY_PATH = "${lib.getLib stdenv.cc.libc}/lib"; + LIBRARY_PATH = "${getLib stdenv.cc.libc}/lib"; nativeBuildInputs = [ gcc ]; @@ -83,4 +87,4 @@ stdenv.mkDerivation (finalAttrs: ({ ''; } -// removeAttrs args [ "buildInputs" "packageRequires" "meta" ])) +// removeAttrs args handledArgs)) diff --git a/pkgs/build-support/emacs/melpa.nix b/pkgs/build-support/emacs/melpa.nix index 85bc8aa37b3aa..323c6e65d9d90 100644 --- a/pkgs/build-support/emacs/melpa.nix +++ b/pkgs/build-support/emacs/melpa.nix @@ -3,7 +3,30 @@ { lib, stdenv, fetchFromGitHub, emacs, texinfo, writeText, gcc }: -with lib; +let + genericBuild = import ./generic.nix { inherit lib stdenv emacs texinfo writeText gcc; }; + + packageBuild = stdenv.mkDerivation { + name = "package-build"; + src = fetchFromGitHub { + owner = "melpa"; + repo = "package-build"; + rev = "c48aa078c01b4f07b804270c4583a0a58ffea1c0"; + sha256 = "sha256-MzPj375upIiYXdQR+wWXv3A1zMqbSrZlH0taLuxx/1M="; + }; + + patches = [ ./package-build-dont-use-mtime.patch ]; + + dontConfigure = true; + dontBuild = true; + + installPhase = " + mkdir -p $out + cp -r * $out + "; + }; + +in { /* pname: Nix package name without special symbols and without version or @@ -20,44 +43,18 @@ with lib; , ... }@args: -let - - defaultMeta = { - homepage = args.src.meta.homepage or "https://melpa.org/#/${pname}"; - }; - -in - -import ./generic.nix { inherit lib stdenv emacs texinfo writeText gcc; } ({ +genericBuild ({ ename = if ename == null then pname else ename; - packageBuild = stdenv.mkDerivation { - name = "package-build"; - src = fetchFromGitHub { - owner = "melpa"; - repo = "package-build"; - rev = "c3c535e93d9dc92acd21ebc4b15016b5c3b90e7d"; - sha256 = "17z0wbqdd6fspbj43yq8biff6wfggk74xgnaf1xx6ynsp1i74is5"; - }; - - patches = [ ./package-build-dont-use-mtime.patch ]; - - dontConfigure = true; - dontBuild = true; - - installPhase = " - mkdir -p $out - cp -r * $out - "; - }; - elpa2nix = ./elpa2nix.el; melpa2nix = ./melpa2nix.el; + inherit packageBuild; + preUnpack = '' mkdir -p "$NIX_BUILD_TOP/recipes" if [ -n "$recipe" ]; then @@ -104,7 +101,9 @@ import ./generic.nix { inherit lib stdenv emacs texinfo writeText gcc; } ({ runHook postInstall ''; - meta = defaultMeta // meta; + meta = { + homepage = args.src.meta.homepage or "https://melpa.org/#/${pname}"; + } // meta; } // removeAttrs args [ "meta" ]) diff --git a/pkgs/build-support/emacs/melpa2nix.el b/pkgs/build-support/emacs/melpa2nix.el index 72667dea652c1..3de77dbf5e5c6 100644 --- a/pkgs/build-support/emacs/melpa2nix.el +++ b/pkgs/build-support/emacs/melpa2nix.el @@ -11,22 +11,22 @@ ;; Allow installing package tarfiles larger than 10MB (setq large-file-warning-threshold nil) -(defun melpa2nix-build-package-1 (rcp version commit) - (let ((source-dir (package-recipe--working-tree rcp))) +(defun melpa2nix-build-package-1 (rcp) + (let* ((default-directory (package-recipe--working-tree rcp))) (unwind-protect (let ((files (package-build-expand-files-spec rcp t))) - (cond - ((= (length files) 1) - (package-build--build-single-file-package - rcp version commit files source-dir)) - ((> (length files) 1) - (package-build--build-multi-file-package - rcp version commit files source-dir)) - (t (error "Unable to find files matching recipe patterns"))))))) + (unless files + (error "Unable to find files matching recipe patterns")) + (if (> (length files) 1) + (package-build--build-multi-file-package rcp files) + (package-build--build-single-file-package rcp files)))))) (defun melpa2nix-build-package () - (if (not noninteractive) - (error "`melpa2nix-build-package' is to be used only with -batch")) + (unless noninteractive + (error "`melpa2nix-build-package' is to be used only with -batch")) (pcase command-line-args-left (`(,package ,version ,commit) - (melpa2nix-build-package-1 (package-recipe-lookup package) version commit)))) + (let ((recipe (package-recipe-lookup package))) + (setf (oref recipe commit) commit) + (setf (oref recipe version) version) + (melpa2nix-build-package-1 recipe))))) diff --git a/pkgs/build-support/emacs/package-build-dont-use-mtime.patch b/pkgs/build-support/emacs/package-build-dont-use-mtime.patch index fe94de57a3002..1ace7771ea3ac 100644 --- a/pkgs/build-support/emacs/package-build-dont-use-mtime.patch +++ b/pkgs/build-support/emacs/package-build-dont-use-mtime.patch @@ -1,40 +1,21 @@ diff --git a/package-build.el b/package-build.el -index e572045..9eb0f82 100644 +index 29cdb61..c19be1b 100644 --- a/package-build.el +++ b/package-build.el -@@ -415,7 +415,7 @@ (defun package-build--write-pkg-file (desc dir) - (princ ";; Local Variables:\n;; no-byte-compile: t\n;; End:\n" - (current-buffer))))) - --(defun package-build--create-tar (name version directory mtime) -+(defun package-build--create-tar (name version directory) - "Create a tar file containing the contents of VERSION of package NAME. - DIRECTORY is a temporary directory that contains the directory - that is put in the tarball. MTIME is used as the modification -@@ -434,7 +434,7 @@ (defun package-build--create-tar (name version directory mtime) - ;; prevent a reproducable tarball as described at +@@ -923,7 +923,6 @@ DIRECTORY is a temporary directory that contains the directory + that is put in the tarball." + (let* ((name (oref rcp name)) + (version (oref rcp version)) +- (time (oref rcp time)) + (tar (expand-file-name (concat name "-" version ".tar") + package-build-archive-dir)) + (dir (concat name "-" version))) +@@ -939,7 +938,7 @@ that is put in the tarball." + ;; prevent a reproducible tarball as described at ;; https://reproducible-builds.org/docs/archives. "--sort=name" -- (format "--mtime=@%d" mtime) +- (format "--mtime=@%d" time) + "--mtime=@0" "--owner=0" "--group=0" "--numeric-owner" "--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime")) (when (and package-build-verbose noninteractive) -@@ -848,12 +848,11 @@ (defun package-build--build-multi-file-package (rcp version commit files source- - (package-build--desc-from-library - name version commit files 'tar) - (error "%s[-pkg].el matching package name is missing" -- name)))) -- (mtime (package-build--get-commit-time rcp commit))) -+ name))))) - (package-build--copy-package-files files source-dir target) - (package-build--write-pkg-file desc target) - (package-build--generate-info-files files source-dir target) -- (package-build--create-tar name version tmp-dir mtime) -+ (package-build--create-tar name version tmp-dir) - (package-build--write-pkg-readme name files source-dir) - (package-build--write-archive-entry desc)) - (delete-directory tmp-dir t nil)))) --- -2.37.2 - diff --git a/pkgs/build-support/emacs/trivial.nix b/pkgs/build-support/emacs/trivial.nix index abe4d761c6b5c..11c28c0133e40 100644 --- a/pkgs/build-support/emacs/trivial.nix +++ b/pkgs/build-support/emacs/trivial.nix @@ -2,8 +2,6 @@ { callPackage, lib, ... }@envargs: -with lib; - args: callPackage ./generic.nix envargs ({ diff --git a/pkgs/build-support/emacs/wrapper.nix b/pkgs/build-support/emacs/wrapper.nix index 6f46bb692a43e..59a694286d09e 100644 --- a/pkgs/build-support/emacs/wrapper.nix +++ b/pkgs/build-support/emacs/wrapper.nix @@ -32,12 +32,12 @@ in customEmacsPackages.withPackages (epkgs: [ epkgs.evil epkgs.magit ]) */ -{ lib, lndir, makeWrapper, runCommand, gcc }: +{ lib, lndir, makeBinaryWrapper, runCommand, gcc }: self: let inherit (self) emacs; - withNativeCompilation = emacs.withNativeCompilation or emacs.nativeComp or false; - withTreeSitter = emacs.withTreeSitter or emacs.treeSitter or false; + withNativeCompilation = emacs.withNativeCompilation or false; + withTreeSitter = emacs.withTreeSitter or false; in packagesFun: # packages explicitly requested by the user let @@ -50,7 +50,7 @@ runCommand (lib.appendToName "with-packages" emacs).name { inherit emacs explicitRequires; - nativeBuildInputs = [ emacs lndir makeWrapper ]; + nativeBuildInputs = [ emacs lndir makeBinaryWrapper ]; preferLocalBuild = true; allowSubstitutes = false; @@ -201,6 +201,11 @@ runCommand --subst-var-by wrapperSiteLispNative "$deps/share/emacs/native-lisp" \ --subst-var prog chmod +x $out/bin/$progname + # Create a “NOP” binary wrapper for the pure sake of it becoming a + # non-shebang, actual binary. See the makeBinaryWrapper docs for rationale + # (summary: it allows you to use emacs as a shebang itself on Darwin, + # e.g. #!$ {emacs}/bin/emacs --script) + wrapProgramBinary $out/bin/$progname done # Wrap MacOS app @@ -220,6 +225,7 @@ runCommand --subst-var-by wrapperSiteLispNative "$deps/share/emacs/native-lisp" \ --subst-var-by prog "$emacs/Applications/Emacs.app/Contents/MacOS/Emacs" chmod +x $out/Applications/Emacs.app/Contents/MacOS/Emacs + wrapProgramBinary $out/Applications/Emacs.app/Contents/MacOS/Emacs fi mkdir -p $out/share diff --git a/pkgs/build-support/expand-response-params/default.nix b/pkgs/build-support/expand-response-params/default.nix index 9371b77023626..7ce15e98c8d99 100644 --- a/pkgs/build-support/expand-response-params/default.nix +++ b/pkgs/build-support/expand-response-params/default.nix @@ -1,5 +1,12 @@ { stdenv }: +# A "response file" is a sequence of arguments that is passed via a +# file, rather than via argv[]. + +# For more information see: +# https://gcc.gnu.org/wiki/Response_Files +# https://www.intel.com/content/www/us/en/docs/dpcpp-cpp-compiler/developer-guide-reference/2023-0/use-response-files.html + stdenv.mkDerivation { name = "expand-response-params"; src = ./expand-response-params.c; diff --git a/pkgs/build-support/fetchbzr/builder.sh b/pkgs/build-support/fetchbzr/builder.sh index 991864719a073..4515e0e82f497 100644 --- a/pkgs/build-support/fetchbzr/builder.sh +++ b/pkgs/build-support/fetchbzr/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source "$stdenv/setup" echo "exporting \`$url' (revision $rev) into \`$out'" diff --git a/pkgs/build-support/fetchcvs/builder.sh b/pkgs/build-support/fetchcvs/builder.sh index 4b49e9676ec02..7e4dde4a64c8b 100644 --- a/pkgs/build-support/fetchcvs/builder.sh +++ b/pkgs/build-support/fetchcvs/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup (echo "#!$SHELL"; \ diff --git a/pkgs/build-support/fetchdarcs/builder.sh b/pkgs/build-support/fetchdarcs/builder.sh index 75b9066dba6f4..39838db255a6e 100644 --- a/pkgs/build-support/fetchdarcs/builder.sh +++ b/pkgs/build-support/fetchdarcs/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup tagtext="" diff --git a/pkgs/build-support/fetchdocker/credentials.nix b/pkgs/build-support/fetchdocker/credentials.nix index da19848326840..f8a229ccb6bb1 100644 --- a/pkgs/build-support/fetchdocker/credentials.nix +++ b/pkgs/build-support/fetchdocker/credentials.nix @@ -1,3 +1,4 @@ +{ lib }: # We provide three paths to get the credentials into the builder's # environment: # diff --git a/pkgs/build-support/fetchdocker/fetchdocker-builder.sh b/pkgs/build-support/fetchdocker/fetchdocker-builder.sh index 4eb70f672d488..489914a2a8b43 100644 --- a/pkgs/build-support/fetchdocker/fetchdocker-builder.sh +++ b/pkgs/build-support/fetchdocker/fetchdocker-builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source "${stdenv}/setup" echo "exporting ${repository}/${imageName} (tag: ${tag}) into ${out}" mkdir -p "${out}" diff --git a/pkgs/build-support/fetchdocker/generic-fetcher.nix b/pkgs/build-support/fetchdocker/generic-fetcher.nix index 6a7b977db29f8..95b193490a82d 100644 --- a/pkgs/build-support/fetchdocker/generic-fetcher.nix +++ b/pkgs/build-support/fetchdocker/generic-fetcher.nix @@ -1,7 +1,7 @@ { stdenv, lib, haskellPackages, writeText, gawk }: let awk = "${gawk}/bin/awk"; - dockerCredentialsFile = import ./credentials.nix; + dockerCredentialsFile = import ./credentials.nix { inherit lib; }; in { fetcher , name diff --git a/pkgs/build-support/fetchfossil/builder.sh b/pkgs/build-support/fetchfossil/builder.sh index 36b758ab574e4..fe828d0ada9d0 100644 --- a/pkgs/build-support/fetchfossil/builder.sh +++ b/pkgs/build-support/fetchfossil/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup echo "Cloning Fossil $url [$rev] into $out" diff --git a/pkgs/build-support/fetchfossil/default.nix b/pkgs/build-support/fetchfossil/default.nix index 7866c403ec425..3f3bf69db0477 100644 --- a/pkgs/build-support/fetchfossil/default.nix +++ b/pkgs/build-support/fetchfossil/default.nix @@ -1,7 +1,15 @@ {stdenv, lib, fossil, cacert}: -{name ? null, url, rev, sha256}: +{ name ? null +, url +, rev +, sha256 ? "" +, hash ? "" +}: +if hash != "" && sha256 != "" then + throw "Only one of sha256 or hash can be set" +else stdenv.mkDerivation { name = "fossil-archive" + (lib.optionalString (name != null) "-${name}"); builder = ./builder.sh; @@ -11,9 +19,14 @@ stdenv.mkDerivation { # https://www.fossil-scm.org/index.html/doc/trunk/www/env-opts.md impureEnvVars = [ "http_proxy" ]; - outputHashAlgo = "sha256"; + outputHashAlgo = if hash != "" then null else "sha256"; outputHashMode = "recursive"; - outputHash = sha256; + outputHash = if hash != "" then + hash + else if sha256 != "" then + sha256 + else + lib.fakeSha256; inherit url rev; preferLocalBuild = true; diff --git a/pkgs/build-support/fetchgit/builder.sh b/pkgs/build-support/fetchgit/builder.sh index 77f6381b09ab2..95176beea8c13 100644 --- a/pkgs/build-support/fetchgit/builder.sh +++ b/pkgs/build-support/fetchgit/builder.sh @@ -3,7 +3,7 @@ # - revision specified and remote has a HEAD # - revision specified and remote without HEAD # -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup echo "exporting $url (rev $rev) into $out" diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 1d06ce44a91ea..66bb3b7c09ffc 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -90,6 +90,7 @@ stdenvNoCC.mkDerivation { ${netrcPhase} # required that git uses the netrc file mv {,.}netrc + export NETRC=$PWD/.netrc export HOME=$PWD ''; diff --git a/pkgs/build-support/fetchgit/nix-prefetch-git b/pkgs/build-support/fetchgit/nix-prefetch-git index 2a53fd94e7f2b..a3ea0297673da 100755 --- a/pkgs/build-support/fetchgit/nix-prefetch-git +++ b/pkgs/build-support/fetchgit/nix-prefetch-git @@ -207,7 +207,10 @@ checkout_ref(){ # Update submodules init_submodules(){ - clean_git submodule update --init --recursive -j ${NIX_BUILD_CORES:-1} + # shallow with leaveDotGit will change hashes + [[ -z "$deepClone" ]] && [[ -z "$leaveDotGit" ]] && \ + clean_git submodule update --init --recursive -j ${NIX_BUILD_CORES:-1} --progress --depth 1 || \ + clean_git submodule update --init --recursive -j ${NIX_BUILD_CORES:-1} --progress } clone(){ @@ -257,9 +260,15 @@ make_deterministic_repo(){ cd "$repo" # Remove files that contain timestamps or otherwise have non-deterministic # properties. - rm -rf .git/logs/ .git/hooks/ .git/index .git/FETCH_HEAD .git/ORIG_HEAD \ - .git/refs/remotes/origin/HEAD .git/config - + if [ -f .git ]; then + local dotgit_content=$(<.git) + local dotgit_dir="${dotgit_content#gitdir: }" + else + local dotgit_dir=".git" + fi + pushd "$dotgit_dir" >/dev/null + rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config + popd >/dev/null # Remove all remote branches. git branch -r | while read -r branch; do clean_git branch -rD "$branch" @@ -277,7 +286,7 @@ make_deterministic_repo(){ # Do a full repack. Must run single-threaded, or else we lose determinism. clean_git config pack.threads 1 clean_git repack -A -d -f - rm -f .git/config + rm -f "$dotgit_dir/config" # Garbage collect unreferenced objects. # Note: --keep-largest-pack prevents non-deterministic ordering of packs @@ -293,9 +302,6 @@ clone_user_rev() { local rev="${3:-HEAD}" if [ -n "$fetchLFS" ]; then - tmpHomePath="$(mktemp -d "${TMPDIR:-/tmp}/nix-prefetch-git-tmp-home-XXXXXXXXXX")" - exit_handlers+=(remove_tmpHomePath) - HOME="$tmpHomePath" clean_git lfs install fi @@ -326,7 +332,7 @@ clone_user_rev() { find "$dir" -name .git -print0 | xargs -0 rm -rf else find "$dir" -name .git | while read -r gitdir; do - make_deterministic_repo "$(readlink -f "$gitdir/..")" + make_deterministic_repo "$(readlink -f "$(dirname "$gitdir")")" done fi } @@ -417,6 +423,13 @@ if test -z "$branchName"; then branchName=fetchgit fi +tmpHomePath="$(mktemp -d "${TMPDIR:-/tmp}/nix-prefetch-git-tmp-home-XXXXXXXXXX")" +exit_handlers+=(remove_tmpHomePath) +ln -s "${NETRC:-$HOME/.netrc}" "$tmpHomePath/.netrc" +HOME="$tmpHomePath" +unset XDG_CONFIG_HOME +export GIT_CONFIG_NOSYSTEM=1 + if test -n "$builder"; then test -n "$out" -a -n "$url" -a -n "$rev" || usage mkdir -p "$out" diff --git a/pkgs/build-support/fetchgit/tests.nix b/pkgs/build-support/fetchgit/tests.nix index a18be65327b5d..23e5fa299010d 100644 --- a/pkgs/build-support/fetchgit/tests.nix +++ b/pkgs/build-support/fetchgit/tests.nix @@ -1,15 +1,13 @@ -{ testers, fetchgit, ... }: - -{ +{ testers, fetchgit, ... }: { simple = testers.invalidateFetcherByDrvHash fetchgit { - name = "nix-source"; + name = "simple-nix-source"; url = "https://github.com/NixOS/nix"; rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; }; sparseCheckout = testers.invalidateFetcherByDrvHash fetchgit { - name = "nix-source"; + name = "sparse-checkout-nix-source"; url = "https://github.com/NixOS/nix"; rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; sparseCheckout = [ @@ -20,7 +18,7 @@ }; sparseCheckoutNonConeMode = testers.invalidateFetcherByDrvHash fetchgit { - name = "nix-source"; + name = "sparse-checkout-non-cone-nix-source"; url = "https://github.com/NixOS/nix"; rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; sparseCheckout = [ @@ -30,4 +28,48 @@ nonConeMode = true; sha256 = "sha256-FknO6C/PSnMPfhUqObD4vsW4PhkwdmPa9blNzcNvJQ4="; }; + + leave-git = testers.invalidateFetcherByDrvHash fetchgit { + name = "leave-git-nix-source"; + url = "https://github.com/NixOS/nix"; + rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; + sha256 = "sha256-zZxDxqaeWvuWuzwPizBLR7d59zP24+zqnWllNICenko="; + leaveDotGit = true; + }; + + submodule-simple = testers.invalidateFetcherByDrvHash fetchgit { + name = "submodule-simple-source"; + url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule"; + rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a"; + sha256 = "sha256-rmP8PQT0wJBopdtr/hsB7Y/L1G+ZPdHC2r9LB05Qrj4="; + fetchSubmodules = true; + }; + + submodule-leave-git = testers.invalidateFetcherByDrvHash fetchgit { + name = "submodule-leave-git-source"; + url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule"; + rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a"; + sha256 = "sha256-+uXIClcRJ4S1rdgx2Oyww+Jv4h1VXp8tfeh9lb07Fhk="; + leaveDotGit = true; + fetchSubmodules = true; + }; + + submodule-deep = testers.invalidateFetcherByDrvHash fetchgit { + name = "submodule-deep-source"; + url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule"; + rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a"; + sha256 = "sha256-LL7uhXQk3N3DcvBBxwjmfVx55tTXCGQ19T91tknopzw="; + deepClone = true; + fetchSubmodules = true; + }; + + submodule-leave-git-deep = testers.invalidateFetcherByDrvHash fetchgit { + name = "submodule-leave-git-deep-source"; + url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule"; + rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a"; + sha256 = "sha256-LL7uhXQk3N3DcvBBxwjmfVx55tTXCGQ19T91tknopzw="; + deepClone = true; + leaveDotGit = true; + fetchSubmodules = true; + }; } diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index a2498700b545d..4ce3c6e84d768 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -19,7 +19,7 @@ let baseUrl = "https://${githubBase}/${owner}/${repo}"; newMeta = meta // { homepage = meta.homepage or baseUrl; - + } // lib.optionalAttrs (position != null) { # to indicate where derivation originates, similar to make-derivation.nix's mkDerivation position = "${position.file}:${toString position.line}"; }; @@ -28,7 +28,11 @@ let useFetchGit = fetchSubmodules || (leaveDotGit == true) || deepClone || forceFetchGit || (sparseCheckout != []); # We prefer fetchzip in cases we don't need submodules as the hash # is more stable in that case. - fetcher = if useFetchGit then fetchgit else fetchzip; + fetcher = + if useFetchGit then fetchgit + # fetchzip may not be overridable when using external tools, for example nix-prefetch + else if fetchzip ? override then fetchzip.override { withUnzip = false; } + else fetchzip; privateAttrs = lib.optionalAttrs private { netrcPhase = '' if [ -z "''$${varBase}USERNAME" -o -z "''$${varBase}PASSWORD" ]; then diff --git a/pkgs/build-support/fetchgitlab/default.nix b/pkgs/build-support/fetchgitlab/default.nix index 146845e06a71c..749883f2365eb 100644 --- a/pkgs/build-support/fetchgitlab/default.nix +++ b/pkgs/build-support/fetchgitlab/default.nix @@ -1,9 +1,11 @@ -{ fetchgit, fetchzip, lib }: +{ lib, fetchgit, fetchzip }: lib.makeOverridable ( # gitlab example { owner, repo, rev, protocol ? "https", domain ? "gitlab.com", name ? "source", group ? null -, fetchSubmodules ? false, leaveDotGit ? false, deepClone ? false +, fetchSubmodules ? false, leaveDotGit ? false +, deepClone ? false, forceFetchGit ? false +, sparseCheckout ? [] , ... # For hash agility } @ args: @@ -11,15 +13,15 @@ let slug = lib.concatStringsSep "/" ((lib.optional (group != null) group) ++ [ owner repo ]); escapedSlug = lib.replaceStrings [ "." "/" ] [ "%2E" "%2F" ] slug; escapedRev = lib.replaceStrings [ "+" "%" "/" ] [ "%2B" "%25" "%2F" ] rev; - passthruAttrs = removeAttrs args [ "protocol" "domain" "owner" "group" "repo" "rev" "fetchSubmodules" "leaveDotGit" "deepClone" ]; + passthruAttrs = removeAttrs args [ "protocol" "domain" "owner" "group" "repo" "rev" "fetchSubmodules" "forceFetchGit" "leaveDotGit" "deepClone" ]; - useFetchGit = deepClone || fetchSubmodules || leaveDotGit; + useFetchGit = fetchSubmodules || leaveDotGit || deepClone || forceFetchGit || (sparseCheckout != []); fetcher = if useFetchGit then fetchgit else fetchzip; gitRepoUrl = "${protocol}://${domain}/${slug}.git"; fetcherArgs = (if useFetchGit then { - inherit rev deepClone fetchSubmodules leaveDotGit; + inherit rev deepClone fetchSubmodules sparseCheckout leaveDotGit; url = gitRepoUrl; } else { url = "${protocol}://${domain}/api/v4/projects/${escapedSlug}/repository/archive.tar.gz?sha=${escapedRev}"; @@ -30,5 +32,5 @@ let }) // passthruAttrs // { inherit name; }; in -fetcher fetcherArgs // { meta.homepage = "${protocol}://${domain}/${slug}/"; inherit rev; } +fetcher fetcherArgs // { meta.homepage = "${protocol}://${domain}/${slug}/"; inherit rev owner repo; } ) diff --git a/pkgs/build-support/fetchhg/builder.sh b/pkgs/build-support/fetchhg/builder.sh index 1ce294757713c..20dfde4b10d44 100644 --- a/pkgs/build-support/fetchhg/builder.sh +++ b/pkgs/build-support/fetchhg/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup echo "getting $url${rev:+ ($rev)} into $out" diff --git a/pkgs/build-support/fetchipfs/builder.sh b/pkgs/build-support/fetchipfs/builder.sh index ca77962b53847..1ddd111b15182 100644 --- a/pkgs/build-support/fetchipfs/builder.sh +++ b/pkgs/build-support/fetchipfs/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup # Curl flags to handle redirects, not use EPSV, handle cookies for diff --git a/pkgs/build-support/fetchmtn/builder.sh b/pkgs/build-support/fetchmtn/builder.sh index 1aabd7949ee1d..de929fad55a92 100644 --- a/pkgs/build-support/fetchmtn/builder.sh +++ b/pkgs/build-support/fetchmtn/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup set -x diff --git a/pkgs/build-support/fetchpijul/default.nix b/pkgs/build-support/fetchpijul/default.nix index ca7e1a7926e80..fd41cfa55355c 100644 --- a/pkgs/build-support/fetchpijul/default.nix +++ b/pkgs/build-support/fetchpijul/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenvNoCC, pijul }: +{ lib, stdenvNoCC, pijul, cacert }: lib.makeOverridable ( { url @@ -17,7 +17,8 @@ if change != null && state != null then else stdenvNoCC.mkDerivation { inherit name; - nativeBuildInputs = [ pijul ]; + nativeBuildInputs = [ pijul cacert ]; + strictDeps = true; dontUnpack = true; dontConfigure = true; @@ -52,5 +53,7 @@ else lib.fakeSha256; inherit url change state channel; + + impureEnvVars = lib.fetchers.proxyImpureEnvVars; } ) diff --git a/pkgs/build-support/fetchsvn/builder.sh b/pkgs/build-support/fetchsvn/builder.sh index aa4d049aba431..e187747f14e6a 100644 --- a/pkgs/build-support/fetchsvn/builder.sh +++ b/pkgs/build-support/fetchsvn/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup echo "exporting $url (r$rev) into $out" diff --git a/pkgs/build-support/fetchsvnssh/builder.sh b/pkgs/build-support/fetchsvnssh/builder.sh index 5782151524f7a..e553446346de3 100644 --- a/pkgs/build-support/fetchsvnssh/builder.sh +++ b/pkgs/build-support/fetchsvnssh/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup echo "exporting $url (r$rev) into $out" diff --git a/pkgs/build-support/fetchtorrent/default.nix b/pkgs/build-support/fetchtorrent/default.nix new file mode 100644 index 0000000000000..126748678bf25 --- /dev/null +++ b/pkgs/build-support/fetchtorrent/default.nix @@ -0,0 +1,62 @@ +{ lib, runCommand, transmission_noSystemd, rqbit, writeShellScript, formats, cacert, rsync }: +let + urlRegexp = ''.*xt=urn:bt[im]h:([^&]{64}|[^&]{40}).*''; +in +{ url +, name ? + if (builtins.match urlRegexp url) == null then + "bittorrent" + else + "bittorrent-" + builtins.head (builtins.match urlRegexp url) +, config ? if (backend == "transmission") then { } else throw "json config for configuring fetchFromBitorrent only works with the transmission backend" +, hash +, backend ? "transmission" +, recursiveHash ? true +, postFetch ? "" +, postUnpack ? "" +, meta ? {} +}: +let + afterSuccess = writeShellScript "fetch-bittorrent-done.sh" '' + ${postUnpack} + # Flatten the directory, so that only the torrent contents are in $out, not + # the folder name + shopt -s dotglob + mv -v $downloadedDirectory/*/* $out + rm -v -rf $downloadedDirectory + unset downloadedDirectory + ${postFetch} + kill $PPID + ''; + jsonConfig = (formats.json {}).generate "jsonConfig" config; +in +runCommand name { + inherit meta; + nativeBuildInputs = [ cacert ] ++ (if (backend == "transmission" ) then [ transmission_noSystemd ] else if (backend == "rqbit") then [ rqbit ] else throw "rqbit or transmission are the only available backends for fetchtorrent"); + outputHashAlgo = if hash != "" then null else "sha256"; + outputHash = hash; + outputHashMode = if recursiveHash then "recursive" else "flat"; + + # url will be written to the derivation, meaning it can be parsed and utilized + # by external tools, such as tools that may want to seed fetchtorrent calls + # in nixpkgs + inherit url; +} +(if (backend == "transmission") then '' + export HOME=$TMP + export downloadedDirectory=$out/downloadedDirectory + mkdir -p $downloadedDirectory + mkdir -p $HOME/.config/transmission + cp ${jsonConfig} $HOME/.config/transmission/settings.json + function handleChild { + # This detects failures and logs the contents of the transmission fetch + find $out + exit 0 + } + trap handleChild CHLD + transmission-cli --port $(shuf -n 1 -i 49152-65535) --portmap --finish ${afterSuccess} --download-dir $downloadedDirectory --config-dir "$HOME"/.config/transmission "$url" +'' else +'' + export HOME=$TMP + rqbit --disable-dht-persistence --http-api-listen-addr "127.0.0.1:$(shuf -n 1 -i 49152-65535)" download -o $out --exit-on-finish "$url" +'') diff --git a/pkgs/build-support/fetchtorrent/tests.nix b/pkgs/build-support/fetchtorrent/tests.nix new file mode 100644 index 0000000000000..e8cc3f33878bf --- /dev/null +++ b/pkgs/build-support/fetchtorrent/tests.nix @@ -0,0 +1,47 @@ +{ testers, fetchtorrent, lib, ... }: + +let + wired-cd.meta.license = [ + # track 1, 4 and 11 + { + spdxID = "CC NC-SAMPLING+ 1.0 Deed"; + fullName = "NonCommercial Sampling Plus 1.0 Generic"; + url = "https://creativecommons.org/licenses/nc-sampling+/1.0/"; + free = false; # for noncommercial purposes only + } + # the rest + { + spdxID = "CC SAMPLING+ 1.0 Deed"; + fullName = "Sampling Plus 1.0 Generic"; + url = "https://creativecommons.org/licenses/sampling+/1.0/"; + free = true; # no use in advertisement + } + ]; +in + +{ + http-link = testers.invalidateFetcherByDrvHash fetchtorrent { + url = "https://webtorrent.io/torrents/wired-cd.torrent"; + hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo="; + backend = "transmission"; + inherit (wired-cd) meta; + }; + magnet-link = testers.invalidateFetcherByDrvHash fetchtorrent { + url = "magnet:?xt=urn:btih:a88fda5954e89178c372716a6a78b8180ed4dad3&dn=The+WIRED+CD+-+Rip.+Sample.+Mash.+Share&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fwired-cd.torrent"; + hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo="; + backend = "transmission"; + inherit (wired-cd) meta; + }; + http-link-rqbit = testers.invalidateFetcherByDrvHash fetchtorrent { + url = "https://webtorrent.io/torrents/wired-cd.torrent"; + hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo="; + backend = "rqbit"; + inherit (wired-cd) meta; + }; + magnet-link-rqbit = testers.invalidateFetcherByDrvHash fetchtorrent { + url = "magnet:?xt=urn:btih:a88fda5954e89178c372716a6a78b8180ed4dad3&dn=The+WIRED+CD+-+Rip.+Sample.+Mash.+Share&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fwired-cd.torrent"; + hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo="; + backend = "rqbit"; + inherit (wired-cd) meta; + }; +} diff --git a/pkgs/build-support/fetchurl/builder.sh b/pkgs/build-support/fetchurl/builder.sh index e8eaba934858e..a82728ef1025c 100644 --- a/pkgs/build-support/fetchurl/builder.sh +++ b/pkgs/build-support/fetchurl/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup source $mirrorsFile diff --git a/pkgs/build-support/fetchurl/mirrors.nix b/pkgs/build-support/fetchurl/mirrors.nix index 2546b8e6dc99c..af0468b3e4944 100644 --- a/pkgs/build-support/fetchurl/mirrors.nix +++ b/pkgs/build-support/fetchurl/mirrors.nix @@ -125,6 +125,7 @@ # Mirrors from https://download.kde.org/ls-lR.mirrorlist kde = [ + "https://cdn.download.kde.org/" "https://download.kde.org/download.php?url=" "https://ftp.gwdg.de/pub/linux/kde/" "https://mirrors.ocf.berkeley.edu/kde/" diff --git a/pkgs/build-support/fetchzip/default.nix b/pkgs/build-support/fetchzip/default.nix index 0446851d64098..dd04ccb6e0931 100644 --- a/pkgs/build-support/fetchzip/default.nix +++ b/pkgs/build-support/fetchzip/default.nix @@ -5,7 +5,7 @@ # (e.g. due to minor changes in the compression algorithm, or changes # in timestamps). -{ lib, fetchurl, unzip, glibcLocalesUtf8 }: +{ lib, fetchurl, withUnzip ? true, unzip, glibcLocalesUtf8 }: { name ? "source" , url ? "" @@ -24,7 +24,7 @@ # the rest are given to fetchurl as is , ... } @ args: -assert (extraPostFetch != "") -> lib.warn "use 'postFetch' instead of 'extraPostFetch' with 'fetchzip' and 'fetchFromGitHub'." true; +assert (extraPostFetch != "") -> lib.warn "use 'postFetch' instead of 'extraPostFetch' with 'fetchzip' and 'fetchFromGitHub' or 'fetchFromGitLab'." true; let tmpFilename = @@ -42,7 +42,7 @@ fetchurl ({ # Have to pull in glibcLocalesUtf8 for unzip in setup-hook.sh to handle # UTF-8 aware locale: # https://github.com/NixOS/nixpkgs/issues/176225#issuecomment-1146617263 - nativeBuildInputs = [ unzip glibcLocalesUtf8 ] ++ nativeBuildInputs; + nativeBuildInputs = lib.optionals withUnzip [ unzip glibcLocalesUtf8 ] ++ nativeBuildInputs; postFetch = '' diff --git a/pkgs/build-support/flutter/default.nix b/pkgs/build-support/flutter/default.nix index c109804680f3e..5d7cd7d984c17 100644 --- a/pkgs/build-support/flutter/default.nix +++ b/pkgs/build-support/flutter/default.nix @@ -1,182 +1,179 @@ { lib , callPackage -, stdenvNoCC , runCommand , makeWrapper -, llvmPackages_13 +, wrapGAppsHook +, buildDartApplication , cacert +, glib , flutter +, pkg-config , jq +, yq +, moreutils }: # absolutely no mac support for now { pubGetScript ? "flutter pub get" , flutterBuildFlags ? [ ] -, runtimeDependencies ? [ ] -, customPackageOverrides ? { } -, autoDepsList ? false -, depsListFile ? null -, vendorHash ? "" -, pubspecLockFile ? null -, nativeBuildInputs ? [ ] -, preUnpack ? "" -, postFixup ? "" +, targetFlutterPlatform ? "linux" , extraWrapProgramArgs ? "" , ... }@args: + let - flutterSetupScript = '' - # Pub needs SSL certificates. Dart normally looks in a hardcoded path. - # https://github.com/dart-lang/sdk/blob/3.1.0/runtime/bin/security_context_linux.cc#L48 - # - # Dart does not respect SSL_CERT_FILE... - # https://github.com/dart-lang/sdk/issues/48506 - # ...and Flutter does not support --root-certs-file, so the path cannot be manually set. - # https://github.com/flutter/flutter/issues/56607 - # https://github.com/flutter/flutter/issues/113594 - # - # libredirect is of no use either, as Flutter does not pass any - # environment variables (including LD_PRELOAD) to the Pub process. - # - # Instead, Flutter is patched to allow the path to the Dart binary used for - # Pub commands to be overriden. - export NIX_FLUTTER_PUB_DART="${runCommand "dart-with-certs" { nativeBuildInputs = [ makeWrapper ]; } '' - mkdir -p "$out/bin" - makeWrapper ${flutter.dart}/bin/dart "$out/bin/dart" \ - --add-flags "--root-certs-file=${cacert}/etc/ssl/certs/ca-bundle.crt" - ''}/bin/dart" - - export HOME="$NIX_BUILD_TOP" - flutter config --no-analytics &>/dev/null # mute first-run - flutter config --enable-linux-desktop >/dev/null - ''; - - deps = callPackage ../dart/fetch-dart-deps { dart = flutter; } { - sdkSetupScript = flutterSetupScript; - inherit pubGetScript vendorHash pubspecLockFile; - buildDrvArgs = args; - }; - - baseDerivation = llvmPackages_13.stdenv.mkDerivation (finalAttrs: args // { - inherit flutterBuildFlags runtimeDependencies; - - outputs = [ "out" "debug" ]; - - nativeBuildInputs = [ - makeWrapper - deps - flutter - jq - ] ++ nativeBuildInputs; - - preUnpack = '' - ${lib.optionalString (!autoDepsList) '' - if ! { [ '${lib.boolToString (depsListFile != null)}' = 'true' ] ${lib.optionalString (depsListFile != null) "&& cmp -s <(jq -Sc . '${depsListFile}') <(jq -Sc . '${finalAttrs.passthru.depsListFile}')"}; }; then - echo 1>&2 -e '\nThe dependency list file was either not given or differs from the expected result.' \ - '\nPlease choose one of the following solutions:' \ - '\n - Duplicate the following file and pass it to the depsListFile argument.' \ - '\n ${finalAttrs.passthru.depsListFile}' \ - '\n - Set autoDepsList to true (not supported by Hydra or permitted in Nixpkgs)'. - exit 1 + builderArgs = rec { + universal = args // { + sdkSetupScript = '' + # Pub needs SSL certificates. Dart normally looks in a hardcoded path. + # https://github.com/dart-lang/sdk/blob/3.1.0/runtime/bin/security_context_linux.cc#L48 + # + # Dart does not respect SSL_CERT_FILE... + # https://github.com/dart-lang/sdk/issues/48506 + # ...and Flutter does not support --root-certs-file, so the path cannot be manually set. + # https://github.com/flutter/flutter/issues/56607 + # https://github.com/flutter/flutter/issues/113594 + # + # libredirect is of no use either, as Flutter does not pass any + # environment variables (including LD_PRELOAD) to the Pub process. + # + # Instead, Flutter is patched to allow the path to the Dart binary used for + # Pub commands to be overriden. + export NIX_FLUTTER_PUB_DART="${runCommand "dart-with-certs" { nativeBuildInputs = [ makeWrapper ]; } '' + mkdir -p "$out/bin" + makeWrapper ${flutter.dart}/bin/dart "$out/bin/dart" \ + --add-flags "--root-certs-file=${cacert}/etc/ssl/certs/ca-bundle.crt" + ''}/bin/dart" + + export HOME="$NIX_BUILD_TOP" + flutter config --no-analytics &>/dev/null # mute first-run + flutter config --enable-linux-desktop >/dev/null + ''; + + inherit pubGetScript; + + sdkSourceBuilders = { + # https://github.com/dart-lang/pub/blob/68dc2f547d0a264955c1fa551fa0a0e158046494/lib/src/sdk/flutter.dart#L81 + "flutter" = name: runCommand "flutter-sdk-${name}" { passthru.packageRoot = "."; } '' + for path in '${flutter}/packages/${name}' '${flutter}/bin/cache/pkg/${name}'; do + if [ -d "$path" ]; then + ln -s "$path" "$out" + break + fi + done + + if [ ! -e "$out" ]; then + echo 1>&2 'The Flutter SDK does not contain the requested package: ${name}!' + exit 1 + fi + ''; + }; + + extraPackageConfigSetup = '' + # https://github.com/flutter/flutter/blob/3.13.8/packages/flutter_tools/lib/src/dart/pub.dart#L755 + if [ "$('${yq}/bin/yq' '.flutter.generate // false' pubspec.yaml)" = "true" ]; then + '${jq}/bin/jq' '.packages |= . + [{ + name: "flutter_gen", + rootUri: "flutter_gen", + languageVersion: "2.12", + }]' "$out" | '${moreutils}/bin/sponge' "$out" fi - ''} + ''; + }; - ${preUnpack} - ''; + linux = universal // { + outputs = universal.outputs or [ ] ++ [ "debug" ]; - configurePhase = '' - runHook preConfigure + nativeBuildInputs = (universal.nativeBuildInputs or [ ]) ++ [ + wrapGAppsHook - ${flutterSetupScript} + # Flutter requires pkg-config for Linux desktop support, and many plugins + # attempt to use it. + # + # It is available to the `flutter` tool through its wrapper, but it must be + # added here as well so the setup hook adds plugin dependencies to the + # pkg-config search paths. + pkg-config + ]; - runHook postConfigure - ''; + buildInputs = (universal.buildInputs or [ ]) ++ [ glib ]; - buildPhase = '' - runHook preBuild + dontDartBuild = true; + buildPhase = universal.buildPhase or '' + runHook preBuild - mkdir -p build/flutter_assets/fonts + mkdir -p build/flutter_assets/fonts - doPubGet flutter pub get --offline -v - flutter build linux -v --release --split-debug-info="$debug" ${builtins.concatStringsSep " " (map (flag: "\"${flag}\"") finalAttrs.flutterBuildFlags)} + flutter build linux -v --release --split-debug-info="$debug" ${builtins.concatStringsSep " " (map (flag: "\"${flag}\"") flutterBuildFlags)} - runHook postBuild - ''; + runHook postBuild + ''; - installPhase = '' - runHook preInstall + dontDartInstall = true; + installPhase = universal.installPhase or '' + runHook preInstall - built=build/linux/*/release/bundle + built=build/linux/*/release/bundle - mkdir -p $out/bin - mv $built $out/app + mkdir -p $out/bin + mv $built $out/app - for f in $(find $out/app -iname "*.desktop" -type f); do - install -D $f $out/share/applications/$(basename $f) - done + for f in $(find $out/app -iname "*.desktop" -type f); do + install -D $f $out/share/applications/$(basename $f) + done - for f in $(find $out/app -maxdepth 1 -type f); do - ln -s $f $out/bin/$(basename $f) - done + for f in $(find $out/app -maxdepth 1 -type f); do + ln -s $f $out/bin/$(basename $f) + done - # make *.so executable - find $out/app -iname "*.so" -type f -exec chmod +x {} + + # make *.so executable + find $out/app -iname "*.so" -type f -exec chmod +x {} + - # remove stuff like /build/source/packages/ubuntu_desktop_installer/linux/flutter/ephemeral - for f in $(find $out/app -executable -type f); do - if patchelf --print-rpath "$f" | grep /build; then # this ignores static libs (e,g. libapp.so) also - echo "strip RPath of $f" - newrp=$(patchelf --print-rpath $f | sed -r "s|/build.*ephemeral:||g" | sed -r "s|/build.*profile:||g") - patchelf --set-rpath "$newrp" "$f" - fi - done - - runHook postInstall - ''; - - postFixup = '' - # Add runtime library dependencies to the LD_LIBRARY_PATH. - # For some reason, the RUNPATH of the executable is not used to load dynamic libraries in dart:ffi with DynamicLibrary.open(). - # - # This could alternatively be fixed with patchelf --add-needed, but this would cause all the libraries to be opened immediately, - # which is not what application authors expect. - for f in "$out"/bin/*; do - wrapProgram "$f" \ - --suffix LD_LIBRARY_PATH : '${lib.makeLibraryPath finalAttrs.runtimeDependencies}' \ - ${extraWrapProgramArgs} - done - - ${postFixup} - ''; - - passthru = (args.passthru or {}) // { - inherit (deps) depsListFile; + # remove stuff like /build/source/packages/ubuntu_desktop_installer/linux/flutter/ephemeral + for f in $(find $out/app -executable -type f); do + if patchelf --print-rpath "$f" | grep /build; then # this ignores static libs (e,g. libapp.so) also + echo "strip RPath of $f" + newrp=$(patchelf --print-rpath $f | sed -r "s|/build.*ephemeral:||g" | sed -r "s|/build.*profile:||g") + patchelf --set-rpath "$newrp" "$f" + fi + done + + runHook postInstall + ''; + + dontWrapGApps = true; + extraWrapProgramArgs = '' + ''${gappsWrapperArgs[@]} \ + ${extraWrapProgramArgs} + ''; }; - }); - - packageOverrideRepository = (callPackage ../../development/compilers/flutter/package-overrides { }) // customPackageOverrides; - productPackages = builtins.filter (package: package.kind != "dev") - (if autoDepsList - then lib.importJSON deps.depsListFile - else - if depsListFile == null - then [ ] - else lib.importJSON depsListFile); + + web = universal // { + dontDartBuild = true; + buildPhase = universal.buildPhase or '' + runHook preBuild + + mkdir -p build/flutter_assets/fonts + + flutter build web -v --release ${builtins.concatStringsSep " " (map (flag: "\"${flag}\"") flutterBuildFlags)} + + runHook postBuild + ''; + + dontDartInstall = true; + installPhase = universal.installPhase or '' + runHook preInstall + + cp -r build/web "$out" + + runHook postInstall + ''; + }; + }.${targetFlutterPlatform} or (throw "Unsupported Flutter host platform: ${targetFlutterPlatform}"); + + minimalFlutter = flutter.override { supportedTargetFlutterPlatforms = [ "universal" targetFlutterPlatform ]; }; + + buildAppWith = flutter: buildDartApplication.override { dart = flutter; }; in -builtins.foldl' - (prev: package: - if packageOverrideRepository ? ${package.name} - then - prev.overrideAttrs - (packageOverrideRepository.${package.name} { - inherit (package) - name - version - kind - source - dependencies; - }) - else prev) - baseDerivation - productPackages +buildAppWith minimalFlutter (builderArgs // { passthru = builderArgs.passthru or { } // { multiShell = buildAppWith flutter builderArgs; }; }) diff --git a/pkgs/build-support/go/module.nix b/pkgs/build-support/go/module.nix index 09b43063fb96a..153b675d48aef 100644 --- a/pkgs/build-support/go/module.nix +++ b/pkgs/build-support/go/module.nix @@ -39,6 +39,8 @@ # Not needed with buildGoModule , goPackagePath ? "" +, ldflags ? [ ] + # needed for buildFlags{,Array} warning , buildFlags ? "" , buildFlagsArray ? "" @@ -154,6 +156,9 @@ let GOFLAGS = lib.optionals (!proxyVendor) [ "-mod=vendor" ] ++ lib.optionals (!allowGoReference) [ "-trimpath" ]; inherit CGO_ENABLED enableParallelBuilding GO111MODULE GOTOOLCHAIN; + # If not set to an explicit value, set the buildid empty for reproducibility. + ldflags = ldflags ++ lib.optionals (!lib.any (lib.hasPrefix "-buildid=") ldflags) [ "-buildid=" ]; + configurePhase = args.configurePhase or ('' runHook preConfigure @@ -289,7 +294,8 @@ let disallowedReferences = lib.optional (!allowGoReference) go; - passthru = passthru // { inherit go goModules vendorHash; } // { inherit (args') vendorSha256; }; + passthru = passthru // { inherit go goModules vendorHash; } + // lib.optionalAttrs (args' ? vendorSha256 ) { inherit (args') vendorSha256; }; meta = { # Add default meta information @@ -297,6 +303,8 @@ let } // meta; }); in +lib.warnIf (args' ? vendorSha256) "`vendorSha256` is deprecated. Use `vendorHash` instead" lib.warnIf (buildFlags != "" || buildFlagsArray != "") "Use the `ldflags` and/or `tags` attributes instead of `buildFlags`/`buildFlagsArray`" +lib.warnIf (builtins.elem "-buildid=" ldflags) "`-buildid=` is set by default as ldflag by buildGoModule" package diff --git a/pkgs/build-support/go/package.nix b/pkgs/build-support/go/package.nix index 7e099b76f0b76..8ca5ca0dca011 100644 --- a/pkgs/build-support/go/package.nix +++ b/pkgs/build-support/go/package.nix @@ -37,6 +37,8 @@ , CGO_ENABLED ? go.CGO_ENABLED +, ldflags ? [ ] + # needed for buildFlags{,Array} warning , buildFlags ? "" , buildFlagsArray ? "" @@ -91,6 +93,9 @@ let GOARM = toString (lib.intersectLists [(stdenv.hostPlatform.parsed.cpu.version or "")] ["5" "6" "7"]); + # If not set to an explicit value, set the buildid empty for reproducibility. + ldflags = ldflags ++ lib.optionals (!lib.any (lib.hasPrefix "-buildid=") ldflags) [ "-buildid=" ]; + configurePhase = args.configurePhase or ('' runHook preConfigure @@ -280,4 +285,5 @@ let in lib.warnIf (buildFlags != "" || buildFlagsArray != "") "Use the `ldflags` and/or `tags` attributes instead of `buildFlags`/`buildFlagsArray`" +lib.warnIf (builtins.elem "-buildid=" ldflags) "`-buildid=` is set by default as ldflag by buildGoModule" package diff --git a/pkgs/build-support/kernel/compress-firmware-xz.nix b/pkgs/build-support/kernel/compress-firmware-xz.nix index cfb06a5c0f159..cb9ce7a713389 100644 --- a/pkgs/build-support/kernel/compress-firmware-xz.nix +++ b/pkgs/build-support/kernel/compress-firmware-xz.nix @@ -3,7 +3,9 @@ firmware: let - args = lib.optionalAttrs (firmware ? meta) { inherit (firmware) meta; }; + args = { + allowedRequisites = []; + } // lib.optionalAttrs (firmware ? meta) { inherit (firmware) meta; }; in runCommand "${firmware.name}-xz" args '' @@ -15,6 +17,13 @@ runCommand "${firmware.name}-xz" args '' sh -c 'xz -9c -T1 -C crc32 --lzma2=dict=2MiB "${firmware}/$1" > "$1.xz"' --) (cd ${firmware} && find lib/firmware -type l) | while read link; do target="$(readlink "${firmware}/$link")" - ln -vs -- "''${target/^${firmware}/$out}.xz" "$out/$link.xz" + if [ -f "${firmware}/$link" ]; then + ln -vs -- "''${target/^${firmware}/$out}.xz" "$out/$link.xz" + else + ln -vs -- "''${target/^${firmware}/$out}" "$out/$link" + fi done + + echo "Checking for broken symlinks:" + find -L $out -type l -print -execdir false -- '{}' '+' '' diff --git a/pkgs/build-support/kernel/make-initrd-ng.nix b/pkgs/build-support/kernel/make-initrd-ng.nix index 2418838176ef3..65e143cb7349d 100644 --- a/pkgs/build-support/kernel/make-initrd-ng.nix +++ b/pkgs/build-support/kernel/make-initrd-ng.nix @@ -8,7 +8,7 @@ let # compression type and filename extension. compressorName = fullCommand: builtins.elemAt (builtins.match "([^ ]*/)?([^ ]+).*" fullCommand) 1; in -{ stdenvNoCC, perl, cpio, ubootTools, lib, pkgsBuildHost, makeInitrdNGTool, binutils, runCommand +{ stdenvNoCC, libarchive, ubootTools, lib, pkgsBuildHost, makeInitrdNGTool, binutils, runCommand # Name of the derivation (not of the resulting file!) , name ? "initrd" @@ -54,7 +54,7 @@ in # guess may not align with u-boot's nomenclature correctly, so it can # be overridden. # See https://gitlab.denx.de/u-boot/u-boot/-/blob/9bfb567e5f1bfe7de8eb41f8c6d00f49d2b9a426/common/image.c#L81-106 for a list. -, uInitrdArch ? stdenvNoCC.hostPlatform.linuxArch +, uInitrdArch ? stdenvNoCC.hostPlatform.ubootArch # The name of the compression, as recognised by u-boot. # See https://gitlab.denx.de/u-boot/u-boot/-/blob/9bfb567e5f1bfe7de8eb41f8c6d00f49d2b9a426/common/image.c#L195-204 for a list. @@ -74,18 +74,18 @@ in passAsFile = ["contents"]; contents = lib.concatMapStringsSep "\n" ({ object, symlink, ... }: "${object}\n${lib.optionalString (symlink != null) symlink}") contents + "\n"; - nativeBuildInputs = [makeInitrdNGTool cpio] ++ lib.optional makeUInitrd ubootTools ++ lib.optional strip binutils; + nativeBuildInputs = [makeInitrdNGTool libarchive] ++ lib.optional makeUInitrd ubootTools ++ lib.optional strip binutils; STRIP = if strip then "${pkgsBuildHost.binutils.targetPrefix}strip" else null; }) '' mkdir -p ./root/var/empty make-initrd-ng "$contentsPath" ./root mkdir "$out" - (cd root && find * .[^.*] -exec touch -h -d '@1' '{}' +) + (cd root && find . -exec touch -h -d '@1' '{}' +) for PREP in $prepend; do cat $PREP >> $out/initrd done - (cd root && find . -print0 | sort -z | cpio -o -H newc -R +0:+0 --reproducible --null | eval -- $compress >> "$out/initrd") + (cd root && find . -print0 | sort -z | bsdtar --uid 0 --gid 0 -cnf - -T - | bsdtar --null -cf - --format=newc @- | eval -- $compress >> "$out/initrd") if [ -n "$makeUInitrd" ]; then mkimage -A "$uInitrdArch" -O linux -T ramdisk -C "$uInitrdCompression" -d "$out/initrd" $out/initrd.img diff --git a/pkgs/build-support/kernel/make-initrd-ng/Cargo.lock b/pkgs/build-support/kernel/make-initrd-ng/Cargo.lock index 78ae4a01da6f1..83e0fd3a2c5ef 100644 --- a/pkgs/build-support/kernel/make-initrd-ng/Cargo.lock +++ b/pkgs/build-support/kernel/make-initrd-ng/Cargo.lock @@ -3,16 +3,10 @@ version = 3 [[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] name = "eyre" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb" +checksum = "b6267a1fa6f59179ea4afc8e50fd8612a3cc60bc858f786ff877a4a8cb042799" dependencies = [ "indenter", "once_cell", @@ -20,9 +14,9 @@ dependencies = [ [[package]] name = "goblin" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91766b1121940d622933a13e20665857648681816089c9bc2075c4b75a6e4f6b" +checksum = "a7666983ed0dd8d21a6f6576ee00053ca0926fb281a5522577a4dbd0f1b54143" dependencies = [ "log", "plain", @@ -37,12 +31,9 @@ checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "make-initrd-ng" @@ -54,9 +45,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "plain" @@ -66,18 +57,18 @@ checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] name = "proc-macro2" -version = "1.0.42" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c278e965f1d8cf32d6e0e96de3d3e79712178ae67986d9cf9151f51e95aac89b" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.20" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -93,9 +84,9 @@ dependencies = [ [[package]] name = "scroll_derive" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdbda6ac5cd1321e724fa9cee216f3a61885889b896f073b8f82322789c5250e" +checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" dependencies = [ "proc-macro2", "quote", @@ -104,9 +95,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.98" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -115,6 +106,6 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.2" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" diff --git a/pkgs/build-support/kernel/make-initrd-ng/src/main.rs b/pkgs/build-support/kernel/make-initrd-ng/src/main.rs index 53096a842329c..daa688976c6c8 100644 --- a/pkgs/build-support/kernel/make-initrd-ng/src/main.rs +++ b/pkgs/build-support/kernel/make-initrd-ng/src/main.rs @@ -195,7 +195,7 @@ fn handle_path( .wrap_err_with(|| format!("failed to resolve symlink of {:?}", source))?; // Create the link, then push its target to the queue - if !target.exists() { + if !target.exists() && !target.is_symlink() { unix::fs::symlink(&link_target, &target).wrap_err_with(|| { format!("failed to symlink {:?} to {:?}", link_target, target) })?; diff --git a/pkgs/build-support/kernel/modules-closure.sh b/pkgs/build-support/kernel/modules-closure.sh index 74bc490eb15c9..5f61bac751af2 100644 --- a/pkgs/build-support/kernel/modules-closure.sh +++ b/pkgs/build-support/kernel/modules-closure.sh @@ -66,8 +66,8 @@ for module in $rootModules; do fi done -mkdir -p $out/lib/firmware -for module in $(cat closure); do +cd "$firmware" +for module in $(< ~-/closure); do # for builtin modules, modinfo will reply with a wrong output looking like: # $ modinfo -F firmware unix # name: unix @@ -78,16 +78,15 @@ for module in $(cat closure); do # # For now, the workaround is just to filter out the extraneous lines out # of its output. - for i in $(modinfo -b $kernel --set-version "$version" -F firmware $module | grep -v '^name:'); do - mkdir -p "$out/lib/firmware/$(dirname "$i")" + modinfo -b $kernel --set-version "$version" -F firmware $module | grep -v '^name:' | while read -r i; do echo "firmware for $module: $i" for name in "$i" "$i.xz" ""; do [ -z "$name" ] && echo "WARNING: missing firmware $i for module $module" - if cp "$firmware/lib/firmware/$name" "$out/lib/firmware/$name" 2>/dev/null; then + if cp -v --parents --no-preserve=mode lib/firmware/$name "$out" 2>/dev/null; then break fi done - done + done || : done # copy module ordering hints for depmod diff --git a/pkgs/build-support/make-hardcode-gsettings-patch/default.nix b/pkgs/build-support/make-hardcode-gsettings-patch/default.nix index a1d2de21c4cb3..820b003e3c6fa 100644 --- a/pkgs/build-support/make-hardcode-gsettings-patch/default.nix +++ b/pkgs/build-support/make-hardcode-gsettings-patch/default.nix @@ -6,42 +6,61 @@ }: /* - Can be used as part of an update script to automatically create a patch - hardcoding the path of all GSettings schemas in C code. - For example: - passthru = { - hardcodeGsettingsPatch = makeHardcodeGsettingsPatch { - inherit src; - schemaIdToVariableMapping = { - ... + Creates a patch that replaces every instantiation of GSettings in a C project + with a code that loads a GSettings schema from a hardcoded path. + + This is useful so that libraries can find schemas even though Nix lacks + a standard location like /usr/share, where GSettings system could look for schemas. + The derivation is is somewhat dependency-heavy so it is best used as part of an update script. + + For each schema id referenced in the source code (e.g. org.gnome.evolution), + a variable name such as `EVOLUTION` must be provided. + It will end up in the generated patch as `@EVOLUTION@` placeholder, which should be replaced at build time + with a path to the directory containing a `gschemas.compiled` file that includes the schema. + + + Arguments: + - `src`: source to generate the patch for. + + - `schemaIdToVariableMapping`: attrset assigning schema ids to variable names. + All used schemas must be listed. + + For example, `{ "org.gnome.evolution" = "EVOLUTION_SCHEMA_PATH"; }` + hardcodes looking for `org.gnome.evolution` into `@EVOLUTION_SCHEMA_PATH@`. + + - `patches`: A list of patches to apply before generating the patch. + + Example: + passthru = { + hardcodeGsettingsPatch = makeHardcodeGsettingsPatch { + inherit (finalAttrs) src; + schemaIdToVariableMapping = { + ... + }; }; - }; - - updateScript = - let - updateSource = ...; - updatePatch = _experimental-update-script-combinators.copyAttrOutputToFile "evolution-ews.hardcodeGsettingsPatch" ./hardcode-gsettings.patch; - in - _experimental-update-script-combinators.sequence [ - updateSource - updatePatch - ]; - }; - } - takes as input a mapping from schema path to variable name. - For example `{ "org.gnome.evolution" = "EVOLUTION_SCHEMA_PATH"; }` - hardcodes looking for `org.gnome.evolution` into `@EVOLUTION_SCHEMA_PATH@`. - All schemas must be listed. + + updateScript = + let + updateSource = ...; + updatePatch = _experimental-update-script-combinators.copyAttrOutputToFile "evolution-ews.hardcodeGsettingsPatch" ./hardcode-gsettings.patch; + in + _experimental-update-script-combinators.sequence [ + updateSource + updatePatch + ]; + }; + } */ { src, + patches ? [ ], schemaIdToVariableMapping, }: runCommand "hardcode-gsettings.patch" { - inherit src; + inherit src patches; nativeBuildInputs = [ git coccinelle @@ -51,6 +70,7 @@ runCommand '' unpackPhase cd "''${sourceRoot:-.}" + patchPhase set -x cp ${builtins.toFile "glib-schema-to-var.json" (builtins.toJSON schemaIdToVariableMapping)} ./glib-schema-to-var.json git init diff --git a/pkgs/build-support/make-hardcode-gsettings-patch/hardcode-gsettings.cocci b/pkgs/build-support/make-hardcode-gsettings-patch/hardcode-gsettings.cocci index a265f5fac384e..e916d74fd1f58 100644 --- a/pkgs/build-support/make-hardcode-gsettings-patch/hardcode-gsettings.cocci +++ b/pkgs/build-support/make-hardcode-gsettings-patch/hardcode-gsettings.cocci @@ -1,11 +1,14 @@ /** - * Since Nix does not have a standard location like /usr/share, - * where GSettings system could look for schemas, we need to point the software to a correct location somehow. + * Since Nix does not have a standard location like /usr/share where GSettings system + * could look for schemas, we need to point the software to a correct location somehow. * For executables, we handle this using wrappers but this is not an option for libraries like e-d-s. - * Instead, we hardcode the schema path when creating the settings. - * A schema path (ie org.gnome.evolution) can be replaced by @EVOLUTION_SCHEMA_ID@ - * which is then replaced at build time by substituteAll. - * The mapping is provided in a json file ./glib-schema-to-var.json + * Instead, we patch the source code to look for the schema in a schema source + * through a hardcoded path to the schema. + * + * For each schema id referenced in the source code (e.g. org.gnome.evolution), + * a variable name such as `EVOLUTION` must be provided in the ./glib-schema-to-var.json JSON file. + * It will end up in the resulting patch as `@EVOLUTION@` placeholder, which should be replaced at build time + * with a path to the directory containing a `gschemas.compiled` file that includes the schema. */ @initialize:python@ diff --git a/pkgs/build-support/mkshell/default.nix b/pkgs/build-support/mkshell/default.nix index 3517e949f67a9..5369301ea1052 100644 --- a/pkgs/build-support/mkshell/default.nix +++ b/pkgs/build-support/mkshell/default.nix @@ -16,6 +16,10 @@ let mergeInputs = name: (attrs.${name} or [ ]) ++ + # 1. get all `{build,nativeBuild,...}Inputs` from the elements of `inputsFrom` + # 2. since that is a list of lists, `flatten` that into a regular list + # 3. filter out of the result everything that's in `inputsFrom` itself + # this leaves actual dependencies of the derivations in `inputsFrom`, but never the derivations themselves (lib.subtractLists inputsFrom (lib.flatten (lib.catAttrs name inputsFrom))); rest = builtins.removeAttrs attrs [ diff --git a/pkgs/build-support/node/build-npm-package/default.nix b/pkgs/build-support/node/build-npm-package/default.nix index 7cfc0e9f9c0a5..42c6a9c065b2e 100644 --- a/pkgs/build-support/node/build-npm-package/default.nix +++ b/pkgs/build-support/node/build-npm-package/default.nix @@ -1,4 +1,10 @@ -{ lib, stdenv, fetchNpmDeps, buildPackages, nodejs }: +{ lib +, stdenv +, fetchNpmDeps +, buildPackages +, nodejs +, darwin +} @ topLevelArgs: { name ? "${args.pname}-${args.version}" , src ? null @@ -15,6 +21,9 @@ # Whether to force the usage of Git dependencies that have install scripts, but not a lockfile. # Use with care. , forceGitDeps ? false + # Whether to force allow an empty dependency cache. + # This can be enabled if there are truly no remote dependencies, but generally an empty cache indicates something is wrong. +, forceEmptyCache ? false # Whether to make the cache writable prior to installing dependencies. # Don't set this unless npm tries to write to the cache directory, as it can slow down the build. , makeCacheWritable ? false @@ -34,16 +43,16 @@ , npmPruneFlags ? npmInstallFlags # Value for npm `--workspace` flag and directory in which the files to be installed are found. , npmWorkspace ? null +, nodejs ? topLevelArgs.nodejs +, npmDeps ? fetchNpmDeps { + inherit forceGitDeps forceEmptyCache src srcs sourceRoot prePatch patches postPatch; + name = "${name}-npm-deps"; + hash = npmDepsHash; +} , ... } @ args: let - npmDeps = fetchNpmDeps { - inherit forceGitDeps src srcs sourceRoot prePatch patches postPatch; - name = "${name}-npm-deps"; - hash = npmDepsHash; - }; - # .override {} negates splicing, so we need to use buildPackages explicitly npmHooks = buildPackages.npmHooks.override { inherit nodejs; @@ -54,7 +63,9 @@ in stdenv.mkDerivation (args // { inherit npmDeps npmBuildScript; - nativeBuildInputs = nativeBuildInputs ++ [ nodejs npmConfigHook npmBuildHook npmInstallHook ]; + nativeBuildInputs = nativeBuildInputs + ++ [ nodejs npmConfigHook npmBuildHook npmInstallHook nodejs.python ] + ++ lib.optionals stdenv.isDarwin [ darwin.cctools ]; buildInputs = buildInputs ++ [ nodejs ]; strictDeps = true; diff --git a/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh b/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh index 64ddcbd567fce..56e3a883b99a3 100644 --- a/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh +++ b/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh @@ -5,36 +5,55 @@ npmInstallHook() { runHook preInstall - # `npm pack` writes to cache - npm config delete cache - local -r packageOut="$out/lib/node_modules/$(@jq@ --raw-output '.name' package.json)" + # `npm pack` writes to cache so temporarily override it while IFS= read -r file; do local dest="$packageOut/$(dirname "$file")" mkdir -p "$dest" cp "${npmWorkspace-.}/$file" "$dest" - done < <(@jq@ --raw-output '.[0].files | map(.path) | join("\n")' <<< "$(npm pack --json --dry-run ${npmWorkspace+--workspace=$npmWorkspace} $npmPackFlags "${npmPackFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}")") + done < <(@jq@ --raw-output '.[0].files | map(.path) | join("\n")' <<< "$(npm_config_cache="$HOME/.npm" npm pack --json --dry-run --loglevel=warn --no-foreground-scripts ${npmWorkspace+--workspace=$npmWorkspace} $npmPackFlags "${npmPackFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}")") + # Based on code from Python's buildPythonPackage wrap.sh script, for + # supporting both the case when makeWrapperArgs is an array and a + # IFS-separated string. + # + # TODO: remove the string branch when __structuredAttrs are used. + if [[ "${makeWrapperArgs+defined}" == "defined" && "$(declare -p makeWrapperArgs)" =~ ^'declare -a makeWrapperArgs=' ]]; then + local -a user_args=("${makeWrapperArgs[@]}") + else + local -a user_args="(${makeWrapperArgs:-})" + fi while IFS=" " read -ra bin; do mkdir -p "$out/bin" - makeWrapper @hostNode@ "$out/bin/${bin[0]}" --add-flags "$packageOut/${bin[1]}" + makeWrapper @hostNode@ "$out/bin/${bin[0]}" --add-flags "$packageOut/${bin[1]}" "${user_args[@]}" done < <(@jq@ --raw-output '(.bin | type) as $typ | if $typ == "string" then .name + " " + .bin elif $typ == "object" then .bin | to_entries | map(.key + " " + .value) | join("\n") + elif $typ == "null" then empty else "invalid type " + $typ | halt_error end' "${npmWorkspace-.}/package.json") while IFS= read -r man; do installManPage "$packageOut/$man" done < <(@jq@ --raw-output '(.man | type) as $typ | if $typ == "string" then .man elif $typ == "list" then .man | join("\n") + elif $typ == "null" then empty else "invalid type " + $typ | halt_error end' "${npmWorkspace-.}/package.json") local -r nodeModulesPath="$packageOut/node_modules" if [ ! -d "$nodeModulesPath" ]; then if [ -z "${dontNpmPrune-}" ]; then - npm prune --omit=dev --no-save ${npmWorkspace+--workspace=$npmWorkspace} $npmPruneFlags "${npmPruneFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}" + if ! npm prune --omit=dev --no-save ${npmWorkspace+--workspace=$npmWorkspace} $npmPruneFlags "${npmPruneFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then + echo + echo + echo "ERROR: npm prune step failed" + echo + echo 'If npm tried to download additional dependencies above, try setting `dontNpmPrune = true`.' + echo + + exit 1 + fi fi find node_modules -maxdepth 1 -type d -empty -delete diff --git a/pkgs/build-support/node/fetch-npm-deps/Cargo.lock b/pkgs/build-support/node/fetch-npm-deps/Cargo.lock index 482eb6c7beab6..8ba72a7b76c49 100644 --- a/pkgs/build-support/node/fetch-npm-deps/Cargo.lock +++ b/pkgs/build-support/node/fetch-npm-deps/Cargo.lock @@ -4,24 +4,24 @@ version = 3 [[package]] name = "aho-corasick" -version = "1.0.2" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "async-channel" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" dependencies = [ "concurrent-queue", "event-listener", @@ -47,9 +47,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.2" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "bitflags" @@ -59,9 +59,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.3" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "block-buffer" @@ -74,9 +74,9 @@ dependencies = [ [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "castaway" @@ -86,9 +86,12 @@ checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] [[package]] name = "cfg-if" @@ -98,33 +101,23 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "concurrent-queue" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" +checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" dependencies = [ "crossbeam-utils", ] [[package]] name = "cpufeatures" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ "libc", ] [[package]] -name = "crossbeam-channel" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] name = "crossbeam-deque" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -184,9 +177,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.63+curl-8.1.2" +version = "0.4.68+curl-8.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeb0fef7046022a1e2ad67a004978f0e3cacb9e3123dc62ce768f92197b771dc" +checksum = "b4a0d18d88360e374b16b2273c832b5e57258ffc1d4aa4f96b108e0738d5752f" dependencies = [ "cc", "libc", @@ -194,7 +187,7 @@ dependencies = [ "openssl-sys", "pkg-config", "vcpkg", - "winapi", + "windows-sys", ] [[package]] @@ -209,15 +202,15 @@ dependencies = [ [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" dependencies = [ "humantime", "is-terminal", @@ -228,26 +221,15 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.1" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" dependencies = [ - "errno-dragonfly", "libc", "windows-sys", ] [[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - -[[package]] name = "event-listener" version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -263,6 +245,12 @@ dependencies = [ ] [[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -279,15 +267,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" [[package]] name = "futures-lite" @@ -295,7 +283,7 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" dependencies = [ - "fastrand", + "fastrand 1.9.0", "futures-core", "futures-io", "memchr", @@ -316,9 +304,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "libc", @@ -327,15 +315,15 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -368,24 +356,13 @@ dependencies = [ ] [[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys", -] - -[[package]] name = "is-terminal" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix 0.38.2", + "rustix", "windows-sys", ] @@ -416,21 +393,21 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libz-sys" -version = "1.1.9" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" dependencies = [ "cc", "libc", @@ -440,27 +417,21 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" - -[[package]] -name = "linux-raw-sys" -version = "0.4.3" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "log" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -472,16 +443,6 @@ dependencies = [ ] [[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] name = "once_cell" version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -495,9 +456,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.90" +version = "0.9.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" +checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" dependencies = [ "cc", "libc", @@ -507,9 +468,9 @@ dependencies = [ [[package]] name = "parking" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "percent-encoding" @@ -519,18 +480,18 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", @@ -539,9 +500,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.10" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pkg-config" @@ -581,6 +542,7 @@ dependencies = [ "digest", "env_logger", "isahc", + "log", "rayon", "serde", "serde_json", @@ -593,18 +555,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.63" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.29" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -641,9 +603,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ "either", "rayon-core", @@ -651,74 +613,70 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.8.4" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", + "regex-automata", "regex-syntax", ] [[package]] -name = "regex-syntax" -version = "0.7.2" +name = "regex-automata" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] [[package]] -name = "rustix" -version = "0.37.22" +name = "regex-syntax" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8818fa822adcc98b18fedbb3632a6a33213c070556b5aa7c4c8cc21cff565c4c" -dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys", -] +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "rustix" -version = "0.38.2" +version = "0.38.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aabcb0461ebd01d6b79945797c27f8529082226cb630a9865a71870ff63532a4" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.4.1", "errno", "libc", - "linux-raw-sys 0.4.3", + "linux-raw-sys", "windows-sys", ] [[package]] name = "ryu" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "same-file" @@ -740,24 +698,24 @@ dependencies = [ [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.166" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01b7404f9d441d3ad40e6a636a7782c377d2abdbe4fa2440e2edcc2f4f10db8" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.166" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd83d6dde2b6b2d466e14d9d1acce8816dedee94f735eac6395808b3483c6d6" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", @@ -766,9 +724,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.99" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -777,9 +735,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -788,9 +746,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -799,9 +757,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] @@ -819,9 +777,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -829,9 +787,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.23" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -840,23 +798,22 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.6.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ - "autocfg", "cfg-if", - "fastrand", + "fastrand 2.0.1", "redox_syscall", - "rustix 0.37.22", + "rustix", "windows-sys", ] [[package]] name = "termcolor" -version = "1.2.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" dependencies = [ "winapi-util", ] @@ -878,11 +835,10 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -891,9 +847,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", @@ -902,9 +858,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", ] @@ -921,9 +877,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" @@ -933,9 +889,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.10" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -948,9 +904,9 @@ dependencies = [ [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", @@ -972,15 +928,15 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "waker-fn" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", @@ -1010,9 +966,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -1034,9 +990,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -1049,42 +1005,42 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" diff --git a/pkgs/build-support/node/fetch-npm-deps/Cargo.toml b/pkgs/build-support/node/fetch-npm-deps/Cargo.toml index 41347b6c2cc37..ea121c510c95b 100644 --- a/pkgs/build-support/node/fetch-npm-deps/Cargo.toml +++ b/pkgs/build-support/node/fetch-npm-deps/Cargo.toml @@ -6,17 +6,18 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -anyhow = "1.0.71" +anyhow = "1.0.75" backoff = "0.4.0" -base64 = "0.21.2" +base64 = "0.21.5" digest = "0.10.7" -env_logger = "0.10.0" +env_logger = "0.10.1" isahc = { version = "1.7.2", default_features = false } -rayon = "1.7.0" -serde = { version = "1.0.164", features = ["derive"] } -serde_json = "1.0.99" -sha1 = "0.10.5" -sha2 = "0.10.7" -tempfile = "3.6.0" -url = { version = "2.4.0", features = ["serde"] } -walkdir = "2.3.3" +log = "0.4.20" +rayon = "1.8.0" +serde = { version = "1.0.193", features = ["derive"] } +serde_json = "1.0.108" +sha1 = "0.10.6" +sha2 = "0.10.8" +tempfile = "3.8.1" +url = { version = "2.4.1", features = ["serde"] } +walkdir = "2.4.0" diff --git a/pkgs/build-support/node/fetch-npm-deps/default.nix b/pkgs/build-support/node/fetch-npm-deps/default.nix index 67a4c337c0d2d..725f9ba3bb017 100644 --- a/pkgs/build-support/node/fetch-npm-deps/default.nix +++ b/pkgs/build-support/node/fetch-npm-deps/default.nix @@ -36,8 +36,8 @@ ''; }; - makeTest = { name, src, hash, forceGitDeps ? false }: testers.invalidateFetcherByDrvHash fetchNpmDeps { - inherit name hash forceGitDeps; + makeTest = { name, src, hash, forceGitDeps ? false, forceEmptyCache ? false }: testers.invalidateFetcherByDrvHash fetchNpmDeps { + inherit name hash forceGitDeps forceEmptyCache; src = makeTestSrc { inherit name src; }; }; @@ -98,6 +98,20 @@ hash = "sha256-VzQhArHoznYSXUT7l9HkJV4yoSOmoP8eYTLel1QwmB4="; }; + # This package has no resolved deps whatsoever, which will not actually work but does test the forceEmptyCache option. + emptyCache = makeTest { + name = "empty-cache"; + + src = fetchurl { + url = "https://raw.githubusercontent.com/bufbuild/protobuf-es/v1.2.1/package-lock.json"; + hash = "sha256-UdBUEb4YRHsbvyjymIyjemJEiaI9KQRirqt+SFSK0wA="; + }; + + hash = "sha256-Cdv40lQjRszzJtJydZt25uYfcJVeJGwH54A+agdH9wI="; + + forceEmptyCache = true; + }; + # This package contains both hosted Git shorthand, and a bundled dependency that happens to override an existing one. etherpadLite1818 = makeTest { name = "etherpad-lite-1.8.18"; @@ -111,6 +125,18 @@ forceGitDeps = true; }; + + # This package has a lockfile v1 git dependency with no `dependencies` attribute, since it sementically has no dependencies. + jitsiMeet9111 = makeTest { + name = "jitsi-meet-9111"; + + src = fetchurl { + url = "https://raw.githubusercontent.com/jitsi/jitsi-meet/stable/jitsi-meet_9111/package-lock.json"; + hash = "sha256-NU+eQD4WZ4BMur8uX79uk8wUPsZvIT02KhPWHTmaihk="; + }; + + hash = "sha256-FhxlJ0HdJMPiWe7+n1HaGLWOr/2HJEPwiS65uqXZM8Y="; + }; }; meta = with lib; { @@ -124,6 +150,7 @@ { name ? "npm-deps" , hash ? "" , forceGitDeps ? false + , forceEmptyCache ? false , ... } @ args: let @@ -136,6 +163,7 @@ }; forceGitDeps_ = lib.optionalAttrs forceGitDeps { FORCE_GIT_DEPS = true; }; + forceEmptyCache_ = lib.optionalAttrs forceEmptyCache { FORCE_EMPTY_CACHE = true; }; in stdenvNoCC.mkDerivation (args // { inherit name; @@ -174,5 +202,5 @@ else "/no-cert-file.crt"; outputHashMode = "recursive"; - } // hash_ // forceGitDeps_); + } // hash_ // forceGitDeps_ // forceEmptyCache_); } diff --git a/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs b/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs index b7efedac59bdd..c49c094b85c68 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs @@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize}; use sha1::Sha1; use sha2::{Sha256, Sha512}; use std::{ + fmt::Write as FmtWrite, fs::{self, File}, io::Write, path::PathBuf, @@ -43,6 +44,13 @@ impl Cache { Cache(path) } + pub fn init(&self) -> anyhow::Result<()> { + fs::create_dir_all(self.0.join("content-v2"))?; + fs::create_dir_all(self.0.join("index-v5"))?; + + Ok(()) + } + pub fn put( &self, key: String, @@ -71,10 +79,10 @@ impl Cache { push_hash_segments( &mut p, - &hash - .into_iter() - .map(|n| format!("{n:02x}")) - .collect::<String>(), + &hash.into_iter().fold(String::new(), |mut out, n| { + let _ = write!(out, "{n:02x}"); + out + }), ); p diff --git a/pkgs/build-support/node/fetch-npm-deps/src/main.rs b/pkgs/build-support/node/fetch-npm-deps/src/main.rs index 9d86bd8091a79..dc20c72970491 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/main.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/main.rs @@ -234,14 +234,21 @@ fn main() -> anyhow::Result<()> { (out_tempdir.path(), true) }; - let packages = parse::lockfile(&lock_content, env::var("FORCE_GIT_DEPS").is_ok())?; + let packages = parse::lockfile( + &lock_content, + env::var("FORCE_GIT_DEPS").is_ok(), + env::var("FORCE_EMPTY_CACHE").is_ok(), + )?; let cache = Cache::new(out.join("_cacache")); + cache.init()?; packages.into_par_iter().try_for_each(|package| { eprintln!("{}", package.name); - let tarball = package.tarball()?; + let tarball = package + .tarball() + .map_err(|e| anyhow!("couldn't fetch {} at {}: {e:?}", package.name, package.url))?; let integrity = package.integrity().map(ToString::to_string); cache diff --git a/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs b/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs index f50a31651d0ee..c6e77153a0b80 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs @@ -18,23 +18,20 @@ pub(super) fn packages(content: &str) -> anyhow::Result<Vec<Package>> { 1 => { let initial_url = get_initial_url()?; - lockfile - .dependencies - .map(|p| to_new_packages(p, &initial_url)) - .transpose()? + to_new_packages(lockfile.dependencies.unwrap_or_default(), &initial_url)? } - 2 | 3 => lockfile.packages.map(|pkgs| { - pkgs.into_iter() - .filter(|(n, p)| !n.is_empty() && matches!(p.resolved, Some(UrlOrString::Url(_)))) - .map(|(n, p)| Package { name: Some(n), ..p }) - .collect() - }), + 2 | 3 => lockfile + .packages + .unwrap_or_default() + .into_iter() + .filter(|(n, p)| !n.is_empty() && matches!(p.resolved, Some(UrlOrString::Url(_)))) + .map(|(n, p)| Package { name: Some(n), ..p }) + .collect(), _ => bail!( "We don't support lockfile version {}, please file an issue.", lockfile.version ), - } - .expect("lockfile should have packages"); + }; packages.par_sort_by(|x, y| { x.resolved @@ -182,6 +179,7 @@ impl fmt::Display for Hash { } } +#[allow(clippy::incorrect_partial_ord_impl_on_ord_type)] impl PartialOrd for Hash { fn partial_cmp(&self, other: &Hash) -> Option<Ordering> { let lhs = self.0.split_once('-')?.0; @@ -216,29 +214,35 @@ fn to_new_packages( } if let UrlOrString::Url(v) = &package.version { - for (scheme, host) in [ - ("github", "github.com"), - ("bitbucket", "bitbucket.org"), - ("gitlab", "gitlab.com"), - ] { - if v.scheme() == scheme { - package.version = { - let mut new_url = initial_url.clone(); - - new_url.set_host(Some(host))?; - - if v.path().ends_with(".git") { - new_url.set_path(v.path()); - } else { - new_url.set_path(&format!("{}.git", v.path())); - } - - new_url.set_fragment(v.fragment()); - - UrlOrString::Url(new_url) - }; - - break; + if v.scheme() == "npm" { + if let Some(UrlOrString::Url(ref url)) = &package.resolved { + package.version = UrlOrString::Url(url.clone()); + } + } else { + for (scheme, host) in [ + ("github", "github.com"), + ("bitbucket", "bitbucket.org"), + ("gitlab", "gitlab.com"), + ] { + if v.scheme() == scheme { + package.version = { + let mut new_url = initial_url.clone(); + + new_url.set_host(Some(host))?; + + if v.path().ends_with(".git") { + new_url.set_path(v.path()); + } else { + new_url.set_path(&format!("{}.git", v.path())); + } + + new_url.set_fragment(v.fragment()); + + UrlOrString::Url(new_url) + }; + + break; + } } } } @@ -268,7 +272,8 @@ fn get_initial_url() -> anyhow::Result<Url> { #[cfg(test)] mod tests { use super::{ - get_initial_url, to_new_packages, Hash, HashCollection, OldPackage, Package, UrlOrString, + get_initial_url, packages, to_new_packages, Hash, HashCollection, OldPackage, Package, + UrlOrString, }; use std::{ cmp::Ordering, @@ -330,4 +335,36 @@ mod tests { Some(Hash(String::from("sha512-foo"))) ); } + + #[test] + fn parse_lockfile_correctly() { + let packages = packages( + r#"{ + "name": "node-ddr", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "string-width-cjs": { + "version": "npm:string-width@4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + } + } + }"#).unwrap(); + + assert_eq!(packages.len(), 1); + assert_eq!( + packages[0].resolved, + Some(UrlOrString::Url( + Url::parse("https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz") + .unwrap() + )) + ); + } } diff --git a/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs b/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs index b37652ffdf82d..0bca33f039152 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs @@ -1,10 +1,11 @@ use anyhow::{anyhow, bail, Context}; use lock::UrlOrString; +use log::{debug, info}; use rayon::prelude::*; use serde_json::{Map, Value}; use std::{ fs, - io::{self, Read}, + io::Write, process::{Command, Stdio}, }; use tempfile::{tempdir, TempDir}; @@ -14,7 +15,13 @@ use crate::util; pub mod lock; -pub fn lockfile(content: &str, force_git_deps: bool) -> anyhow::Result<Vec<Package>> { +pub fn lockfile( + content: &str, + force_git_deps: bool, + force_empty_cache: bool, +) -> anyhow::Result<Vec<Package>> { + debug!("parsing lockfile with contents:\n{content}"); + let mut packages = lock::packages(content) .context("failed to extract packages from lockfile")? .into_par_iter() @@ -25,6 +32,10 @@ pub fn lockfile(content: &str, force_git_deps: bool) -> anyhow::Result<Vec<Packa }) .collect::<anyhow::Result<Vec<_>>>()?; + if packages.is_empty() && !force_empty_cache { + bail!("No cacheable dependencies were found. Please inspect the upstream `package-lock.json` file and ensure that remote dependencies have `resolved` URLs and `integrity` hashes. If the lockfile is missing this data, attempt to get upstream to fix it via a tool like <https://github.com/jeslie0/npm-lockfile-fix>. If generating an empty cache is intentional and you would like to do it anyways, set `forceEmptyCache = true`."); + } + let mut new = Vec::new(); for pkg in packages @@ -38,6 +49,8 @@ pub fn lockfile(content: &str, force_git_deps: bool) -> anyhow::Result<Vec<Packa let path = dir.path().join("package"); + info!("recursively parsing lockfile for {} at {path:?}", pkg.name); + let lockfile_contents = fs::read_to_string(path.join("package-lock.json")); let package_json_path = path.join("package.json"); @@ -64,7 +77,13 @@ pub fn lockfile(content: &str, force_git_deps: bool) -> anyhow::Result<Vec<Packa } if let Ok(lockfile_contents) = lockfile_contents { - new.append(&mut lockfile(&lockfile_contents, force_git_deps)?); + new.append(&mut lockfile( + &lockfile_contents, + force_git_deps, + // force_empty_cache is turned on here since recursively parsed lockfiles should be + // allowed to have an empty cache without erroring by default + true, + )?); } } @@ -106,7 +125,7 @@ impl Package { let specifics = match get_hosted_git_url(&resolved)? { Some(hosted) => { - let mut body = util::get_url_with_retry(&hosted)?; + let body = util::get_url_body_with_retry(&hosted)?; let workdir = tempdir()?; @@ -120,7 +139,7 @@ impl Package { .stdin(Stdio::piped()) .spawn()?; - io::copy(&mut body, &mut cmd.stdin.take().unwrap())?; + cmd.stdin.take().unwrap().write_all(&body)?; let exit = cmd.wait()?; @@ -154,13 +173,7 @@ impl Package { pub fn tarball(&self) -> anyhow::Result<Vec<u8>> { match &self.specifics { - Specifics::Registry { .. } => { - let mut body = Vec::new(); - - util::get_url_with_retry(&self.url)?.read_to_end(&mut body)?; - - Ok(body) - } + Specifics::Registry { .. } => Ok(util::get_url_body_with_retry(&self.url)?), Specifics::Git { workdir } => Ok(Command::new("tar") .args([ "--sort=name", diff --git a/pkgs/build-support/node/fetch-npm-deps/src/util.rs b/pkgs/build-support/node/fetch-npm-deps/src/util.rs index 7a220f681c0d8..7dd928fdc43fa 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/util.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/util.rs @@ -4,7 +4,7 @@ use isahc::{ Body, Request, RequestExt, }; use serde_json::{Map, Value}; -use std::{env, path::Path}; +use std::{env, io::Read, path::Path}; use url::Url; pub fn get_url(url: &Url) -> Result<Body, isahc::Error> { @@ -28,7 +28,7 @@ pub fn get_url(url: &Url) -> Result<Body, isahc::Error> { if let Some(host) = url.host_str() { if let Ok(npm_tokens) = env::var("NIX_NPM_TOKENS") { if let Ok(tokens) = serde_json::from_str::<Map<String, Value>>(&npm_tokens) { - if let Some(token) = tokens.get(host).and_then(|val| val.as_str()) { + if let Some(token) = tokens.get(host).and_then(serde_json::Value::as_str) { request = request.header("Authorization", format!("Bearer {token}")); } } @@ -38,15 +38,23 @@ pub fn get_url(url: &Url) -> Result<Body, isahc::Error> { Ok(request.body(())?.send()?.into_body()) } -pub fn get_url_with_retry(url: &Url) -> Result<Body, isahc::Error> { +pub fn get_url_body_with_retry(url: &Url) -> Result<Vec<u8>, isahc::Error> { retry(ExponentialBackoff::default(), || { - get_url(url).map_err(|err| { - if err.is_network() || err.is_timeout() { - backoff::Error::transient(err) - } else { - backoff::Error::permanent(err) - } - }) + get_url(url) + .and_then(|mut body| { + let mut buf = Vec::new(); + + body.read_to_end(&mut buf)?; + + Ok(buf) + }) + .map_err(|err| { + if err.is_network() || err.is_timeout() { + backoff::Error::transient(err) + } else { + backoff::Error::permanent(err) + } + }) }) .map_err(|backoff_err| match backoff_err { backoff::Error::Permanent(err) diff --git a/pkgs/build-support/node/fetch-yarn-deps/default.nix b/pkgs/build-support/node/fetch-yarn-deps/default.nix index d95b1078c162e..e837f7457d176 100644 --- a/pkgs/build-support/node/fetch-yarn-deps/default.nix +++ b/pkgs/build-support/node/fetch-yarn-deps/default.nix @@ -3,7 +3,7 @@ let yarnpkg-lockfile-tar = fetchurl { url = "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz"; - sha512 = "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ=="; + hash = "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ=="; }; tests = callPackage ./tests {}; @@ -62,8 +62,9 @@ in { dontUnpack = src == null; dontInstall = true; - nativeBuildInputs = [ prefetch-yarn-deps ]; + nativeBuildInputs = [ prefetch-yarn-deps cacert ]; GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + NODE_EXTRA_CA_CERTS = "${cacert}/etc/ssl/certs/ca-bundle.crt"; buildPhase = '' runHook preBuild diff --git a/pkgs/build-support/node/fetch-yarn-deps/fixup.js b/pkgs/build-support/node/fetch-yarn-deps/fixup.js index 8b91e7efa63fd..732e569aba7bd 100755 --- a/pkgs/build-support/node/fetch-yarn-deps/fixup.js +++ b/pkgs/build-support/node/fetch-yarn-deps/fixup.js @@ -21,6 +21,8 @@ const fixupYarnLock = async (lockContents, verbose) => { if (verbose) console.log(`Rewriting URL ${url} for dependency ${dep}`) pkg.resolved = urlToName(url) + if (hash) + pkg.resolved += `#${hash}` return [dep, pkg] }) diff --git a/pkgs/build-support/node/fetch-yarn-deps/index.js b/pkgs/build-support/node/fetch-yarn-deps/index.js index 04f47362b10dc..e60fdeb543305 100755 --- a/pkgs/build-support/node/fetch-yarn-deps/index.js +++ b/pkgs/build-support/node/fetch-yarn-deps/index.js @@ -37,7 +37,9 @@ const downloadFileHttps = (fileName, url, expectedHash, hashType = 'sha1') => { res.on('end', () => { file.close() const h = hash.read() - if (h != expectedHash) return reject(new Error(`hash mismatch, expected ${expectedHash}, got ${h}`)) + if (expectedHash === undefined){ + console.log(`Warning: lockfile url ${url} doesn't end in "#<hash>" to validate against. Downloaded file had hash ${h}.`); + } else if (h != expectedHash) return reject(new Error(`hash mismatch, expected ${expectedHash}, got ${h}`)) resolve() }) res.on('error', e => reject(e)) @@ -86,10 +88,17 @@ const isGitUrl = pattern => { } const downloadPkg = (pkg, verbose) => { - const [ name, spec ] = pkg.key.split('@', 2); - if (spec.startsWith('file:')) { - console.info(`ignoring relative file:path dependency "${spec}"`) + const fileMarker = '@file:' + const split = pkg.key.split(fileMarker) + if (split.length == 2) { + console.info(`ignoring lockfile entry "${split[0]}" which points at path "${split[1]}"`) return + } else if (split.length > 2) { + throw new Error(`The lockfile entry key "${pkg.key}" contains "${fileMarker}" more than once. Processing is not implemented.`) + } + + if (pkg.resolved === undefined) { + throw new Error(`The lockfile entry with key "${pkg.key}" cannot be downloaded because it is missing the "resolved" attribute, which should contain the URL to download from. The lockfile might be invalid.`) } const [ url, hash ] = pkg.resolved.split('#') @@ -131,19 +140,10 @@ const performParallel = tasks => { const prefetchYarnDeps = async (lockContents, verbose) => { const lockData = lockfile.parse(lockContents) - const tasks = Object.values( + await performParallel( Object.entries(lockData.object) - .map(([key, value]) => { - return { key, ...value } - }) - .reduce((out, pkg) => { - out[pkg.resolved] = pkg - return out - }, {}) + .map(([key, value]) => () => downloadPkg({ key, ...value }, verbose)) ) - .map(pkg => () => downloadPkg(pkg, verbose)) - - await performParallel(tasks) await fs.promises.writeFile('yarn.lock', lockContents) if (verbose) console.log('Done') } diff --git a/pkgs/build-support/node/fetch-yarn-deps/tests/default.nix b/pkgs/build-support/node/fetch-yarn-deps/tests/default.nix index 8ffe103a9548a..8057d05ba72ca 100644 --- a/pkgs/build-support/node/fetch-yarn-deps/tests/default.nix +++ b/pkgs/build-support/node/fetch-yarn-deps/tests/default.nix @@ -1,6 +1,10 @@ { testers, fetchYarnDeps, ... }: { + file = testers.invalidateFetcherByDrvHash fetchYarnDeps { + yarnLock = ./file.lock; + sha256 = "sha256-BPuyQVCbdpFL/iRhmarwWAmWO2NodlVCOY9JU+4pfa4="; + }; simple = testers.invalidateFetcherByDrvHash fetchYarnDeps { yarnLock = ./simple.lock; sha256 = "sha256-FRrt8BixleILmFB2ZV8RgPNLqgS+dlH5nWoPgeaaNQ8="; diff --git a/pkgs/build-support/node/fetch-yarn-deps/tests/file.lock b/pkgs/build-support/node/fetch-yarn-deps/tests/file.lock new file mode 100644 index 0000000000000..4881d83a7de98 --- /dev/null +++ b/pkgs/build-support/node/fetch-yarn-deps/tests/file.lock @@ -0,0 +1,9 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@org/somepack@file:vendor/orgpacks/somepack/assets": + version "1.0.0" + +"otherpack@file:vendor/otherpack": + version "1.0.0" diff --git a/pkgs/build-support/nuke-references/default.nix b/pkgs/build-support/nuke-references/default.nix index 8f02c559238e5..4472d6eeddc10 100644 --- a/pkgs/build-support/nuke-references/default.nix +++ b/pkgs/build-support/nuke-references/default.nix @@ -38,4 +38,6 @@ stdenvNoCC.mkDerivation { shell = lib.getBin shell + (shell.shellPath or ""); signingUtils = lib.optionalString darwinCodeSign signingUtils; }; + + meta.mainProgram = "nuke-refs"; } diff --git a/pkgs/build-support/ocaml/topkg.nix b/pkgs/build-support/ocaml/topkg.nix new file mode 100644 index 0000000000000..73be5815e44c7 --- /dev/null +++ b/pkgs/build-support/ocaml/topkg.nix @@ -0,0 +1,28 @@ +{ lib, stdenv, fetchurl, ocaml, findlib, topkg, ocamlbuild, cmdliner, odoc, b0 +}: + +{ pname, version, nativeBuildInputs ? [ ], buildInputs ? [ ], ... }@args: + +lib.throwIf (args ? minimalOCamlVersion + && lib.versionOlder ocaml.version args.minimalOCamlVersion) +"${pname}-${version} is not available for OCaml ${ocaml.version}" + +stdenv.mkDerivation ({ + + dontAddStaticConfigureFlags = true; + configurePlatforms = [ ]; + strictDeps = true; + inherit (topkg) buildPhase installPhase; + +} // (builtins.removeAttrs args [ "minimalOCamlVersion" ]) // { + + name = "ocaml${ocaml.version}-${pname}-${version}"; + + nativeBuildInputs = [ ocaml findlib ocamlbuild topkg ] ++ nativeBuildInputs; + buildInputs = [ topkg ] ++ buildInputs; + + meta = (args.meta or { }) // { + platforms = args.meta.platforms or ocaml.meta.platforms; + }; + +}) diff --git a/pkgs/build-support/oci-tools/default.nix b/pkgs/build-support/oci-tools/default.nix index 18b238033ffde..67e081522d64c 100644 --- a/pkgs/build-support/oci-tools/default.nix +++ b/pkgs/build-support/oci-tools/default.nix @@ -42,7 +42,7 @@ "/sys/fs/cgroup" = { type = "cgroup"; source = "cgroup"; - options = [ "nosuid" "noexec" "nodev" "realatime" "ro" ]; + options = [ "nosuid" "noexec" "nodev" "relatime" "ro" ]; }; }; config = writeText "config.json" (builtins.toJSON { diff --git a/pkgs/build-support/php/build-composer-project.nix b/pkgs/build-support/php/build-composer-project.nix index 6aecf43457730..80c63bcde71b9 100644 --- a/pkgs/build-support/php/build-composer-project.nix +++ b/pkgs/build-support/php/build-composer-project.nix @@ -13,10 +13,12 @@ let composerNoDev = previousAttrs.composerNoDev or true; composerNoPlugins = previousAttrs.composerNoPlugins or true; composerNoScripts = previousAttrs.composerNoScripts or true; + composerStrictValidation = previousAttrs.composerStrictValidation or true; nativeBuildInputs = (previousAttrs.nativeBuildInputs or [ ]) ++ [ composer composer-local-repo-plugin + phpDrv phpDrv.composerHooks.composerInstallHook ]; @@ -53,6 +55,13 @@ let runHook postInstall ''; + doInstallCheck = previousAttrs.doInstallCheck or false; + installCheckPhase = previousAttrs.installCheckPhase or '' + runHook preInstallCheck + + runHook postInstallCheck + ''; + composerRepository = phpDrv.mkComposerRepository { inherit composer composer-local-repo-plugin; inherit (finalAttrs) patches pname src vendorHash version; @@ -61,8 +70,13 @@ let composerNoDev = previousAttrs.composerNoDev or true; composerNoPlugins = previousAttrs.composerNoPlugins or true; composerNoScripts = previousAttrs.composerNoScripts or true; + composerStrictValidation = previousAttrs.composerStrictValidation or true; }; + COMPOSER_CACHE_DIR="/dev/null"; + COMPOSER_DISABLE_NETWORK="1"; + COMPOSER_MIRROR_PATH_REPOS="1"; + meta = previousAttrs.meta or { } // { platforms = lib.platforms.all; }; diff --git a/pkgs/build-support/php/build-composer-repository.nix b/pkgs/build-support/php/build-composer-repository.nix index 30b0b48de7515..e359c0829aaf7 100644 --- a/pkgs/build-support/php/build-composer-repository.nix +++ b/pkgs/build-support/php/build-composer-repository.nix @@ -32,6 +32,7 @@ let composerNoDev = previousAttrs.composerNoDev or true; composerNoPlugins = previousAttrs.composerNoPlugins or true; composerNoScripts = previousAttrs.composerNoScripts or true; + composerStrictValidation = previousAttrs.composerStrictValidation or true; name = "${previousAttrs.pname}-${previousAttrs.version}-composer-repository"; @@ -41,6 +42,7 @@ let nativeBuildInputs = (previousAttrs.nativeBuildInputs or [ ]) ++ [ composer composer-local-repo-plugin + phpDrv phpDrv.composerHooks.composerRepositoryHook ]; @@ -74,6 +76,13 @@ let runHook postInstall ''; + doInstallCheck = previousAttrs.doInstallCheck or false; + installCheckPhase = previousAttrs.installCheckPhase or '' + runHook preInstallCheck + + runHook postInstallCheck + ''; + COMPOSER_CACHE_DIR = "/dev/null"; COMPOSER_MIRROR_PATH_REPOS = "1"; COMPOSER_HTACCESS_PROTECT = "0"; diff --git a/pkgs/build-support/php/build-pecl.nix b/pkgs/build-support/php/build-pecl.nix index 389447e066fa2..6f38a668f3a34 100644 --- a/pkgs/build-support/php/build-pecl.nix +++ b/pkgs/build-support/php/build-pecl.nix @@ -8,10 +8,9 @@ , nativeBuildInputs ? [ ] , postPhpize ? "" , makeFlags ? [ ] -, src ? fetchurl { +, src ? fetchurl ({ url = "https://pecl.php.net/get/${pname}-${version}.tgz"; - inherit (args) sha256; - } + } // lib.filterAttrs (attrName: _: lib.elem attrName [ "sha256" "hash" ]) args) , passthru ? { } , ... }@args: diff --git a/pkgs/build-support/php/hooks/composer-install-hook.sh b/pkgs/build-support/php/hooks/composer-install-hook.sh index bb6cb47e861b9..6e7fb5d7503bf 100644 --- a/pkgs/build-support/php/hooks/composer-install-hook.sh +++ b/pkgs/build-support/php/hooks/composer-install-hook.sh @@ -22,13 +22,49 @@ composerInstallConfigureHook() { fi if [[ ! -f "composer.lock" ]]; then - echo "No composer.lock file found, consider adding one to your repository to ensure reproducible builds." + composer \ + --no-ansi \ + --no-install \ + --no-interaction \ + ${composerNoDev:+--no-dev} \ + ${composerNoPlugins:+--no-plugins} \ + ${composerNoScripts:+--no-scripts} \ + update + + mkdir -p $out + cp composer.lock $out/ + + echo + echo -e "\e[31mERROR: No composer.lock found\e[0m" + echo + echo -e '\e[31mNo composer.lock file found, consider adding one to your repository to ensure reproducible builds.\e[0m' + echo -e "\e[31mIn the meantime, a composer.lock file has been generated for you in $out/composer.lock\e[0m" + echo + echo -e '\e[31mTo fix the issue:\e[0m' + echo -e "\e[31m1. Copy the composer.lock file from $out/composer.lock to the project's source:\e[0m" + echo -e "\e[31m cp $out/composer.lock <path>\e[0m" + echo -e '\e[31m2. Add the composerLock attribute, pointing to the copied composer.lock file:\e[0m' + echo -e '\e[31m composerLock = ./composer.lock;\e[0m' + echo - if [[ -f "${composerRepository}/composer.lock" ]]; then - cp ${composerRepository}/composer.lock composer.lock - fi + exit 1 + fi + + echo "Validating consistency between composer.lock and ${composerRepository}/composer.lock" + if ! @cmp@ -s "composer.lock" "${composerRepository}/composer.lock"; then + echo + echo -e "\e[31mERROR: vendorHash is out of date\e[0m" + echo + echo -e "\e[31mcomposer.lock is not the same in $composerRepository\e[0m" + echo + echo -e "\e[31mTo fix the issue:\e[0m" + echo -e '\e[31m1. Set vendorHash to an empty string: `vendorHash = "";`\e[0m' + echo -e '\e[31m2. Build the derivation and wait for it to fail with a hash mismatch\e[0m' + echo -e '\e[31m3. Copy the "got: sha256-..." value back into the vendorHash field\e[0m' + echo -e '\e[31m You should have: vendorHash = "sha256-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX=";\e[0m' + echo - echo "Using an autogenerated composer.lock file." + exit 1 fi chmod +w composer.json composer.lock @@ -41,11 +77,11 @@ composerInstallBuildHook() { # Since this file cannot be generated in the composer-repository-hook.sh # because the file contains hardcoded nix store paths, we generate it here. - composer-local-repo-plugin --no-ansi build-local-repo -p "${composerRepository}" > packages.json + composer-local-repo-plugin --no-ansi build-local-repo -m "${composerRepository}" . - # Remove all the repositories of type "composer" + # Remove all the repositories of type "composer" and "vcs" # from the composer.json file. - jq -r -c 'del(try .repositories[] | select(.type == "composer"))' composer.json | sponge composer.json + jq -r -c 'del(try .repositories[] | select(.type == "composer" or .type == "vcs"))' composer.json | sponge composer.json # Configure composer to disable packagist and avoid using the network. composer config repo.packagist false @@ -54,7 +90,6 @@ composerInstallBuildHook() { # Since the composer.json file has been modified in the previous step, the # composer.lock file needs to be updated. - COMPOSER_DISABLE_NETWORK=1 \ COMPOSER_ROOT_VERSION="${version}" \ composer \ --lock \ @@ -72,7 +107,26 @@ composerInstallBuildHook() { composerInstallCheckHook() { echo "Executing composerInstallCheckHook" - composer validate --no-ansi --no-interaction + if ! composer validate --strict --no-ansi --no-interaction --quiet; then + if [ ! -z "${composerStrictValidation-}" ]; then + echo + echo -e "\e[31mERROR: composer files validation failed\e[0m" + echo + echo -e '\e[31mThe validation of the composer.json and composer.lock failed.\e[0m' + echo -e '\e[31mMake sure that the file composer.lock is consistent with composer.json.\e[0m' + echo + exit 1 + else + echo + echo -e "\e[33mWARNING: composer files validation failed\e[0m" + echo + echo -e '\e[33mThe validation of the composer.json and composer.lock failed.\e[0m' + echo -e '\e[33mMake sure that the file composer.lock is consistent with composer.json.\e[0m' + echo + echo -e '\e[33mThis check is not blocking, but it is recommended to fix the issue.\e[0m' + echo + fi + fi echo "Finished composerInstallCheckHook" } @@ -84,10 +138,7 @@ composerInstallInstallHook() { # the autoloader. # The COMPOSER_ROOT_VERSION environment variable is needed only for # vimeo/psalm. - COMPOSER_CACHE_DIR=/dev/null \ - COMPOSER_DISABLE_NETWORK=1 \ COMPOSER_ROOT_VERSION="${version}" \ - COMPOSER_MIRROR_PATH_REPOS="1" \ composer \ --no-ansi \ --no-interaction \ @@ -104,7 +155,7 @@ composerInstallInstallHook() { cp -r . "$out"/share/php/"${pname}"/ # Create symlinks for the binaries. - jq -r -c 'try .bin[]' composer.json | while read -r bin; do + jq -r -c 'try (.bin[] | select(test(".bat$")? | not) )' composer.json | while read -r bin; do mkdir -p "$out"/share/php/"${pname}" "$out"/bin makeWrapper "$out"/share/php/"${pname}"/"$bin" "$out"/bin/"$(basename "$bin")" done diff --git a/pkgs/build-support/php/hooks/composer-repository-hook.sh b/pkgs/build-support/php/hooks/composer-repository-hook.sh index 057acf1fcc30e..03783d9d639c6 100644 --- a/pkgs/build-support/php/hooks/composer-repository-hook.sh +++ b/pkgs/build-support/php/hooks/composer-repository-hook.sh @@ -3,6 +3,7 @@ declare version declare composerNoDev declare composerNoPlugins declare composerNoScripts +declare composerStrictValidation preConfigureHooks+=(composerRepositoryConfigureHook) preBuildHooks+=(composerRepositoryBuildHook) @@ -17,7 +18,7 @@ composerRepositoryConfigureHook() { fi if [[ ! -f "composer.lock" ]]; then - echo "No composer.lock file found, consider adding one to your repository to ensure reproducible builds." + COMPOSER_ROOT_VERSION="${version}" \ composer \ --no-ansi \ --no-install \ @@ -26,7 +27,24 @@ composerRepositoryConfigureHook() { ${composerNoPlugins:+--no-plugins} \ ${composerNoScripts:+--no-scripts} \ update - echo "Using an autogenerated composer.lock file." + + mkdir -p $out + cp composer.lock $out/ + + echo + echo -e "\e[31mERROR: No composer.lock found\e[0m" + echo + echo -e '\e[31mNo composer.lock file found, consider adding one to your repository to ensure reproducible builds.\e[0m' + echo -e "\e[31mIn the meantime, a composer.lock file has been generated for you in $out/composer.lock\e[0m" + echo + echo -e '\e[31mTo fix the issue:\e[0m' + echo -e "\e[31m1. Copy the composer.lock file from $out/composer.lock to the project's source:\e[0m" + echo -e "\e[31m cp $out/composer.lock <path>\e[0m" + echo -e '\e[31m2. Add the composerLock attribute, pointing to the copied composer.lock file:\e[0m' + echo -e '\e[31m composerLock = ./composer.lock;\e[0m' + echo + + exit 1 fi echo "Finished composerRepositoryConfigureHook" @@ -40,7 +58,6 @@ composerRepositoryBuildHook() { # Build the local composer repository # The command 'build-local-repo' is provided by the Composer plugin # nix-community/composer-local-repo-plugin. - COMPOSER_CACHE_DIR=/dev/null \ composer-local-repo-plugin --no-ansi build-local-repo ${composerNoDev:+--no-dev} -r repository echo "Finished composerRepositoryBuildHook" @@ -49,7 +66,26 @@ composerRepositoryBuildHook() { composerRepositoryCheckHook() { echo "Executing composerRepositoryCheckHook" - composer validate --no-ansi --no-interaction + if ! composer validate --strict --no-ansi --no-interaction --quiet; then + if [ ! -z "${composerStrictValidation-}" ]; then + echo + echo -e "\e[31mERROR: composer files validation failed\e[0m" + echo + echo -e '\e[31mThe validation of the composer.json and composer.lock failed.\e[0m' + echo -e '\e[31mMake sure that the file composer.lock is consistent with composer.json.\e[0m' + echo + exit 1 + else + echo + echo -e "\e[33mWARNING: composer files validation failed\e[0m" + echo + echo -e '\e[33mThe validation of the composer.json and composer.lock failed.\e[0m' + echo -e '\e[33mMake sure that the file composer.lock is consistent with composer.json.\e[0m' + echo + echo -e '\e[33mThis check is not blocking, but it is recommended to fix the issue.\e[0m' + echo + fi + fi echo "Finished composerRepositoryCheckHook" } @@ -61,8 +97,8 @@ composerRepositoryInstallHook() { cp -ar repository/. $out/ - # Copy the composer.lock files to the output directory, in case it has been - # autogenerated. + # Copy the composer.lock files to the output directory, to be able to validate consistency with + # the src composer.lock file where this fixed-output derivation is used cp composer.lock $out/ echo "Finished composerRepositoryInstallHook" diff --git a/pkgs/build-support/php/hooks/default.nix b/pkgs/build-support/php/hooks/default.nix index e7de98647c397..240ec640723ad 100644 --- a/pkgs/build-support/php/hooks/default.nix +++ b/pkgs/build-support/php/hooks/default.nix @@ -1,22 +1,29 @@ -{ makeSetupHook +{ lib +, makeSetupHook +, diffutils , jq , moreutils , makeBinaryWrapper -, php +, cacert +, buildPackages }: { composerRepositoryHook = makeSetupHook { name = "composer-repository-hook.sh"; - propagatedBuildInputs = [ jq moreutils php ]; + propagatedBuildInputs = [ jq moreutils cacert ]; substitutions = { }; } ./composer-repository-hook.sh; composerInstallHook = makeSetupHook { name = "composer-install-hook.sh"; - propagatedBuildInputs = [ jq makeBinaryWrapper moreutils php ]; - substitutions = { }; + propagatedBuildInputs = [ jq makeBinaryWrapper moreutils cacert ]; + substitutions = { + # Specify the stdenv's `diff` by abspath to ensure that the user's build + # inputs do not cause us to find the wrong `diff`. + cmp = "${lib.getBin buildPackages.diffutils}/bin/cmp"; + }; } ./composer-install-hook.sh; } diff --git a/pkgs/build-support/php/pkgs/composer-local-repo-plugin.nix b/pkgs/build-support/php/pkgs/composer-local-repo-plugin.nix index 67edbf1f44f9a..48d05b7a00089 100644 --- a/pkgs/build-support/php/pkgs/composer-local-repo-plugin.nix +++ b/pkgs/build-support/php/pkgs/composer-local-repo-plugin.nix @@ -1,7 +1,9 @@ -{ callPackage, stdenvNoCC, lib, fetchFromGitHub, makeBinaryWrapper }: +{ php, callPackage, stdenvNoCC, lib, fetchFromGitHub, makeBinaryWrapper }: let - composer = callPackage ./composer-phar.nix { }; + composer = callPackage ./composer-phar.nix { + inherit (php.packages.composer) version pharHash; + }; composerKeys = stdenvNoCC.mkDerivation (finalComposerKeysAttrs: { pname = "composer-keys"; @@ -27,13 +29,13 @@ let in stdenvNoCC.mkDerivation (finalAttrs: { pname = "composer-local-repo-plugin"; - version = "1.0.0"; + version = "1.0.3"; src = fetchFromGitHub { owner = "nix-community"; repo = "composer-local-repo-plugin"; rev = finalAttrs.version; - hash = "sha256-sjWV4JXK8YJ5XLASMPipKlk9u57352wIDV2PPFIP+sk="; + hash = "sha256-fLJlxcAQ7X28GDK8PVYKxJgTzbspfWxvgRmRK4NZRIA="; }; COMPOSER_CACHE_DIR = "/dev/null"; @@ -69,7 +71,6 @@ stdenvNoCC.mkDerivation (finalAttrs: { composer global config --quiet minimum-stability dev composer global config --quiet prefer-stable true - composer global config --quiet autoloader-suffix "nixPredictableAutoloaderSuffix" composer global config --quiet apcu-autoloader false composer global config --quiet allow-plugins.nix-community/composer-local-repo-plugin true composer global config --quiet repo.packagist false diff --git a/pkgs/build-support/php/pkgs/composer-phar.nix b/pkgs/build-support/php/pkgs/composer-phar.nix index 41cba03f4f5d6..f281334ab2d9f 100644 --- a/pkgs/build-support/php/pkgs/composer-phar.nix +++ b/pkgs/build-support/php/pkgs/composer-phar.nix @@ -10,15 +10,17 @@ , stdenvNoCC , unzip , xz + , version + , pharHash }: stdenvNoCC.mkDerivation (finalAttrs: { pname = "composer-phar"; - version = "2.6.2"; + inherit version; src = fetchurl { url = "https://github.com/composer/composer/releases/download/${finalAttrs.version}/composer.phar"; - hash = "sha256-iMhNSlP88cJ9Z2Lh1da3DVfG3J0uIxT9Cdv4a/YeGu8="; + hash = pharHash; }; dontUnpack = true; diff --git a/pkgs/build-support/prefer-remote-fetch/default.nix b/pkgs/build-support/prefer-remote-fetch/default.nix index a1f2d0c56cffe..3257e7000fe38 100644 --- a/pkgs/build-support/prefer-remote-fetch/default.nix +++ b/pkgs/build-support/prefer-remote-fetch/default.nix @@ -11,9 +11,16 @@ # $ echo 'self: super: super.prefer-remote-fetch self super' > ~/.config/nixpkgs/overlays/prefer-remote-fetch.nix # self: super: { - fetchurl = args: super.fetchurl ({ preferLocalBuild = false; } // args); + binary-cache = args: super.binary-cache ({ preferLocalBuild = false; } // args); + buildenv = args: super.buildenv ({ preferLocalBuild = false; } // args); + fetchfossil = args: super.fetchfossil ({ preferLocalBuild = false; } // args); + fetchdocker = args: super.fetchdocker ({ preferLocalBuild = false; } // args); fetchgit = args: super.fetchgit ({ preferLocalBuild = false; } // args); + fetchgx = args: super.fetchgx ({ preferLocalBuild = false; } // args); fetchhg = args: super.fetchhg ({ preferLocalBuild = false; } // args); - fetchsvn = args: super.fetchsvn ({ preferLocalBuild = false; } // args); fetchipfs = args: super.fetchipfs ({ preferLocalBuild = false; } // args); + fetchrepoproject = args: super.fetchrepoproject ({ preferLocalBuild = false; } // args); + fetchs3 = args: super.fetchs3 ({ preferLocalBuild = false; } // args); + fetchsvn = args: super.fetchsvn ({ preferLocalBuild = false; } // args); + fetchurl = args: super.fetchurl ({ preferLocalBuild = false; } // args); } diff --git a/pkgs/build-support/references-by-popularity/default.nix b/pkgs/build-support/references-by-popularity/default.nix index dfc25275f34c5..2171c622f000a 100644 --- a/pkgs/build-support/references-by-popularity/default.nix +++ b/pkgs/build-support/references-by-popularity/default.nix @@ -6,11 +6,8 @@ path: runCommand "closure-paths" exportReferencesGraph.graph = path; __structuredAttrs = true; preferLocalBuild = true; - PATH = "${coreutils}/bin:${python3}/bin"; - builder = builtins.toFile "builder" - '' - . .attrs.sh - python3 ${./closure-graph.py} .attrs.json graph > ''${outputs[out]} - ''; - } - "" + nativeBuildInputs = [ coreutils python3 ]; +} +'' + python3 ${./closure-graph.py} "$NIX_ATTRS_JSON_FILE" graph > ''${outputs[out]} +'' diff --git a/pkgs/build-support/release/default.nix b/pkgs/build-support/release/default.nix index d09f6c8568be7..1cc6a5812f1f5 100644 --- a/pkgs/build-support/release/default.nix +++ b/pkgs/build-support/release/default.nix @@ -88,7 +88,8 @@ rec { preferLocalBuild = true; _hydraAggregate = true; - phases = [ "unpackPhase" "patchPhase" "installPhase" ]; + dontConfigure = true; + dontBuild = true; patchPhase = lib.optionalString isNixOS '' touch .update-on-nixos-rebuild diff --git a/pkgs/build-support/replace-secret/replace-secret.nix b/pkgs/build-support/replace-secret/replace-secret.nix index 4881ba25f5d27..41f5cb042558c 100644 --- a/pkgs/build-support/replace-secret/replace-secret.nix +++ b/pkgs/build-support/replace-secret/replace-secret.nix @@ -32,5 +32,6 @@ stdenv.mkDerivation { Since the secret is read from a file, it won't be leaked through '/proc/<pid>/cmdline', unlike when 'sed' or 'replace' is used. ''; + mainProgram = "replace-secret"; }; } diff --git a/pkgs/build-support/rust/build-rust-crate/build-crate.nix b/pkgs/build-support/rust/build-rust-crate/build-crate.nix index 37bf3ec26f770..bbb26606a6a4d 100644 --- a/pkgs/build-support/rust/build-rust-crate/build-crate.nix +++ b/pkgs/build-support/rust/build-rust-crate/build-crate.nix @@ -1,6 +1,5 @@ { lib, stdenv , mkRustcDepArgs, mkRustcFeatureArgs, needUnstableCLI -, rust }: { crateName, @@ -21,7 +20,7 @@ (mkRustcDepArgs dependencies crateRenames) (mkRustcFeatureArgs crateFeatures) ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ - "--target" (rust.toRustTargetSpec stdenv.hostPlatform) + "--target" stdenv.hostPlatform.rust.rustcTargetSpec ] ++ lib.optionals (needUnstableCLI dependencies) [ "-Z" "unstable-options" ] ++ extraRustcOpts @@ -41,6 +40,7 @@ ); binRustcOpts = lib.concatStringsSep " " ( + [ "-C linker=${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc" ] ++ baseRustcOpts ); @@ -51,7 +51,7 @@ # configure & source common build functions LIB_RUSTC_OPTS="${libRustcOpts}" BIN_RUSTC_OPTS="${binRustcOpts}" - LIB_EXT="${stdenv.hostPlatform.extensions.sharedLibrary}" + LIB_EXT="${stdenv.hostPlatform.extensions.sharedLibrary or ""}" LIB_PATH="${libPath}" LIB_NAME="${libName}" diff --git a/pkgs/build-support/rust/build-rust-crate/configure-crate.nix b/pkgs/build-support/rust/build-rust-crate/configure-crate.nix index 60310f178747e..6b88271602623 100644 --- a/pkgs/build-support/rust/build-rust-crate/configure-crate.nix +++ b/pkgs/build-support/rust/build-rust-crate/configure-crate.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, rust, echo_colored, noisily, mkRustcDepArgs, mkRustcFeatureArgs }: +{ lib, stdenv, echo_colored, noisily, mkRustcDepArgs, mkRustcFeatureArgs }: { build , buildDependencies @@ -8,10 +8,16 @@ , completeDeps , crateAuthors , crateDescription -, crateHomepage , crateFeatures +, crateHomepage +, crateLicense +, crateLicenseFile +, crateLinks , crateName +, crateReadme , crateRenames +, crateRepository +, crateRustVersion , crateVersion , extraLinkFlags , extraRustcOptsForBuildRs @@ -119,13 +125,15 @@ in '' EXTRA_BUILD="" BUILD_OUT_DIR="" + + # Set up Cargo Environment variables: https://doc.rust-lang.org/cargo/reference/environment-variables.html export CARGO_PKG_NAME=${crateName} export CARGO_PKG_VERSION=${crateVersion} export CARGO_PKG_AUTHORS="${authors}" export CARGO_PKG_DESCRIPTION="${crateDescription}" - export CARGO_CFG_TARGET_ARCH=${rust.toTargetArch stdenv.hostPlatform} - export CARGO_CFG_TARGET_OS=${rust.toTargetOs stdenv.hostPlatform} + export CARGO_CFG_TARGET_ARCH=${stdenv.hostPlatform.rust.platform.arch} + export CARGO_CFG_TARGET_OS=${stdenv.hostPlatform.rust.platform.os} export CARGO_CFG_TARGET_FAMILY="unix" export CARGO_CFG_UNIX=1 export CARGO_CFG_TARGET_ENV="gnu" @@ -134,10 +142,11 @@ in '' export CARGO_CFG_TARGET_VENDOR=${stdenv.hostPlatform.parsed.vendor.name} export CARGO_MANIFEST_DIR=$(pwd) + export CARGO_MANIFEST_LINKS=${crateLinks} export DEBUG="${toString (!release)}" export OPT_LEVEL="${toString optLevel}" - export TARGET="${rust.toRustTargetSpec stdenv.hostPlatform}" - export HOST="${rust.toRustTargetSpec stdenv.buildPlatform}" + export TARGET="${stdenv.hostPlatform.rust.rustcTargetSpec}" + export HOST="${stdenv.buildPlatform.rust.rustcTargetSpec}" export PROFILE=${if release then "release" else "debug"} export OUT_DIR=$(pwd)/target/build/${crateName}.out export CARGO_PKG_VERSION_MAJOR=${lib.elemAt version 0} @@ -145,6 +154,11 @@ in '' export CARGO_PKG_VERSION_PATCH=${lib.elemAt version 2} export CARGO_PKG_VERSION_PRE="${versionPre}" export CARGO_PKG_HOMEPAGE="${crateHomepage}" + export CARGO_PKG_LICENSE="${crateLicense}" + export CARGO_PKG_LICENSE_FILE="${crateLicenseFile}" + export CARGO_PKG_README="${crateReadme}" + export CARGO_PKG_REPOSITORY="${crateRepository}" + export CARGO_PKG_RUST_VERSION="${crateRustVersion}" export NUM_JOBS=$NIX_BUILD_CORES export RUSTC="rustc" export RUSTDOC="rustdoc" diff --git a/pkgs/build-support/rust/build-rust-crate/default.nix b/pkgs/build-support/rust/build-rust-crate/default.nix index f6079b7316d61..4a7fd114829ad 100644 --- a/pkgs/build-support/rust/build-rust-crate/default.nix +++ b/pkgs/build-support/rust/build-rust-crate/default.nix @@ -10,7 +10,6 @@ , fetchCrate , pkgsBuildBuild , rustc -, rust , cargo , jq , libiconv @@ -71,18 +70,14 @@ let inherit (import ./log.nix { inherit lib; }) noisily echo_colored; configureCrate = import ./configure-crate.nix { - inherit lib stdenv rust echo_colored noisily mkRustcDepArgs mkRustcFeatureArgs; + inherit lib stdenv echo_colored noisily mkRustcDepArgs mkRustcFeatureArgs; }; buildCrate = import ./build-crate.nix { - inherit lib stdenv mkRustcDepArgs mkRustcFeatureArgs needUnstableCLI rust; + inherit lib stdenv mkRustcDepArgs mkRustcFeatureArgs needUnstableCLI; }; installCrate = import ./install-crate.nix { inherit stdenv; }; - - # Allow access to the rust attribute set from inside buildRustCrate, which - # has a parameter that shadows the name. - rustAttrs = rust; in /* The overridable pkgs.buildRustCrate function. @@ -240,6 +235,7 @@ crate_: lib.makeOverridable "edition" "buildTests" "codegenUnits" + "links" ]; extraDerivationAttrs = builtins.removeAttrs crate processedAttrs; nativeBuildInputs_ = nativeBuildInputs; @@ -310,7 +306,7 @@ crate_: lib.makeOverridable depsMetadata = lib.foldl' (str: dep: str + dep.metadata) "" (dependencies ++ buildDependencies); hashedMetadata = builtins.hashString "sha256" (crateName + "-" + crateVersion + "___" + toString (mkRustcFeatureArgs crateFeatures) + - "___" + depsMetadata + "___" + rustAttrs.toRustTarget stdenv.hostPlatform); + "___" + depsMetadata + "___" + stdenv.hostPlatform.rust.rustcTarget); in lib.substring 0 10 hashedMetadata; @@ -318,10 +314,16 @@ crate_: lib.makeOverridable # Either set to a concrete sub path to the crate root # or use `null` for auto-detect. workspace_member = crate.workspace_member or "."; - crateVersion = crate.version; - crateDescription = crate.description or ""; crateAuthors = if crate ? authors && lib.isList crate.authors then crate.authors else [ ]; + crateDescription = crate.description or ""; crateHomepage = crate.homepage or ""; + crateLicense = crate.license or ""; + crateLicenseFile = crate.license-file or ""; + crateLinks = crate.links or ""; + crateReadme = crate.readme or ""; + crateRepository = crate.repository or ""; + crateRustVersion = crate.rust-version or ""; + crateVersion = crate.version; crateType = if lib.attrByPath [ "procMacro" ] false crate then [ "proc-macro" ] else if lib.attrByPath [ "plugin" ] false crate then [ "dylib" ] else @@ -342,8 +344,9 @@ crate_: lib.makeOverridable configurePhase = configureCrate { inherit crateName buildDependencies completeDeps completeBuildDeps crateDescription - crateFeatures crateRenames libName build workspace_member release libPath crateVersion + crateFeatures crateRenames libName build workspace_member release libPath crateVersion crateLinks extraLinkFlags extraRustcOptsForBuildRs + crateLicense crateLicenseFile crateReadme crateRepository crateRustVersion crateAuthors crateHomepage verbose colors codegenUnits; }; buildPhase = buildCrate { @@ -353,6 +356,10 @@ crate_: lib.makeOverridable extraRustcOpts buildTests codegenUnits; }; dontStrip = !release; + + # We need to preserve metadata in .rlib, which might get stripped on macOS. See https://github.com/NixOS/nixpkgs/issues/218712 + stripExclude = [ "*.rlib" ]; + installPhase = installCrate crateName metadata buildTests; # depending on the test setting we are either producing something with bins @@ -362,6 +369,10 @@ crate_: lib.makeOverridable meta = { mainProgram = crateName; + badPlatforms = [ + # Rust is currently unable to target the n32 ABI + lib.systems.inspect.patterns.isMips64n32 + ]; }; } // extraDerivationAttrs ) diff --git a/pkgs/build-support/rust/build-rust-package/default.nix b/pkgs/build-support/rust/build-rust-package/default.nix index 9b2e62fc240a6..cd90c68c79303 100644 --- a/pkgs/build-support/rust/build-rust-package/default.nix +++ b/pkgs/build-support/rust/build-rust-package/default.nix @@ -1,7 +1,6 @@ { lib , importCargoLock , fetchCargoTarball -, rust , stdenv , callPackage , cargoBuildHook @@ -45,7 +44,8 @@ , buildFeatures ? [ ] , checkFeatures ? buildFeatures , useNextest ? false -, auditable ? !cargo-auditable.meta.broken +# Enable except on aarch64 pkgsStatic, where we use lld for reasons +, auditable ? !cargo-auditable.meta.broken && !(stdenv.hostPlatform.isStatic && stdenv.hostPlatform.isAarch64 && !stdenv.hostPlatform.isDarwin) , depsExtraArgs ? {} @@ -62,7 +62,6 @@ assert cargoVendorDir == null && cargoLock == null -> !(args ? cargoSha256 && args.cargoSha256 != null) && !(args ? cargoHash && args.cargoHash != null) -> throw "cargoSha256, cargoHash, cargoVendorDir, or cargoLock must be set"; -assert buildType == "release" || buildType == "debug"; let @@ -79,18 +78,13 @@ let sha256 = args.cargoSha256; } // depsExtraArgs); - target = rust.toRustTargetSpec stdenv.hostPlatform; + target = stdenv.hostPlatform.rust.rustcTargetSpec; targetIsJSON = lib.hasSuffix ".json" target; useSysroot = targetIsJSON && !__internal_dontAddSysroot; - # see https://github.com/rust-lang/cargo/blob/964a16a28e234a3d397b2a7031d4ab4a428b1391/src/cargo/core/compiler/compile_kind.rs#L151-L168 - # the "${}" is needed to transform the path into a /nix/store path before baseNameOf - shortTarget = if targetIsJSON then - (lib.removeSuffix ".json" (builtins.baseNameOf "${target}")) - else target; - sysroot = callPackage ./sysroot { } { - inherit target shortTarget; + inherit target; + shortTarget = stdenv.hostPlatform.rust.cargoShortTarget; RUSTFLAGS = args.RUSTFLAGS or ""; originalCargoToml = src + /Cargo.toml; # profile info is later extracted }; @@ -162,10 +156,15 @@ stdenv.mkDerivation ((removeAttrs args [ "depsExtraArgs" "cargoUpdateHook" "carg # Platforms without host tools from # https://doc.rust-lang.org/nightly/rustc/platform-support.html "armv7a-darwin" - "armv5tel-linux" "armv7a-linux" "m68k-linux" "riscv32-linux" + "armv5tel-linux" "armv7a-linux" "m68k-linux" "mipsel-linux" + "mips64el-linux" "riscv32-linux" "armv6l-netbsd" "x86_64-redox" "wasm32-wasi" ]; + badPlatforms = [ + # Rust is currently unable to target the n32 ABI + lib.systems.inspect.patterns.isMips64n32 + ]; } // meta; }) diff --git a/pkgs/build-support/rust/build-rust-package/sysroot/default.nix b/pkgs/build-support/rust/build-rust-package/sysroot/default.nix index a6d53056d9c7c..bb95b7bdc35cd 100644 --- a/pkgs/build-support/rust/build-rust-package/sysroot/default.nix +++ b/pkgs/build-support/rust/build-rust-package/sysroot/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, rust, rustPlatform, buildPackages }: +{ lib, stdenv, rustPlatform, buildPackages }: { shortTarget, originalCargoToml, target, RUSTFLAGS }: @@ -26,7 +26,7 @@ in rustPlatform.buildRustPackage { done export RUST_SYSROOT=$(rustc --print=sysroot) - host=${rust.toRustTarget stdenv.buildPlatform} + host=${stdenv.buildPlatform.rust.rustcTarget} cp -r $RUST_SYSROOT/lib/rustlib/$host $out ''; diff --git a/pkgs/build-support/rust/hooks/cargo-build-hook.sh b/pkgs/build-support/rust/hooks/cargo-build-hook.sh index af94e02e38ae7..ed982c7ff30a3 100644 --- a/pkgs/build-support/rust/hooks/cargo-build-hook.sh +++ b/pkgs/build-support/rust/hooks/cargo-build-hook.sh @@ -17,7 +17,7 @@ cargoBuildHook() { fi if [ "${cargoBuildType}" != "debug" ]; then - cargoBuildProfileFlag="--${cargoBuildType}" + cargoBuildProfileFlag="--profile ${cargoBuildType}" fi if [ -n "${cargoBuildNoDefaultFeatures-}" ]; then @@ -30,13 +30,8 @@ cargoBuildHook() { ( set -x - env \ - "CC_@rustBuildPlatform@=@ccForBuild@" \ - "CXX_@rustBuildPlatform@=@cxxForBuild@" \ - "CC_@rustTargetPlatform@=@ccForHost@" \ - "CXX_@rustTargetPlatform@=@cxxForHost@" \ - cargo build -j $NIX_BUILD_CORES \ - --target @rustTargetPlatformSpec@ \ + @setEnv@ cargo build -j $NIX_BUILD_CORES \ + --target @rustHostPlatformSpec@ \ --frozen \ ${cargoBuildProfileFlag} \ ${cargoBuildNoDefaultFeaturesFlag} \ diff --git a/pkgs/build-support/rust/hooks/cargo-check-hook.sh b/pkgs/build-support/rust/hooks/cargo-check-hook.sh index 57fc2779cfe96..971a140ec178f 100644 --- a/pkgs/build-support/rust/hooks/cargo-check-hook.sh +++ b/pkgs/build-support/rust/hooks/cargo-check-hook.sh @@ -17,7 +17,7 @@ cargoCheckHook() { fi if [ "${cargoCheckType}" != "debug" ]; then - cargoCheckProfileFlag="--${cargoCheckType}" + cargoCheckProfileFlag="--profile ${cargoCheckType}" fi if [ -n "${cargoCheckNoDefaultFeatures-}" ]; then @@ -29,7 +29,7 @@ cargoCheckHook() { fi argstr="${cargoCheckProfileFlag} ${cargoCheckNoDefaultFeaturesFlag} ${cargoCheckFeaturesFlag} - --target @rustTargetPlatformSpec@ --frozen ${cargoTestFlags}" + --target @rustHostPlatformSpec@ --frozen ${cargoTestFlags}" ( set -x diff --git a/pkgs/build-support/rust/hooks/cargo-install-hook.sh b/pkgs/build-support/rust/hooks/cargo-install-hook.sh index 69ce726693665..24a6e6fa9eb3f 100644 --- a/pkgs/build-support/rust/hooks/cargo-install-hook.sh +++ b/pkgs/build-support/rust/hooks/cargo-install-hook.sh @@ -1,7 +1,7 @@ cargoInstallPostBuildHook() { echo "Executing cargoInstallPostBuildHook" - releaseDir=target/@shortTarget@/$cargoBuildType + releaseDir=target/@targetSubdirectory@/$cargoBuildType tmpDir="${releaseDir}-tmp"; mkdir -p $tmpDir @@ -21,7 +21,7 @@ cargoInstallHook() { # rename the output dir to a architecture independent one - releaseDir=target/@shortTarget@/$cargoBuildType + releaseDir=target/@targetSubdirectory@/$cargoBuildType tmpDir="${releaseDir}-tmp"; mapfile -t targets < <(find "$NIX_BUILD_TOP" -type d | grep "${tmpDir}$") diff --git a/pkgs/build-support/rust/hooks/cargo-nextest-hook.sh b/pkgs/build-support/rust/hooks/cargo-nextest-hook.sh index de85683ead2ac..29ba18a6a1e3f 100644 --- a/pkgs/build-support/rust/hooks/cargo-nextest-hook.sh +++ b/pkgs/build-support/rust/hooks/cargo-nextest-hook.sh @@ -17,7 +17,7 @@ cargoNextestHook() { fi if [ "${cargoCheckType}" != "debug" ]; then - cargoCheckProfileFlag="--${cargoCheckType}" + cargoCheckProfileFlag="--cargo-profile ${cargoCheckType}" fi if [ -n "${cargoCheckNoDefaultFeatures-}" ]; then @@ -29,7 +29,7 @@ cargoNextestHook() { fi argstr="${cargoCheckProfileFlag} ${cargoCheckNoDefaultFeaturesFlag} ${cargoCheckFeaturesFlag} - --target @rustTargetPlatformSpec@ --frozen ${cargoTestFlags}" + --target @rustHostPlatformSpec@ --frozen ${cargoTestFlags}" ( set -x diff --git a/pkgs/build-support/rust/hooks/default.nix b/pkgs/build-support/rust/hooks/default.nix index 2eb388fe07ba8..874f23fe7ed39 100644 --- a/pkgs/build-support/rust/hooks/default.nix +++ b/pkgs/build-support/rust/hooks/default.nix @@ -9,32 +9,20 @@ , rust , rustc , stdenv -, target ? rust.toRustTargetSpec stdenv.hostPlatform -}: -let - targetIsJSON = lib.hasSuffix ".json" target; +# This confusingly-named parameter indicates the *subdirectory of +# `target/` from which to copy the build artifacts. It is derived +# from a stdenv platform (or a JSON file). +, target ? stdenv.hostPlatform.rust.cargoShortTarget +}: - # see https://github.com/rust-lang/cargo/blob/964a16a28e234a3d397b2a7031d4ab4a428b1391/src/cargo/core/compiler/compile_kind.rs#L151-L168 - # the "${}" is needed to transform the path into a /nix/store path before baseNameOf - shortTarget = if targetIsJSON then - (lib.removeSuffix ".json" (builtins.baseNameOf "${target}")) - else target; - ccForBuild = "${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc"; - cxxForBuild = "${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}c++"; - ccForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc"; - cxxForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++"; - rustBuildPlatform = rust.toRustTarget stdenv.buildPlatform; - rustTargetPlatform = rust.toRustTarget stdenv.hostPlatform; - rustTargetPlatformSpec = rust.toRustTargetSpec stdenv.hostPlatform; -in { +{ cargoBuildHook = callPackage ({ }: makeSetupHook { name = "cargo-build-hook.sh"; propagatedBuildInputs = [ cargo ]; substitutions = { - inherit ccForBuild ccForHost cxxForBuild cxxForHost - rustBuildPlatform rustTargetPlatform rustTargetPlatformSpec; + inherit (rust.envVars) rustHostPlatformSpec setEnv; }; } ./cargo-build-hook.sh) {}; @@ -43,7 +31,7 @@ in { name = "cargo-check-hook.sh"; propagatedBuildInputs = [ cargo ]; substitutions = { - inherit rustTargetPlatformSpec; + inherit (rust.envVars) rustHostPlatformSpec; }; } ./cargo-check-hook.sh) {}; @@ -52,7 +40,7 @@ in { name = "cargo-install-hook.sh"; propagatedBuildInputs = [ ]; substitutions = { - inherit shortTarget; + targetSubdirectory = target; }; } ./cargo-install-hook.sh) {}; @@ -61,7 +49,7 @@ in { name = "cargo-nextest-hook.sh"; propagatedBuildInputs = [ cargo cargo-nextest ]; substitutions = { - inherit rustTargetPlatformSpec; + inherit (rust.envVars) rustHostPlatformSpec; }; } ./cargo-nextest-hook.sh) {}; @@ -77,24 +65,27 @@ in { diff = "${lib.getBin buildPackages.diffutils}/bin/diff"; cargoConfig = '' - [target."${rust.toRustTarget stdenv.buildPlatform}"] - "linker" = "${ccForBuild}" + [target."${stdenv.buildPlatform.rust.rustcTarget}"] + "linker" = "${rust.envVars.linkerForBuild}" ${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) '' - [target."${shortTarget}"] - "linker" = "${ccForHost}" + [target."${stdenv.hostPlatform.rust.rustcTarget}"] + "linker" = "${rust.envVars.linkerForHost}" ''} "rustflags" = [ "-C", "target-feature=${if stdenv.hostPlatform.isStatic then "+" else "-"}crt-static" ] ''; }; } ./cargo-setup-hook.sh) {}; - maturinBuildHook = callPackage ({ }: + maturinBuildHook = callPackage ({ pkgsHostTarget }: makeSetupHook { name = "maturin-build-hook.sh"; - propagatedBuildInputs = [ cargo maturin rustc ]; + propagatedBuildInputs = [ + pkgsHostTarget.maturin + pkgsHostTarget.cargo + pkgsHostTarget.rustc + ]; substitutions = { - inherit ccForBuild ccForHost cxxForBuild cxxForHost - rustBuildPlatform rustTargetPlatform rustTargetPlatformSpec; + inherit (rust.envVars) rustTargetPlatformSpec setEnv; }; } ./maturin-build-hook.sh) {}; diff --git a/pkgs/build-support/rust/hooks/maturin-build-hook.sh b/pkgs/build-support/rust/hooks/maturin-build-hook.sh index 62d5619660c67..028441d18160e 100644 --- a/pkgs/build-support/rust/hooks/maturin-build-hook.sh +++ b/pkgs/build-support/rust/hooks/maturin-build-hook.sh @@ -9,12 +9,7 @@ maturinBuildHook() { ( set -x - env \ - "CC_@rustBuildPlatform@=@ccForBuild@" \ - "CXX_@rustBuildPlatform@=@cxxForBuild@" \ - "CC_@rustTargetPlatform@=@ccForHost@" \ - "CXX_@rustTargetPlatform@=@cxxForHost@" \ - maturin build \ + @setEnv@ maturin build \ --jobs=$NIX_BUILD_CORES \ --frozen \ --target @rustTargetPlatformSpec@ \ @@ -30,7 +25,7 @@ maturinBuildHook() { # Move the wheel to dist/ so that regular Python tooling can find it. mkdir -p dist - mv target/wheels/*.whl dist/ + mv ${cargoRoot:+$cargoRoot/}target/wheels/*.whl dist/ # These are python build hooks and may depend on ./dist runHook postBuild diff --git a/pkgs/build-support/rust/import-cargo-lock.nix b/pkgs/build-support/rust/import-cargo-lock.nix index c17b0e41cca8e..e3fe57ef06daa 100644 --- a/pkgs/build-support/rust/import-cargo-lock.nix +++ b/pkgs/build-support/rust/import-cargo-lock.nix @@ -193,7 +193,7 @@ let if grep -q workspace "$out/Cargo.toml"; then chmod u+w "$out/Cargo.toml" - ${replaceWorkspaceValues} "$out/Cargo.toml" "${tree}/Cargo.toml" + ${replaceWorkspaceValues} "$out/Cargo.toml" "$(${cargo}/bin/cargo metadata --format-version 1 --no-deps --manifest-path $crateCargoTOML | ${jq}/bin/jq -r .workspace_root)/Cargo.toml" fi # Cargo is happy with empty metadata. diff --git a/pkgs/build-support/rust/lib/default.nix b/pkgs/build-support/rust/lib/default.nix index aa5ba14c1397b..e09f913bfbd39 100644 --- a/pkgs/build-support/rust/lib/default.nix +++ b/pkgs/build-support/rust/lib/default.nix @@ -1,66 +1,96 @@ -{ lib }: +{ lib +, stdenv +, pkgsBuildHost +, pkgsBuildTarget +, pkgsTargetTarget +}: rec { - # https://doc.rust-lang.org/reference/conditional-compilation.html#target_arch - toTargetArch = platform: - /**/ if platform ? rustc.platform then platform.rustc.platform.arch - else if platform.isAarch32 then "arm" - else if platform.isMips64 then "mips64" # never add "el" suffix - else if platform.isPower64 then "powerpc64" # never add "le" suffix - else platform.parsed.cpu.name; + # These environment variables must be set when using `cargo-c` and + # several other tools which do not deal well with cross + # compilation. The symptom of the problem they fix is errors due + # to buildPlatform CFLAGS being passed to the + # hostPlatform-targeted compiler -- for example, `-m64` being + # passed on a build=x86_64/host=aarch64 compilation. + envVars = let - # https://doc.rust-lang.org/reference/conditional-compilation.html#target_os - toTargetOs = platform: - /**/ if platform ? rustc.platform then platform.rustc.platform.os or "none" - else if platform.isDarwin then "macos" - else platform.parsed.kernel.name; + # As a workaround for https://github.com/rust-lang/rust/issues/89626 use lld on pkgsStatic aarch64 + shouldUseLLD = platform: platform.isAarch64 && platform.isStatic && !stdenv.isDarwin; - # https://doc.rust-lang.org/reference/conditional-compilation.html#target_family - toTargetFamily = platform: - if platform ? rustc.platform.target-family - then - ( - # Since https://github.com/rust-lang/rust/pull/84072 - # `target-family` is a list instead of single value. - let - f = platform.rustc.platform.target-family; - in - if builtins.isList f then f else [ f ] - ) - else lib.optional platform.isUnix "unix" - ++ lib.optional platform.isWindows "windows"; + ccForBuild = "${pkgsBuildHost.stdenv.cc}/bin/${pkgsBuildHost.stdenv.cc.targetPrefix}cc"; + cxxForBuild = "${pkgsBuildHost.stdenv.cc}/bin/${pkgsBuildHost.stdenv.cc.targetPrefix}c++"; + linkerForBuild = ccForBuild; - # https://doc.rust-lang.org/reference/conditional-compilation.html#target_vendor - toTargetVendor = platform: let - inherit (platform.parsed) vendor; - in platform.rustc.platform.vendor or { - "w64" = "pc"; - }.${vendor.name} or vendor.name; + ccForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc"; + cxxForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++"; + linkerForHost = if shouldUseLLD stdenv.targetPlatform + && !stdenv.cc.bintools.isLLVM + then "${pkgsBuildHost.llvmPackages.bintools}/bin/${stdenv.cc.targetPrefix}ld.lld" + else ccForHost; - # Returns the name of the rust target, even if it is custom. Adjustments are - # because rust has slightly different naming conventions than we do. - toRustTarget = platform: let - inherit (platform.parsed) cpu kernel abi; - cpu_ = platform.rustc.platform.arch or { - "armv7a" = "armv7"; - "armv7l" = "armv7"; - "armv6l" = "arm"; - "armv5tel" = "armv5te"; - "riscv64" = "riscv64gc"; - }.${cpu.name} or cpu.name; - vendor_ = toTargetVendor platform; - in platform.rustc.config - or "${cpu_}-${vendor_}-${kernel.name}${lib.optionalString (abi.name != "unknown") "-${abi.name}"}"; + # Unfortunately we must use the dangerous `pkgsTargetTarget` here + # because hooks are artificially phase-shifted one slot earlier + # (they go in nativeBuildInputs, so the hostPlatform looks like + # a targetPlatform to them). + ccForTarget = "${pkgsTargetTarget.stdenv.cc}/bin/${pkgsTargetTarget.stdenv.cc.targetPrefix}cc"; + cxxForTarget = "${pkgsTargetTarget.stdenv.cc}/bin/${pkgsTargetTarget.stdenv.cc.targetPrefix}c++"; + linkerForTarget = if shouldUseLLD pkgsTargetTarget.stdenv.targetPlatform + && !pkgsTargetTarget.stdenv.cc.bintools.isLLVM # whether stdenv's linker is lld already + then "${pkgsBuildTarget.llvmPackages.bintools}/bin/${pkgsTargetTarget.stdenv.cc.targetPrefix}ld.lld" + else ccForTarget; - # Returns the name of the rust target if it is standard, or the json file - # containing the custom target spec. - toRustTargetSpec = platform: - if platform ? rustc.platform - then builtins.toFile (toRustTarget platform + ".json") (builtins.toJSON platform.rustc.platform) - else toRustTarget platform; + rustBuildPlatform = stdenv.buildPlatform.rust.rustcTarget; + rustBuildPlatformSpec = stdenv.buildPlatform.rust.rustcTargetSpec; + rustHostPlatform = stdenv.hostPlatform.rust.rustcTarget; + rustHostPlatformSpec = stdenv.hostPlatform.rust.rustcTargetSpec; + rustTargetPlatform = stdenv.targetPlatform.rust.rustcTarget; + rustTargetPlatformSpec = stdenv.targetPlatform.rust.rustcTargetSpec; + in { + inherit + ccForBuild cxxForBuild linkerForBuild rustBuildPlatform rustBuildPlatformSpec + ccForHost cxxForHost linkerForHost rustHostPlatform rustHostPlatformSpec + ccForTarget cxxForTarget linkerForTarget rustTargetPlatform rustTargetPlatformSpec; - # Returns true if the target is no_std - # https://github.com/rust-lang/rust/blob/2e44c17c12cec45b6a682b1e53a04ac5b5fcc9d2/src/bootstrap/config.rs#L415-L421 - IsNoStdTarget = platform: let rustTarget = toRustTarget platform; in - builtins.any (t: lib.hasInfix t rustTarget) ["-none" "nvptx" "switch" "-uefi"]; + # Prefix this onto a command invocation in order to set the + # variables needed by cargo. + # + setEnv = '' + env \ + '' + # Due to a bug in how splicing and pkgsTargetTarget works, in + # situations where pkgsTargetTarget is irrelevant + # pkgsTargetTarget.stdenv.cc is often simply wrong. We must omit + # the following lines when rustTargetPlatform collides with + # rustHostPlatform. + + lib.optionalString (rustTargetPlatform != rustHostPlatform) '' + "CC_${stdenv.targetPlatform.rust.cargoEnvVarTarget}=${ccForTarget}" \ + "CXX_${stdenv.targetPlatform.rust.cargoEnvVarTarget}=${cxxForTarget}" \ + "CARGO_TARGET_${stdenv.targetPlatform.rust.cargoEnvVarTarget}_LINKER=${linkerForTarget}" \ + '' + '' + "CC_${stdenv.hostPlatform.rust.cargoEnvVarTarget}=${ccForHost}" \ + "CXX_${stdenv.hostPlatform.rust.cargoEnvVarTarget}=${cxxForHost}" \ + "CARGO_TARGET_${stdenv.hostPlatform.rust.cargoEnvVarTarget}_LINKER=${linkerForHost}" \ + '' + '' + "CC_${stdenv.buildPlatform.rust.cargoEnvVarTarget}=${ccForBuild}" \ + "CXX_${stdenv.buildPlatform.rust.cargoEnvVarTarget}=${cxxForBuild}" \ + "CARGO_TARGET_${stdenv.buildPlatform.rust.cargoEnvVarTarget}_LINKER=${linkerForBuild}" \ + "CARGO_BUILD_TARGET=${rustBuildPlatform}" \ + "HOST_CC=${pkgsBuildHost.stdenv.cc}/bin/cc" \ + "HOST_CXX=${pkgsBuildHost.stdenv.cc}/bin/c++" \ + ''; + }; +} // lib.mapAttrs (old: new: platform: + # TODO: enable warning after 23.05 is EOL. + # lib.warn "`rust.${old} platform` is deprecated. Use `platform.rust.${new}` instead." + lib.getAttrFromPath new platform.rust) +{ + toTargetArch = [ "platform" "arch" ]; + toTargetOs = [ "platform" "os" ]; + toTargetFamily = [ "platform" "target-family" ]; + toTargetVendor = [ "platform" "vendor" ]; + toRustTarget = [ "rustcTarget" ]; + toRustTargetSpec = [ "rustcTargetSpec" ]; + toRustTargetSpecShort = [ "cargoShortTarget" ]; + toRustTargetForUseInEnvVars = [ "cargoEnvVarTarget" ]; + IsNoStdTarget = [ "isNoStdTarget" ]; } diff --git a/pkgs/build-support/rust/replace-workspace-values.py b/pkgs/build-support/rust/replace-workspace-values.py index acbc38c8ae39c..2b88f1fa79bbe 100644 --- a/pkgs/build-support/rust/replace-workspace-values.py +++ b/pkgs/build-support/rust/replace-workspace-values.py @@ -96,6 +96,13 @@ def main() -> None: workspace_manifest, crate_manifest["target"][key] ) + if ( + "lints" in crate_manifest + and "workspace" in crate_manifest["lints"] + and crate_manifest["lints"]["workspace"] is True + ): + crate_manifest["lints"] = workspace_manifest["lints"] + if not changed: return diff --git a/pkgs/build-support/rust/rustc-wrapper/default.nix b/pkgs/build-support/rust/rustc-wrapper/default.nix new file mode 100644 index 0000000000000..d6034c08af47d --- /dev/null +++ b/pkgs/build-support/rust/rustc-wrapper/default.nix @@ -0,0 +1,30 @@ +{ lib, runCommand, rustc-unwrapped, sysroot ? null }: + +runCommand "${rustc-unwrapped.pname}-wrapper-${rustc-unwrapped.version}" { + preferLocalBuild = true; + strictDeps = true; + inherit (rustc-unwrapped) outputs; + + env = { + prog = "${rustc-unwrapped}/bin/rustc"; + sysroot = lib.optionalString (sysroot != null) "--sysroot ${sysroot}"; + }; + + passthru = { + inherit (rustc-unwrapped) pname version src llvm llvmPackages; + unwrapped = rustc-unwrapped; + }; + + meta = rustc-unwrapped.meta // { + description = "${rustc-unwrapped.meta.description} (wrapper script)"; + priority = 10; + }; +} '' + mkdir -p $out/bin + ln -s ${rustc-unwrapped}/bin/* $out/bin + rm $out/bin/rustc + substituteAll ${./rustc-wrapper.sh} $out/bin/rustc + chmod +x $out/bin/rustc + ${lib.concatMapStrings (output: "ln -s ${rustc-unwrapped.${output}} \$${output}\n") + (lib.remove "out" rustc-unwrapped.outputs)} +'' diff --git a/pkgs/build-support/rust/rustc-wrapper/rustc-wrapper.sh b/pkgs/build-support/rust/rustc-wrapper/rustc-wrapper.sh new file mode 100644 index 0000000000000..4a90e30652fea --- /dev/null +++ b/pkgs/build-support/rust/rustc-wrapper/rustc-wrapper.sh @@ -0,0 +1,29 @@ +#!@shell@ + +defaultSysroot=(@sysroot@) + +for arg; do + case "$arg" in + --sysroot|--sysroot=*) + defaultSysroot=() + ;; + --) + break + ;; + esac +done + +extraBefore=("${defaultSysroot[@]}") +extraAfter=($NIX_RUSTFLAGS) + +# Optionally print debug info. +if (( "${NIX_DEBUG:-0}" >= 1 )); then + echo "extra flags before to @prog@:" >&2 + printf " %q\n" "${extraBefore[@]}" >&2 + echo "original flags to @prog@:" >&2 + printf " %q\n" "$@" >&2 + echo "extra flags after to @prog@:" >&2 + printf " %q\n" "${extraAfter[@]}" >&2 +fi + +exec @prog@ "${extraBefore[@]}" "$@" "${extraAfter[@]}" diff --git a/pkgs/build-support/setup-hooks/auto-patchelf.py b/pkgs/build-support/setup-hooks/auto-patchelf.py index 965384b876fcd..4769179167b3b 100644 --- a/pkgs/build-support/setup-hooks/auto-patchelf.py +++ b/pkgs/build-support/setup-hooks/auto-patchelf.py @@ -174,7 +174,7 @@ class Dependency: found: bool = False # Whether it was found somewhere -def auto_patchelf_file(path: Path, runtime_deps: list[Path], append_rpaths: List[Path] = []) -> list[Dependency]: +def auto_patchelf_file(path: Path, runtime_deps: list[Path], append_rpaths: List[Path] = [], extra_args: List[str] = []) -> list[Dependency]: try: with open_elf(path) as elf: @@ -213,7 +213,7 @@ def auto_patchelf_file(path: Path, runtime_deps: list[Path], append_rpaths: List if file_is_dynamic_executable: print("setting interpreter of", path) subprocess.run( - ["patchelf", "--set-interpreter", interpreter_path.as_posix(), path.as_posix()], + ["patchelf", "--set-interpreter", interpreter_path.as_posix(), path.as_posix()] + extra_args, check=True) rpath += runtime_deps @@ -250,7 +250,7 @@ def auto_patchelf_file(path: Path, runtime_deps: list[Path], append_rpaths: List if rpath: print("setting RPATH to:", rpath_str) subprocess.run( - ["patchelf", "--set-rpath", rpath_str, path.as_posix()], + ["patchelf", "--set-rpath", rpath_str, path.as_posix()] + extra_args, check=True) return dependencies @@ -262,7 +262,8 @@ def auto_patchelf( runtime_deps: List[Path], recursive: bool = True, ignore_missing: List[str] = [], - append_rpaths: List[Path] = []) -> None: + append_rpaths: List[Path] = [], + extra_args: List[str] = []) -> None: if not paths_to_patch: sys.exit("No paths to patch, stopping.") @@ -275,7 +276,7 @@ def auto_patchelf( dependencies = [] for path in chain.from_iterable(glob(p, '*', recursive) for p in paths_to_patch): if not path.is_symlink() and path.is_file(): - dependencies += auto_patchelf_file(path, runtime_deps, append_rpaths) + dependencies += auto_patchelf_file(path, runtime_deps, append_rpaths, extra_args) missing = [dep for dep in dependencies if not dep.found] @@ -333,6 +334,15 @@ def main() -> None: type=Path, help="Paths to append to all runtime paths unconditionally", ) + parser.add_argument( + "--extra-args", + # Undocumented Python argparse feature: consume all remaining arguments + # as values for this one. This means this argument should always be passed + # last. + nargs="...", + type=str, + help="Extra arguments to pass to patchelf. This argument should always come last." + ) print("automatically fixing dependencies for ELF files") args = parser.parse_args() @@ -344,7 +354,8 @@ def main() -> None: args.runtime_dependencies, args.recursive, args.ignore_missing, - append_rpaths=args.append_rpaths) + append_rpaths=args.append_rpaths, + extra_args=args.extra_args) interpreter_path: Path = None # type: ignore diff --git a/pkgs/build-support/setup-hooks/auto-patchelf.sh b/pkgs/build-support/setup-hooks/auto-patchelf.sh index 0625565606f3c..9f6366b3feaed 100644 --- a/pkgs/build-support/setup-hooks/auto-patchelf.sh +++ b/pkgs/build-support/setup-hooks/auto-patchelf.sh @@ -53,16 +53,30 @@ autoPatchelf() { esac done - readarray -td' ' ignoreMissingDepsArray < <(echo -n "$autoPatchelfIgnoreMissingDeps") - if [ "$autoPatchelfIgnoreMissingDeps" == "1" ]; then - echo "autoPatchelf: WARNING: setting 'autoPatchelfIgnoreMissingDeps" \ - "= true;' is deprecated and will be removed in a future release." \ - "Use 'autoPatchelfIgnoreMissingDeps = [ \"*\" ];' instead." >&2 - ignoreMissingDepsArray=( "*" ) + if [ -n "$__structuredAttrs" ]; then + local ignoreMissingDepsArray=( "${autoPatchelfIgnoreMissingDeps[@]}" ) + local appendRunpathsArray=( "${appendRunpaths[@]}" ) + local runtimeDependenciesArray=( "${runtimeDependencies[@]}" ) + local patchelfFlagsArray=( "${patchelfFlags[@]}" ) + else + readarray -td' ' ignoreMissingDepsArray < <(echo -n "$autoPatchelfIgnoreMissingDeps") + local appendRunpathsArray=($appendRunpaths) + local runtimeDependenciesArray=($runtimeDependencies) + local patchelfFlagsArray=($patchelfFlags) fi - local appendRunpathsArray=($appendRunpaths) - local runtimeDependenciesArray=($runtimeDependencies) + # Check if ignoreMissingDepsArray contains "1" and if so, replace it with + # "*", printing a deprecation warning. + for dep in "${ignoreMissingDepsArray[@]}"; do + if [ "$dep" == "1" ]; then + echo "autoPatchelf: WARNING: setting 'autoPatchelfIgnoreMissingDeps" \ + "= true;' is deprecated and will be removed in a future release." \ + "Use 'autoPatchelfIgnoreMissingDeps = [ \"*\" ];' instead." >&2 + ignoreMissingDepsArray=( "*" ) + break + fi + done + @pythonInterpreter@ @autoPatchelfScript@ \ ${norecurse:+--no-recurse} \ --ignore-missing "${ignoreMissingDepsArray[@]}" \ @@ -70,7 +84,8 @@ autoPatchelf() { --libs "${autoPatchelfLibs[@]}" \ "${extraAutoPatchelfLibs[@]}" \ --runtime-dependencies "${runtimeDependenciesArray[@]/%//lib}" \ - --append-rpaths "${appendRunpathsArray[@]}" + --append-rpaths "${appendRunpathsArray[@]}" \ + --extra-args "${patchelfFlagsArray[@]}" } # XXX: This should ultimately use fixupOutputHooks but we currently don't have diff --git a/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh b/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh index 55e196e654dff..e103fe77d9be7 100644 --- a/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh +++ b/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh @@ -36,5 +36,5 @@ fixDarwinDylibNames() { fixDarwinDylibNamesIn() { local dir="$1" - fixDarwinDylibNames $(find "$dir" -name "*.dylib") + fixDarwinDylibNames $(find "$dir" -name "*.dylib" -o -name "*.so" -o -name "*.so.*") } diff --git a/pkgs/build-support/setup-hooks/make-binary-wrapper/default.nix b/pkgs/build-support/setup-hooks/make-binary-wrapper/default.nix index 62ba3705be20d..f364dd5de753d 100644 --- a/pkgs/build-support/setup-hooks/make-binary-wrapper/default.nix +++ b/pkgs/build-support/setup-hooks/make-binary-wrapper/default.nix @@ -19,7 +19,7 @@ makeSetupHook { passthru = { # Extract the function call used to create a binary wrapper from its embedded docstring extractCmd = writeShellScript "extract-binary-wrapper-cmd" '' - strings -dw "$1" | sed -n '/^makeCWrapper/,/^$/ p' + ${cc.bintools.targetPrefix}strings -dw "$1" | sed -n '/^makeCWrapper/,/^$/ p' ''; tests = tests.makeBinaryWrapper; diff --git a/pkgs/build-support/setup-hooks/make-binary-wrapper/make-binary-wrapper.sh b/pkgs/build-support/setup-hooks/make-binary-wrapper/make-binary-wrapper.sh index 88a50befd7328..6cd01f6bf6307 100644 --- a/pkgs/build-support/setup-hooks/make-binary-wrapper/make-binary-wrapper.sh +++ b/pkgs/build-support/setup-hooks/make-binary-wrapper/make-binary-wrapper.sh @@ -193,8 +193,23 @@ makeCWrapper() { addFlags() { local n flag before after var + + # Disable file globbing, since bash will otherwise try to find + # filenames matching the the value to be prefixed/suffixed if + # it contains characters considered wildcards, such as `?` and + # `*`. We want the value as is, except we also want to split + # it on on the separator; hence we can't quote it. + local reenableGlob=0 + if [[ ! -o noglob ]]; then + reenableGlob=1 + fi + set -o noglob # shellcheck disable=SC2086 before=($1) after=($2) + if (( reenableGlob )); then + set +o noglob + fi + var="argv_tmp" printf '%s\n' "char **$var = calloc(${#before[@]} + argc + ${#after[@]} + 1, sizeof(*$var));" printf '%s\n' "assert($var != NULL);" diff --git a/pkgs/build-support/setup-hooks/postgresql-test-hook/postgresql-test-hook.sh b/pkgs/build-support/setup-hooks/postgresql-test-hook/postgresql-test-hook.sh index 3eec67d60feb4..d09153b2d6441 100644 --- a/pkgs/build-support/setup-hooks/postgresql-test-hook/postgresql-test-hook.sh +++ b/pkgs/build-support/setup-hooks/postgresql-test-hook/postgresql-test-hook.sh @@ -56,6 +56,8 @@ EOF echo 'initializing postgresql' initdb -U postgres + echo "$postgresqlExtraSettings" >>"$PGDATA/postgresql.conf" + # Move the socket echo "unix_socket_directories = '$NIX_BUILD_TOP/run/postgresql'" >>"$PGDATA/postgresql.conf" diff --git a/pkgs/build-support/setup-hooks/separate-debug-info.sh b/pkgs/build-support/setup-hooks/separate-debug-info.sh index 3a16ac4fee906..197e8a920b702 100644 --- a/pkgs/build-support/setup-hooks/separate-debug-info.sh +++ b/pkgs/build-support/setup-hooks/separate-debug-info.sh @@ -1,7 +1,7 @@ export NIX_SET_BUILD_ID=1 export NIX_LDFLAGS+=" --compress-debug-sections=zlib" export NIX_CFLAGS_COMPILE+=" -ggdb -Wa,--compress-debug-sections" -export RUSTFLAGS+=" -g" +export NIX_RUSTFLAGS+=" -g" fixupOutputHooks+=(_separateDebugInfo) diff --git a/pkgs/build-support/singularity-tools/default.nix b/pkgs/build-support/singularity-tools/default.nix index 9689e41245909..8d7ad9e742a1a 100644 --- a/pkgs/build-support/singularity-tools/default.nix +++ b/pkgs/build-support/singularity-tools/default.nix @@ -111,7 +111,7 @@ rec { touch .${projectName}.d/env/94-appsbase.sh cd .. - mkdir -p /var/lib/${projectName}/mnt/{container,final,overlay,session,source} + mkdir -p /var/lib/${projectName}/mnt/session echo "root:x:0:0:System administrator:/root:/bin/sh" > /etc/passwd echo > /etc/resolv.conf TMPDIR=$(pwd -P) ${projectName} build $out ./img diff --git a/pkgs/build-support/templaterpm/default.nix b/pkgs/build-support/templaterpm/default.nix deleted file mode 100644 index 56c543e8a9301..0000000000000 --- a/pkgs/build-support/templaterpm/default.nix +++ /dev/null @@ -1,25 +0,0 @@ -{lib, stdenv, makeWrapper, python, toposort, rpm}: - -stdenv.mkDerivation { - pname = "nix-template-rpm"; - version = "0.1"; - - nativeBuildInputs = [ makeWrapper ]; - buildInputs = [ python toposort rpm ]; - - dontUnpack = true; - - installPhase = '' - mkdir -p $out/bin - cp ${./nix-template-rpm.py} $out/bin/nix-template-rpm - wrapProgram $out/bin/nix-template-rpm \ - --set PYTHONPATH "${rpm}/lib/${python.libPrefix}/site-packages":"${toposort}/lib/${python.libPrefix}/site-packages" - ''; - - meta = with lib; { - description = "Create templates of nix expressions from RPM .spec files"; - maintainers = with maintainers; [ ]; - platforms = platforms.unix; - hydraPlatforms = []; - }; -} diff --git a/pkgs/build-support/templaterpm/nix-template-rpm.py b/pkgs/build-support/templaterpm/nix-template-rpm.py deleted file mode 100755 index db8c0f2064c20..0000000000000 --- a/pkgs/build-support/templaterpm/nix-template-rpm.py +++ /dev/null @@ -1,518 +0,0 @@ -#!/bin/env python - -import sys -import os -import subprocess -import argparse -import re -import shutil -import rpm -import urlparse -import traceback -import toposort - - - - - -class SPECTemplate(object): - def __init__(self, specFilename, outputDir, inputDir=None, buildRootInclude=None, translateTable=None, repositoryDir=None, allPackagesDir=None, maintainer="MAINTAINER"): - rpm.addMacro("buildroot","$out") - rpm.addMacro("_libdir","lib") - rpm.addMacro("_libexecdir","libexec") - rpm.addMacro("_sbindir","sbin") - rpm.addMacro("_sysconfdir","etc") - rpm.addMacro("_topdir","SPACER_DIR_FOR_REMOVAL") - rpm.addMacro("_sourcedir","SOURCE_DIR_SPACER") - - self.packageGroups = [ "ocaml", "python" ] - - ts = rpm.TransactionSet() - - self.specFilename = specFilename - self.spec = ts.parseSpec(specFilename) - - self.inputDir = inputDir - self.buildRootInclude = buildRootInclude - self.repositoryDir = repositoryDir - self.allPackagesDir = allPackagesDir - self.maintainer = maintainer - - self.translateTable = translateTable - - self.facts = self.getFacts() - self.key = self.getSelfKey() - - tmpDir = os.path.join(outputDir, self.rewriteName(self.spec.sourceHeader['name'])) - if self.translateTable is not None: - self.relOutputDir = self.translateTable.path(self.key,tmpDir) - else: - self.relOutputDir = tmpDir - - self.final_output_dir = os.path.normpath( self.relOutputDir ) - - if self.repositoryDir is not None: - self.potential_repository_dir = os.path.normpath( os.path.join(self.repositoryDir,self.relOutputDir) ) - - - - def rewriteCommands(self, string): - string = string.replace('SPACER_DIR_FOR_REMOVAL/','') - string = string.replace('SPACER_DIR_FOR_REMOVAL','') - string = '\n'.join(map(lambda line: ' '.join(map(lambda x: x.replace('SOURCE_DIR_SPACER/',('${./' if (self.buildRootInclude is None) else '${buildRoot}/usr/share/buildroot/SOURCES/'))+('}' if (self.buildRootInclude is None) else '') if x.startswith('SOURCE_DIR_SPACER/') else x, line.split(' '))), string.split('\n'))) - string = string.replace('\n','\n ') - string = string.rstrip() - return string - - - def rewriteName(self, string): - parts = string.split('-') - parts = filter(lambda x: not x == "devel", parts) - parts = filter(lambda x: not x == "doc", parts) - if len(parts) > 1 and parts[0] in self.packageGroups: - return parts[0] + '-' + ''.join(parts[1:2] + map(lambda x: x.capitalize(), parts[2:])) - else: - return ''.join(parts[:1] + map(lambda x: x.capitalize(), parts[1:])) - - - def rewriteInputs(self,target,inputs): - camelcase = lambda l: l[:1] + map(lambda x: x.capitalize(), l[1:]) - filterDevel = lambda l: filter(lambda x: not x == "devel", l) - filterDoc = lambda l: filter(lambda x: not x == "doc", l) - rewrite = lambda l: ''.join(camelcase(filterDoc(filterDevel(l)))) - - def filterPackageGroup(target): - if target is None: - return [ rewrite(x.split('-')) for x in inputs if (not x.split('-')[0] in self.packageGroups) or (len(x.split('-')) == 1) ] - elif target in self.packageGroups: - return [ target + '_' + rewrite(x.split('-')[1:]) for x in inputs if (x.split('-')[0] == target) and (len(x.split('-')) > 1)] - else: - raise Exception("Unknown target") - return [] - - if target is None: - packages = filterPackageGroup(None) - packages.sort() - elif target in self.packageGroups: - packages = filterPackageGroup(target) - packages.sort() - elif target == "ALL": - packages = [] - for t in [None] + self.packageGroups: - tmp = filterPackageGroup(t) - tmp.sort() - packages += tmp - else: - raise Exception("Unknown target") - packages = [] - - return packages - - - def getBuildInputs(self,target=None): - inputs = self.rewriteInputs(target,self.spec.sourceHeader['requires']) - if self.translateTable is not None: - return map(lambda x: self.translateTable.name(x), inputs) - else: - return inputs - - def getSelfKey(self): - name = self.spec.sourceHeader['name'] - if len(name.split('-')) > 1 and name.split('-')[0] in self.packageGroups: - key = self.rewriteInputs(name.split('-')[0], [self.spec.sourceHeader['name']])[0] - else: - key = self.rewriteInputs(None, [self.spec.sourceHeader['name']])[0] - return key - - def getSelf(self): - if self.translateTable is not None: - return self.translateTable.name(self.key) - else: - return self.key - - - - - def copyPatches(self, input_dir, output_dir): - patches = [source for (source, _, flag) in self.spec.sources if flag==2] - for filename in patches: - shutil.copyfile(os.path.join(input_dir, filename), os.path.join(output_dir, filename)) - - - def copySources(self, input_dir, output_dir): - filenames = [source for (source, _, flag) in self.spec.sources if flag==1 if not urlparse.urlparse(source).scheme in ["http", "https"] ] - for filename in filenames: - shutil.copyfile(os.path.join(input_dir, filename), os.path.join(output_dir, filename)) - - - def getFacts(self): - facts = {} - facts["name"] = self.rewriteName(self.spec.sourceHeader['name']) - facts["version"] = self.spec.sourceHeader['version'] - - facts["url"] = [] - facts["sha256"] = [] - sources = [source for (source, _, flag) in self.spec.sources if flag==1 if urlparse.urlparse(source).scheme in ["http", "https"] ] - for url in sources: - p = subprocess.Popen(['nix-prefetch-url', url], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, err = p.communicate() - sha256 = output[:-1] #remove new line - facts["url"].append(url) - facts["sha256"].append(sha256) - - patches = [source for (source, _, flag) in self.spec.sources if flag==2] - if self.buildRootInclude is None: - facts["patches"] = map(lambda x: './'+x, patches) - else: - facts["patches"] = map(lambda x: '"${buildRoot}/usr/share/buildroot/SOURCES/'+x+'"', reversed(patches)) - - return facts - - - @property - def name(self): - out = ' name = "' + self.facts["name"] + '-' + self.facts["version"] + '";\n' - out += ' version = "' + self.facts['version'] + '";\n' - return out - - - @property - def src(self): - sources = [source for (source, _, flag) in self.spec.sources if flag==1 if urlparse.urlparse(source).scheme in ["http", "https"] ] - out = '' - for (url,sha256) in zip(self.facts['url'],self.facts['sha256']): - out += ' src = fetchurl {\n' - out += ' url = "' + url + '";\n' - out += ' sha256 = "' + sha256 + '";\n' - out += ' };\n' - return out - - - @property - def patch(self): - out = ' patches = [ ' + ' '.join(self.facts['patches']) + ' ];\n' - return out - - - @property - def buildInputs(self): - out = ' buildInputs = [ ' - out += ' '.join(self.getBuildInputs("ALL")) - out += ' ];\n' - return out - - - @property - def configure(self): - out = ' configurePhase = \'\'\n ' + self.rewriteCommands(self.spec.prep) + '\n \'\';\n'; - return out - - - @property - def build(self): - out = ' buildPhase = \'\'\n ' + self.rewriteCommands(self.spec.build) + '\n \'\';\n'; - return out - - - @property - def install(self): - out = ' installPhase = \'\'\n ' + self.rewriteCommands(self.spec.install) + '\n \'\';\n'; - return out - - @property - def ocamlExtra(self): - if "ocaml" in self.getBuildInputs("ALL"): - return ' createFindlibDestdir = true;\n' - else: - return '' - - - @property - def meta(self): - out = ' meta = with lib; {\n' - out += ' homepage = ' + self.spec.sourceHeader['url'] + ';\n' - out += ' description = "' + self.spec.sourceHeader['summary'] + '";\n' - out += ' license = lib.licenses.' + self.spec.sourceHeader['license'] + ';\n' - out += ' platforms = [ "i686-linux" "x86_64-linux" ];\n' - out += ' maintainers = with lib.maintainers; [ ' + self.maintainer + ' ];\n' - out += ' };\n' - out += '}\n' - return out - - - def __str__(self): - head = '{lib, stdenv, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n' - head += 'stdenv.mkDerivation {\n' - body = [ self.name, self.src, self.patch, self.buildInputs, self.configure, self.build, self.ocamlExtra, self.install, self.meta ] - return head + '\n'.join(body) - - - def getTemplate(self): - head = '{lib, stdenv, buildRoot, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n' - head += 'let\n' - head += ' buildRootInput = (import "${buildRoot}/usr/share/buildroot/buildRootInput.nix") { fetchurl=fetchurl; buildRoot=buildRoot; };\n' - head += 'in\n\n' - head += 'stdenv.mkDerivation {\n' - head += ' inherit (buildRootInput.'+self.rewriteName(self.spec.sourceHeader['name'])+') name version src;\n' - head += ' patches = buildRootInput.'+self.rewriteName(self.spec.sourceHeader['name'])+'.patches ++ [];\n\n' - body = [ self.buildInputs, self.configure, self.build, self.ocamlExtra, self.install, self.meta ] - return head + '\n'.join(body) - - - def getInclude(self): - head = self.rewriteName(self.spec.sourceHeader['name']) + ' = {\n' - body = [ self.name, self.src, self.patch ] - return head + '\n'.join(body) + '};\n' - - - def __cmp__(self,other): - if self.getSelf() in other.getBuildInputs("ALL"): - return 1 - else: - return -1 - - - def callPackage(self): - callPackage = ' ' + self.getSelf() + ' = callPackage ' + os.path.relpath(self.final_output_dir, self.allPackagesDir) + ' {' - newline = False; - for target in self.packageGroups: - tmp = self.getBuildInputs(target) - if len(tmp) > 0: - newline = True; - callPackage += '\n ' + 'inherit (' + target + 'Packages) ' + ' '.join(tmp) + ';' - if newline: - callPackage += '\n };' - else: - callPackage += ' };' - return callPackage - - - - def generateCombined(self): - if not os.path.exists(self.final_output_dir): - os.makedirs(self.final_output_dir) - - if self.inputDir is not None: - self.copySources(self.inputDir, self.final_output_dir) - self.copyPatches(self.inputDir, self.final_output_dir) - - nixfile = open(os.path.join(self.final_output_dir,'default.nix'), 'w') - nixfile.write(str(self)) - nixfile.close() - - shutil.copyfile(self.specFilename, os.path.join(self.final_output_dir, os.path.basename(self.specFilename))) - - - - def generateSplit(self): - if not os.path.exists(self.final_output_dir): - os.makedirs(self.final_output_dir) - - nixfile = open(os.path.join(self.final_output_dir,'default.nix'), 'w') - nixfile.write(self.getTemplate()) - nixfile.close() - - return self.getInclude() - - - - - - -class NixTemplate(object): - def __init__(self, nixfile): - self.nixfile = nixfile - self.original = { "name":None, "version":None, "url":None, "sha256":None, "patches":None } - self.update = { "name":None, "version":None, "url":None, "sha256":None, "patches":None } - self.matchedLines = {} - - if os.path.isfile(nixfile): - with file(nixfile, 'r') as infile: - for (n,line) in enumerate(infile): - name = re.match(r'^\s*name\s*=\s*"(.*?)"\s*;\s*$', line) - version = re.match(r'^\s*version\s*=\s*"(.*?)"\s*;\s*$', line) - url = re.match(r'^\s*url\s*=\s*"?(.*?)"?\s*;\s*$', line) - sha256 = re.match(r'^\s*sha256\s*=\s*"(.*?)"\s*;\s*$', line) - patches = re.match(r'^\s*patches\s*=\s*(\[.*?\])\s*;\s*$', line) - if name is not None and self.original["name"] is None: - self.original["name"] = name.group(1) - self.matchedLines[n] = "name" - if version is not None and self.original["version"] is None: - self.original["version"] = version.group(1) - self.matchedLines[n] = "version" - if url is not None and self.original["url"] is None: - self.original["url"] = url.group(1) - self.matchedLines[n] = "url" - if sha256 is not None and self.original["sha256"] is None: - self.original["sha256"] = sha256.group(1) - self.matchedLines[n] = "sha256" - if patches is not None and self.original["patches"] is None: - self.original["patches"] = patches.group(1) - self.matchedLines[n] = "patches" - - - def generateUpdated(self, nixOut): - nixTemplateFile = open(os.path.normpath(self.nixfile),'r') - nixOutFile = open(os.path.normpath(nixOut),'w') - for (n,line) in enumerate(nixTemplateFile): - if self.matchedLines.has_key(n) and self.update[self.matchedLines[n]] is not None: - nixOutFile.write(line.replace(self.original[self.matchedLines[n]], self.update[self.matchedLines[n]], 1)) - else: - nixOutFile.write(line) - nixTemplateFile.close() - nixOutFile.close() - - - def loadUpdate(self,orig): - if orig.has_key("name") and orig.has_key("version"): - self.update["name"] = orig["name"] + '-' + orig["version"] - self.update["version"] = orig["version"] - if orig.has_key("url") and orig.has_key("sha256") and len(orig["url"])>0: - self.update["url"] = orig["url"][0] - self.update["sha256"] = orig["sha256"][0] - for url in orig["url"][1:-1]: - sys.stderr.write("WARNING: URL has been dropped: %s\n" % url) - if orig.has_key("patches"): - self.update["patches"] = '[ ' + ' '.join(orig['patches']) + ' ]' - - -class TranslationTable(object): - def __init__(self): - self.tablePath = {} - self.tableName = {} - - def update(self, key, path, name=None): - self.tablePath[key] = path - if name is not None: - self.tableName[key] = name - - def readTable(self, tableFile): - with file(tableFile, 'r') as infile: - for line in infile: - match = re.match(r'^(.+?)\s+(.+?)\s+(.+?)\s*$', line) - if match is not None: - if not self.tablePath.has_key(match.group(1)): - self.tablePath[match.group(1)] = match.group(2) - if not self.tableName.has_key(match.group(1)): - self.tableName[match.group(1)] = match.group(3) - else: - match = re.match(r'^(.+?)\s+(.+?)\s*$', line) - if not self.tablePath.has_key(match.group(1)): - self.tablePath[match.group(1)] = match.group(2) - - def writeTable(self, tableFile): - outFile = open(os.path.normpath(tableFile),'w') - keys = self.tablePath.keys() - keys.sort() - for k in keys: - if self.tableName.has_key(k): - outFile.write( k + " " + self.tablePath[k] + " " + self.tableName[k] + "\n" ) - else: - outFile.write( k + " " + self.tablePath[k] + "\n" ) - outFile.close() - - def name(self, key): - if self.tableName.has_key(key): - return self.tableName[key] - else: - return key - - def path(self, key, orig): - if self.tablePath.has_key(key): - return self.tablePath[key] - else: - return orig - - - - - -if __name__ == "__main__": - #Parse command line options - parser = argparse.ArgumentParser(description="Generate .nix templates from RPM spec files") - parser.add_argument("specs", metavar="SPEC", nargs="+", help="spec file") - parser.add_argument("-o", "--output", metavar="OUT_DIR", required=True, help="output directory") - parser.add_argument("-b", "--buildRoot", metavar="BUILDROOT_DIR", default=None, help="buildroot output directory") - parser.add_argument("-i", "--inputSources", metavar="IN_DIR", default=None, help="sources input directory") - parser.add_argument("-m", "--maintainer", metavar="MAINTAINER", default="__NIX_MAINTAINER__", help="package maintainer") - parser.add_argument("-r", "--repository", metavar="REP_DIR", default=None, help="nix repository to compare output against") - parser.add_argument("-t", "--translate", metavar="TRANSLATE_TABLE", default=None, help="path of translation table for name and path") - parser.add_argument("-u", "--translateOut", metavar="TRANSLATE_OUT", default=None, help="output path for updated translation table") - parser.add_argument("-a", "--allPackages", metavar="ALL_PACKAGES", default=None, help="top level dir to call packages from") - args = parser.parse_args() - - allPackagesDir = os.path.normpath( os.path.dirname(args.allPackages) ) - if not os.path.exists(allPackagesDir): - os.makedirs(allPackagesDir) - - buildRootContent = {} - nameMap = {} - - newTable = TranslationTable() - if args.translate is not None: - table = TranslationTable() - table.readTable(args.translate) - newTable.readTable(args.translate) - else: - table = None - - for specPath in args.specs: - try: - sys.stderr.write("INFO: generate nix file from: %s\n" % specPath) - - spec = SPECTemplate(specPath, args.output, args.inputSources, args.buildRoot, table, args.repository, allPackagesDir, args.maintainer) - if args.repository is not None: - if os.path.exists(os.path.join(spec.potential_repository_dir,'default.nix')): - nixTemplate = NixTemplate(os.path.join(spec.potential_repository_dir,'default.nix')) - nixTemplate.loadUpdate(spec.facts) - if not os.path.exists(spec.final_output_dir): - os.makedirs(spec.final_output_dir) - nixTemplate.generateUpdated(os.path.join(spec.final_output_dir,'default.nix')) - else: - sys.stderr.write("WARNING: Repository does not contain template: %s\n" % os.path.join(spec.potential_repository_dir,'default.nix')) - if args.buildRoot is None: - spec.generateCombined() - else: - buildRootContent[spec.key] = spec.generateSplit() - else: - if args.buildRoot is None: - spec.generateCombined() - else: - buildRootContent[spec.key] = spec.generateSplit() - - newTable.update(spec.key,spec.relOutputDir,spec.getSelf()) - nameMap[spec.getSelf()] = spec - - except Exception, e: - sys.stderr.write("ERROR: %s failed with:\n%s\n%s\n" % (specPath,e.message,traceback.format_exc())) - - if args.translateOut is not None: - if not os.path.exists(os.path.dirname(os.path.normpath(args.translateOut))): - os.makedirs(os.path.dirname(os.path.normpath(args.translateOut))) - newTable.writeTable(args.translateOut) - - graph = {} - for k, v in nameMap.items(): - graph[k] = set(v.getBuildInputs("ALL")) - - sortedSpecs = toposort.toposort_flatten(graph) - sortedSpecs = filter( lambda x: x in nameMap.keys(), sortedSpecs) - - allPackagesFile = open(os.path.normpath( args.allPackages ), 'w') - allPackagesFile.write( '\n\n'.join(map(lambda x: x.callPackage(), map(lambda x: nameMap[x], sortedSpecs))) ) - allPackagesFile.close() - - if args.buildRoot is not None: - buildRootFilename = os.path.normpath( args.buildRoot ) - if not os.path.exists(os.path.dirname(buildRootFilename)): - os.makedirs(os.path.dirname(buildRootFilename)) - buildRootFile = open(buildRootFilename, 'w') - buildRootFile.write( "{ fetchurl, buildRoot }: {\n\n" ) - keys = buildRootContent.keys() - keys.sort() - for k in keys: - buildRootFile.write( buildRootContent[k] + '\n' ) - buildRootFile.write( "}\n" ) - buildRootFile.close() - - diff --git a/pkgs/build-support/testers/hasPkgConfigModules/tester.nix b/pkgs/build-support/testers/hasPkgConfigModules/tester.nix index 7555590382717..bbcc4f0c0f710 100644 --- a/pkgs/build-support/testers/hasPkgConfigModules/tester.nix +++ b/pkgs/build-support/testers/hasPkgConfigModules/tester.nix @@ -30,10 +30,11 @@ runCommand testName { } package.meta; } '' + touch "$out" for moduleName in $moduleNames; do echo "checking pkg-config module $moduleName in $buildInputs" set +e - version="$(pkg-config --modversion $moduleName)" + version="$($PKG_CONFIG --modversion $moduleName)" r=$? set -e if [[ $r = 0 ]]; then @@ -41,7 +42,7 @@ runCommand testName { printf '%s\t%s\n' "$moduleName" "$version" >> "$out" else echo "These modules were available in the input propagation closure:" - pkg-config --list-all + $PKG_CONFIG --list-all echo "❌ pkg-config module $moduleName was not found" false fi diff --git a/pkgs/build-support/testers/testMetaPkgConfig/tester.nix b/pkgs/build-support/testers/testMetaPkgConfig/tester.nix index bee97ace1409c..7892a29e4c28d 100644 --- a/pkgs/build-support/testers/testMetaPkgConfig/tester.nix +++ b/pkgs/build-support/testers/testMetaPkgConfig/tester.nix @@ -6,9 +6,7 @@ runCommand "check-meta-pkg-config-modules-for-${package.name}" { meta = { description = "Test whether ${package.name} exposes all pkg-config modules ${toString package.meta.pkgConfigModules}"; }; - dependsOn = map - (moduleName: testers.hasPkgConfigModule { inherit package moduleName; }) - package.meta.pkgConfigModules; + dependsOn = testers.hasPkgConfigModules { inherit package; }; } '' echo "found all of ${toString package.meta.pkgConfigModules}" > "$out" '' diff --git a/pkgs/build-support/trivial-builders/default.nix b/pkgs/build-support/trivial-builders/default.nix index c4f2cfd754cd8..a38231bdcaa32 100644 --- a/pkgs/build-support/trivial-builders/default.nix +++ b/pkgs/build-support/trivial-builders/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, stdenvNoCC, lndir, runtimeShell, shellcheck, haskell }: +{ lib, stdenv, stdenvNoCC, lndir, runtimeShell, shellcheck-minimal }: let inherit (lib) @@ -9,31 +9,41 @@ in rec { - /* Run the shell command `buildCommand' to produce a store path named - `name'. The attributes in `env' are added to the environment - prior to running the command. By default `runCommand` runs in a - stdenv with no compiler environment. `runCommandCC` uses the default - stdenv, `pkgs.stdenv`. + /* + Run the shell command `buildCommand` to produce a store path named `name`. - Example: + The attributes in `env` are added to the environment prior to running the command. + Environment variables set by `stdenv.mkDerivation` take precedence. + By default `runCommand` runs in a stdenv with no compiler environment. + `runCommandCC` uses the default stdenv, `pkgs.stdenv`. - runCommand "name" {envVariable = true;} ''echo hello > $out'' - runCommandCC "name" {} ''gcc -o myfile myfile.c; cp myfile $out''; + Example: + ```nix + runCommand "name" {envVariable = true;} ''echo hello > $out''; + ``` - The `*Local` variants force a derivation to be built locally, - it is not substituted. + ```nix + runCommandCC "name" {} ''gcc -o myfile myfile.c; cp myfile $out''; + ``` - This is intended for very cheap commands (<1s execution time). - It saves on the network roundrip and can speed up a build. + The `*Local` variants force a derivation to be built locally, + it is not substituted. - It is the same as adding the special fields + This is intended for very cheap commands (<1s execution time). + It saves on the network roundrip and can speed up a build. - `preferLocalBuild = true;` - `allowSubstitutes = false;` + It is the same as adding the special fields - to a derivation’s attributes. + ```nix + { + preferLocalBuild = true; + allowSubstitutes = false; + } + ``` + + to a derivation’s attributes. */ runCommand = name: env: runCommandWith { stdenv = stdenvNoCC; @@ -57,7 +67,8 @@ rec { # `runCommandCCLocal` left out on purpose. # We shouldn’t force the user to have a cc in scope. - /* Generalized version of the `runCommand`-variants + /* + Generalized version of the `runCommand`-variants which does customized behavior via a single attribute set passed as the first argument instead of having a lot of variants like @@ -72,36 +83,37 @@ rec { defaultStdenv = stdenv; in { - # which stdenv to use, defaults to a stdenv with a C compiler, pkgs.stdenv + # which stdenv to use, defaults to a stdenv with a C compiler, pkgs.stdenv stdenv ? defaultStdenv - # whether to build this derivation locally instead of substituting + # whether to build this derivation locally instead of substituting , runLocal ? false - # extra arguments to pass to stdenv.mkDerivation - , derivationArgs ? {} - # name of the resulting derivation + # extra arguments to pass to stdenv.mkDerivation + , derivationArgs ? { } + # name of the resulting derivation , name - # TODO(@Artturin): enable strictDeps always + # TODO(@Artturin): enable strictDeps always }: buildCommand: - stdenv.mkDerivation ({ - enableParallelBuilding = true; - inherit buildCommand name; - passAsFile = [ "buildCommand" ] - ++ (derivationArgs.passAsFile or []); - } - // lib.optionalAttrs (! derivationArgs?meta) { - pos = let args = builtins.attrNames derivationArgs; in - if builtins.length args > 0 - then builtins.unsafeGetAttrPos (builtins.head args) derivationArgs - else null; - } - // (lib.optionalAttrs runLocal { - preferLocalBuild = true; - allowSubstitutes = false; - }) - // builtins.removeAttrs derivationArgs [ "passAsFile" ]); + stdenv.mkDerivation ({ + enableParallelBuilding = true; + inherit buildCommand name; + passAsFile = [ "buildCommand" ] + ++ (derivationArgs.passAsFile or [ ]); + } + // lib.optionalAttrs (! derivationArgs?meta) { + pos = let args = builtins.attrNames derivationArgs; in + if builtins.length args > 0 + then builtins.unsafeGetAttrPos (builtins.head args) derivationArgs + else null; + } + // (lib.optionalAttrs runLocal { + preferLocalBuild = true; + allowSubstitutes = false; + }) + // builtins.removeAttrs derivationArgs [ "passAsFile" ]); - /* Writes a text file to the nix store. + /* + Writes a text file to the nix store. The contents of text is added to the file in the store. Example: @@ -140,17 +152,21 @@ rec { , meta ? { } , allowSubstitutes ? false , preferLocalBuild ? true + , derivationArgs ? { } # Extra arguments to pass to `stdenv.mkDerivation` }: let matches = builtins.match "/bin/([^/]+)" destination; in runCommand name - { inherit text executable checkPhase allowSubstitutes preferLocalBuild; - passAsFile = [ "text" ]; - meta = lib.optionalAttrs (executable && matches != null) { - mainProgram = lib.head matches; - } // meta; - } + ({ + inherit text executable checkPhase allowSubstitutes preferLocalBuild; + passAsFile = [ "text" ] + ++ derivationArgs.passAsFile or [ ]; + meta = lib.optionalAttrs (executable && matches != null) + { + mainProgram = lib.head matches; + } // meta // derivationArgs.meta or {}; + } // removeAttrs derivationArgs [ "passAsFile" "meta" ]) '' target=$out${lib.escapeShellArg destination} mkdir -p "$(dirname "$target")" @@ -168,101 +184,32 @@ rec { eval "$checkPhase" ''; - /* - Writes a text file to nix store with no optional parameters available. - - Example: - + # See doc/build-helpers/trivial-build-helpers.chapter.md + # or https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-text-writing + writeText = name: text: writeTextFile { inherit name text; }; - # Writes contents of file to /nix/store/<store path> - writeText "my-file" - '' - Contents of File - ''; - - - */ - writeText = name: text: writeTextFile {inherit name text;}; - - /* - Writes a text file to nix store in a specific directory with no - optional parameters available. - - Example: - - - # Writes contents of file to /nix/store/<store path>/share/my-file - writeTextDir "share/my-file" - '' - Contents of File - ''; - - - */ + # See doc/build-helpers/trivial-build-helpers.chapter.md + # or https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-text-writing writeTextDir = path: text: writeTextFile { inherit text; name = builtins.baseNameOf path; destination = "/${path}"; }; - /* - Writes a text file to /nix/store/<store path> and marks the file as - executable. - - If passed as a build input, will be used as a setup hook. This makes setup - hooks more efficient to create: you don't need a derivation that copies - them to $out/nix-support/setup-hook, instead you can use the file as is. - - Example: - - - # Writes my-file to /nix/store/<store path> and makes executable - writeScript "my-file" - '' - Contents of File - ''; - - - */ - writeScript = name: text: writeTextFile {inherit name text; executable = true;}; - - /* - Writes a text file to /nix/store/<store path>/bin/<name> and - marks the file as executable. - - Example: + # See doc/build-helpers/trivial-build-helpers.chapter.md + # or https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-text-writing + writeScript = name: text: writeTextFile { inherit name text; executable = true; }; - - - # Writes my-file to /nix/store/<store path>/bin/my-file and makes executable. - writeScriptBin "my-file" - '' - Contents of File - ''; - - - */ + # See doc/build-helpers/trivial-build-helpers.chapter.md + # or https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-text-writing writeScriptBin = name: text: writeTextFile { inherit name text; executable = true; destination = "/bin/${name}"; }; - /* - Similar to writeScript. Writes a Shell script and checks its syntax. - Automatically includes interpreter above the contents passed. - - Example: - - - # Writes my-file to /nix/store/<store path> and makes executable. - writeShellScript "my-file" - '' - Contents of File - ''; - - - */ + # See doc/build-helpers/trivial-build-helpers.chapter.md + # or https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-text-writing writeShellScript = name: text: writeTextFile { inherit name; @@ -270,29 +217,15 @@ rec { text = '' #!${runtimeShell} ${text} - ''; + ''; checkPhase = '' ${stdenv.shellDryRun} "$target" ''; }; - /* - Similar to writeShellScript and writeScriptBin. - Writes an executable Shell script to /nix/store/<store path>/bin/<name> and checks its syntax. - Automatically includes interpreter above the contents passed. - - Example: - - - # Writes my-file to /nix/store/<store path>/bin/my-file and makes executable. - writeShellScriptBin "my-file" - '' - Contents of File - ''; - - - */ - writeShellScriptBin = name : text : + # See doc/build-helpers/trivial-build-helpers.chapter.md + # or https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-text-writing + writeShellScriptBin = name: text: writeTextFile { inherit name; executable = true; @@ -300,57 +233,101 @@ rec { text = '' #!${runtimeShell} ${text} - ''; + ''; checkPhase = '' ${stdenv.shellDryRun} "$target" ''; meta.mainProgram = name; }; - /* - Similar to writeShellScriptBin and writeScriptBin. - Writes an executable Shell script to /nix/store/<store path>/bin/<name> and - checks its syntax with shellcheck and the shell's -n option. - Automatically includes sane set of shellopts (errexit, nounset, pipefail) - and handles creation of PATH based on runtimeInputs - - Note that the checkPhase uses stdenv.shell for the test run of the script, - while the generated shebang uses runtimeShell. If, for whatever reason, - those were to mismatch you might lose fidelity in the default checks. - - Example: - - Writes my-file to /nix/store/<store path>/bin/my-file and makes executable. - - - writeShellApplication { - name = "my-file"; - runtimeInputs = [ curl w3m ]; - text = '' - curl -s 'https://nixos.org' | w3m -dump -T text/html - ''; - } - - */ + # See doc/build-helpers/trivial-build-helpers.chapter.md + # or https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-text-writing writeShellApplication = - { name - , text - , runtimeInputs ? [ ] - , meta ? { } - , checkPhase ? null + { + /* + The name of the script to write. + + Type: String + */ + name, + /* + The shell script's text, not including a shebang. + + Type: String + */ + text, + /* + Inputs to add to the shell script's `$PATH` at runtime. + + Type: [String|Derivation] + */ + runtimeInputs ? [ ], + /* + Extra environment variables to set at runtime. + + Type: AttrSet + */ + runtimeEnv ? null, + /* + `stdenv.mkDerivation`'s `meta` argument. + + Type: AttrSet + */ + meta ? { }, + /* + The `checkPhase` to run. Defaults to `shellcheck` on supported + platforms and `bash -n`. + + The script path will be given as `$target` in the `checkPhase`. + + Type: String + */ + checkPhase ? null, + /* + Checks to exclude when running `shellcheck`, e.g. `[ "SC2016" ]`. + + See <https://www.shellcheck.net/wiki/> for a list of checks. + + Type: [String] + */ + excludeShellChecks ? [ ], + /* + Bash options to activate with `set -o` at the start of the script. + + Defaults to `[ "errexit" "nounset" "pipefail" ]`. + + Type: [String] + */ + bashOptions ? [ "errexit" "nounset" "pipefail" ], + /* Extra arguments to pass to `stdenv.mkDerivation`. + + :::{.caution} + Certain derivation attributes are used internally, + overriding those could cause problems. + ::: + + Type: AttrSet + */ + derivationArgs ? { }, }: writeTextFile { - inherit name meta; + inherit name meta derivationArgs; executable = true; destination = "/bin/${name}"; allowSubstitutes = true; preferLocalBuild = false; text = '' #!${runtimeShell} - set -o errexit - set -o nounset - set -o pipefail - '' + lib.optionalString (runtimeInputs != [ ]) '' + ${lib.concatMapStringsSep "\n" (option: "set -o ${option}") bashOptions} + '' + lib.optionalString (runtimeEnv != null) + (lib.concatStrings + (lib.mapAttrsToList + (name: value: '' + ${lib.toShellVar name value} + export ${name} + '') + runtimeEnv)) + + lib.optionalString (runtimeInputs != [ ]) '' export PATH="${lib.makeBinPath runtimeInputs}:$PATH" '' + '' @@ -362,11 +339,12 @@ rec { # GHC (=> shellcheck) isn't supported on some platforms (such as risc-v) # but we still want to use writeShellApplication on those platforms let - shellcheckSupported = lib.meta.availableOn stdenv.buildPlatform shellcheck.compiler; + shellcheckSupported = lib.meta.availableOn stdenv.buildPlatform shellcheck-minimal.compiler; + excludeOption = lib.optionalString (excludeShellChecks != [ ]) "--exclude '${lib.concatStringsSep "," excludeShellChecks}'"; shellcheckCommand = lib.optionalString shellcheckSupported '' # use shellcheck which does not include docs # pandoc takes long to build and documentation isn't needed for just running the cli - ${lib.getExe (haskell.lib.compose.justStaticExecutables shellcheck.unwrapped)} "$target" + ${lib.getExe shellcheck-minimal} ${excludeOption} "$target" ''; in if checkPhase == null then '' @@ -379,25 +357,25 @@ rec { }; # Create a C binary - writeCBin = name: code: - runCommandCC name - { - inherit name code; - executable = true; - passAsFile = ["code"]; - # Pointless to do this on a remote machine. - preferLocalBuild = true; - allowSubstitutes = false; - meta = { - mainProgram = name; - }; - } - '' - n=$out/bin/$name - mkdir -p "$(dirname "$n")" - mv "$codePath" code.c - $CC -x c code.c -o "$n" - ''; + writeCBin = pname: code: + runCommandCC pname + { + inherit pname code; + executable = true; + passAsFile = [ "code" ]; + # Pointless to do this on a remote machine. + preferLocalBuild = true; + allowSubstitutes = false; + meta = { + mainProgram = pname; + }; + } + '' + n=$out/bin/${pname} + mkdir -p "$(dirname "$n")" + mv "$codePath" code.c + $CC -x c code.c -o "$n" + ''; /* concat a list of files to the nix store. @@ -474,7 +452,7 @@ rec { /* - Create a forest of symlinks to the files in `paths'. + Create a forest of symlinks to the files in `paths`. This creates a single derivation that replicates the directory structure of all the input paths. @@ -528,19 +506,20 @@ rec { */ symlinkJoin = args_@{ name - , paths - , preferLocalBuild ? true - , allowSubstitutes ? false - , postBuild ? "" - , ... - }: + , paths + , preferLocalBuild ? true + , allowSubstitutes ? false + , postBuild ? "" + , ... + }: let args = removeAttrs args_ [ "name" "postBuild" ] // { - inherit preferLocalBuild allowSubstitutes; - passAsFile = [ "paths" ]; - }; # pass the defaults - in runCommand name args + inherit preferLocalBuild allowSubstitutes; + passAsFile = [ "paths" ]; + }; # pass the defaults + in + runCommand name args '' mkdir -p $out for i in $(cat $pathsPath); do @@ -580,27 +559,30 @@ rec { See the note on symlinkJoin for the difference between linkFarm and symlinkJoin. */ linkFarm = name: entries: - let - entries' = - if (lib.isAttrs entries) then entries - # We do this foldl to have last-wins semantics in case of repeated entries - else if (lib.isList entries) then lib.foldl (a: b: a // { "${b.name}" = b.path; }) { } entries - else throw "linkFarm entries must be either attrs or a list!"; - - linkCommands = lib.mapAttrsToList (name: path: '' - mkdir -p "$(dirname ${lib.escapeShellArg "${name}"})" - ln -s ${lib.escapeShellArg "${path}"} ${lib.escapeShellArg "${name}"} - '') entries'; - in - runCommand name { - preferLocalBuild = true; - allowSubstitutes = false; - passthru.entries = entries'; - } '' - mkdir -p $out - cd $out - ${lib.concatStrings linkCommands} - ''; + let + entries' = + if (lib.isAttrs entries) then entries + # We do this foldl to have last-wins semantics in case of repeated entries + else if (lib.isList entries) then lib.foldl (a: b: a // { "${b.name}" = b.path; }) { } entries + else throw "linkFarm entries must be either attrs or a list!"; + + linkCommands = lib.mapAttrsToList + (name: path: '' + mkdir -p "$(dirname ${lib.escapeShellArg "${name}"})" + ln -s ${lib.escapeShellArg "${path}"} ${lib.escapeShellArg "${name}"} + '') + entries'; + in + runCommand name + { + preferLocalBuild = true; + allowSubstitutes = false; + passthru.entries = entries'; + } '' + mkdir -p $out + cd $out + ${lib.concatStrings linkCommands} + ''; /* Easily create a linkFarm from a set of derivations. @@ -627,6 +609,19 @@ rec { let mkEntryFromDrv = drv: { name = drv.name; path = drv; }; in linkFarm name (map mkEntryFromDrv drvs); + /* + Produce a derivation that links to the target derivation's `/bin`, + and *only* `/bin`. + + This is useful when your favourite package doesn't have a separate + bin output and other contents of the package's output (e.g. setup + hooks) cause trouble when used in your environment. + */ + onlyBin = drv: runCommand "${drv.name}-only-bin" { } '' + mkdir -p $out + ln -s ${lib.getBin drv}/bin $out/bin + ''; + # docs in doc/builders/special/makesetuphook.section.md makeSetupHook = @@ -658,23 +653,23 @@ rec { # TODO 2023-01, no backport: simplify to inherit passthru; passthru = passthru // optionalAttrs (substitutions?passthru) - (warn "makeSetupHook (name = ${lib.strings.escapeNixString name}): `substitutions.passthru` is deprecated. Please set `passthru` directly." - substitutions.passthru); + (warn "makeSetupHook (name = ${lib.strings.escapeNixString name}): `substitutions.passthru` is deprecated. Please set `passthru` directly." + substitutions.passthru); }) ('' mkdir -p $out/nix-support cp ${script} $out/nix-support/setup-hook recordPropagatedDependencies - '' + lib.optionalString (substitutions != {}) '' + '' + lib.optionalString (substitutions != { }) '' substituteAll ${script} $out/nix-support/setup-hook ''); - # Write the references (i.e. the runtime dependencies in the Nix store) of `path' to a file. + # Write the references (i.e. the runtime dependencies in the Nix store) of `path` to a file. writeReferencesToFile = path: runCommand "runtime-deps" { - exportReferencesGraph = ["graph" path]; + exportReferencesGraph = [ "graph" path ]; } '' touch $out @@ -693,7 +688,7 @@ rec { */ writeDirectReferencesToFile = path: runCommand "runtime-references" { - exportReferencesGraph = ["graph" path]; + exportReferencesGraph = [ "graph" path ]; inherit path; } '' @@ -727,17 +722,17 @@ rec { */ writeStringReferencesToFile = string: /* - The basic operation this performs is to copy the string context - from `string' to a second string and wrap that string in a - derivation. However, that alone is not enough, since nothing in the - string refers to the output paths of the derivations/paths in its - context, meaning they'll be considered build-time dependencies and - removed from the wrapper derivation's closure. Putting the - necessary output paths in the new string is however not very - straightforward - the attrset returned by `getContext' contains - only references to derivations' .drv-paths, not their output - paths. In order to "convert" them, we try to extract the - corresponding paths from the original string using regex. + The basic operation this performs is to copy the string context + from `string` to a second string and wrap that string in a + derivation. However, that alone is not enough, since nothing in the + string refers to the output paths of the derivations/paths in its + context, meaning they'll be considered build-time dependencies and + removed from the wrapper derivation's closure. Putting the + necessary output paths in the new string is however not very + straightforward - the attrset returned by `getContext` contains + only references to derivations' .drv-paths, not their output + paths. In order to "convert" them, we try to extract the + corresponding paths from the original string using regex. */ let # Taken from https://github.com/NixOS/nix/blob/130284b8508dad3c70e8160b15f3d62042fc730a/src/libutil/hash.cc#L84 @@ -781,21 +776,21 @@ rec { if lib.elem "out" value.outputs then lib.filter (x: lib.isList x && - # If the matched path is in `namedOutputPaths`, - # it's a partial match of an output path where - # the output name isn't `out` - lib.all (o: !lib.hasPrefix (lib.head x) o) namedOutputPaths) + # If the matched path is in `namedOutputPaths`, + # it's a partial match of an output path where + # the output name isn't `out` + lib.all (o: !lib.hasPrefix (lib.head x) o) namedOutputPaths) (builtins.split "(${builtins.storeDir}/[${nixHashChars}]+-${name})" string) else - []) + [ ]) packages); allPaths = lib.concatStringsSep "\n" (lib.unique (sources ++ namedOutputPaths ++ outputPaths)); allPathsWithContext = builtins.appendContext allPaths context; in - if builtins ? getContext then - writeText "string-references" allPathsWithContext - else - writeDirectReferencesToFile (writeText "string-file" string); + if builtins ? getContext then + writeText "string-references" allPathsWithContext + else + writeDirectReferencesToFile (writeText "string-file" string); /* Print an error message if the file with the specified name and @@ -813,55 +808,59 @@ rec { } */ - requireFile = { name ? null - , sha256 ? null - , sha1 ? null - , hash ? null - , url ? null - , message ? null - , hashMode ? "flat" - } : - assert (message != null) || (url != null); - assert (sha256 != null) || (sha1 != null) || (hash != null); - assert (name != null) || (url != null); - let msg = - if message != null then message - else '' - Unfortunately, we cannot download file ${name_} automatically. - Please go to ${url} to download it yourself, and add it to the Nix store - using either - nix-store --add-fixed ${hashAlgo} ${name_} - or - nix-prefetch-url --type ${hashAlgo} file:///path/to/${name_} - ''; - hashAlgo = if hash != null then (builtins.head (lib.strings.splitString "-" hash)) - else if sha256 != null then "sha256" - else "sha1"; - hashAlgo_ = if hash != null then "" else hashAlgo; - hash_ = if hash != null then hash - else if sha256 != null then sha256 - else sha1; - name_ = if name == null then baseNameOf (toString url) else name; - in - stdenvNoCC.mkDerivation { - name = name_; - outputHashMode = hashMode; - outputHashAlgo = hashAlgo_; - outputHash = hash_; - preferLocalBuild = true; - allowSubstitutes = false; - builder = writeScript "restrict-message" '' - source ${stdenvNoCC}/setup - cat <<_EOF_ - - *** - ${msg} - *** - - _EOF_ - exit 1 - ''; - }; + requireFile = + { name ? null + , sha256 ? null + , sha1 ? null + , hash ? null + , url ? null + , message ? null + , hashMode ? "flat" + }: + assert (message != null) || (url != null); + assert (sha256 != null) || (sha1 != null) || (hash != null); + assert (name != null) || (url != null); + let + msg = + if message != null then message + else '' + Unfortunately, we cannot download file ${name_} automatically. + Please go to ${url} to download it yourself, and add it to the Nix store + using either + nix-store --add-fixed ${hashAlgo} ${name_} + or + nix-prefetch-url --type ${hashAlgo} file:///path/to/${name_} + ''; + hashAlgo = + if hash != null then (builtins.head (lib.strings.splitString "-" hash)) + else if sha256 != null then "sha256" + else "sha1"; + hashAlgo_ = if hash != null then "" else hashAlgo; + hash_ = + if hash != null then hash + else if sha256 != null then sha256 + else sha1; + name_ = if name == null then baseNameOf (toString url) else name; + in + stdenvNoCC.mkDerivation { + name = name_; + outputHashMode = hashMode; + outputHashAlgo = hashAlgo_; + outputHash = hash_; + preferLocalBuild = true; + allowSubstitutes = false; + builder = writeScript "restrict-message" '' + source ${stdenvNoCC}/setup + cat <<_EOF_ + + *** + ${msg} + *** + + _EOF_ + exit 1 + ''; + }; /* @@ -898,39 +897,45 @@ rec { applyPatches = { src , name ? (if builtins.typeOf src == "path" - then builtins.baseNameOf src - else - if builtins.isAttrs src && builtins.hasAttr "name" src - then src.name - else throw "applyPatches: please supply a `name` argument because a default name can only be computed when the `src` is a path or is an attribute set with a `name` attribute." - ) + "-patched" - , patches ? [] + then builtins.baseNameOf src + else + if builtins.isAttrs src && builtins.hasAttr "name" src + then src.name + else throw "applyPatches: please supply a `name` argument because a default name can only be computed when the `src` is a path or is an attribute set with a `name` attribute." + ) + "-patched" + , patches ? [ ] , postPatch ? "" , ... - }@args: stdenvNoCC.mkDerivation { - inherit name src patches postPatch; - preferLocalBuild = true; - allowSubstitutes = false; - phases = "unpackPhase patchPhase installPhase"; - installPhase = "cp -R ./ $out"; - } + }@args: + if patches == [ ] && postPatch == "" + then src # nothing to do, so use original src to avoid additional drv + else stdenvNoCC.mkDerivation + { + inherit name src patches postPatch; + preferLocalBuild = true; + allowSubstitutes = false; + phases = "unpackPhase patchPhase installPhase"; + installPhase = "cp -R ./ $out"; + } # Carry `meta` information from the underlying `src` if present. // (optionalAttrs (src?meta) { inherit (src) meta; }) // (removeAttrs args [ "src" "name" "patches" "postPatch" ]); /* An immutable file in the store with a length of 0 bytes. */ - emptyFile = runCommand "empty-file" { - outputHashAlgo = "sha256"; - outputHashMode = "recursive"; - outputHash = "0ip26j2h11n1kgkz36rl4akv694yz65hr72q4kv4b3lxcbi65b3p"; - preferLocalBuild = true; - } "touch $out"; + emptyFile = runCommand "empty-file" + { + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = "0ip26j2h11n1kgkz36rl4akv694yz65hr72q4kv4b3lxcbi65b3p"; + preferLocalBuild = true; + } "touch $out"; /* An immutable empty directory in the store. */ - emptyDirectory = runCommand "empty-directory" { - outputHashAlgo = "sha256"; - outputHashMode = "recursive"; - outputHash = "0sjjj9z1dhilhpc8pq4154czrb79z9cm044jvn75kxcjv6v5l2m5"; - preferLocalBuild = true; - } "mkdir $out"; + emptyDirectory = runCommand "empty-directory" + { + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = "0sjjj9z1dhilhpc8pq4154czrb79z9cm044jvn75kxcjv6v5l2m5"; + preferLocalBuild = true; + } "mkdir $out"; } diff --git a/pkgs/build-support/trivial-builders/test/default.nix b/pkgs/build-support/trivial-builders/test/default.nix index 683f4b9fd04f3..59dbba3f18410 100644 --- a/pkgs/build-support/trivial-builders/test/default.nix +++ b/pkgs/build-support/trivial-builders/test/default.nix @@ -14,20 +14,24 @@ { callPackage, lib, stdenv }: let inherit (lib) recurseIntoAttrs; + references = callPackage ./references {}; in recurseIntoAttrs { concat = callPackage ./concat-test.nix {}; linkFarm = callPackage ./link-farm.nix {}; overriding = callPackage ../test-overriding.nix {}; + # VM test not supported beyond linux yet references = - # VM test not supported beyond linux yet if stdenv.hostPlatform.isLinux - then callPackage ./references.nix {} - else null; + then references + else {}; writeCBin = callPackage ./writeCBin.nix {}; + writeShellApplication = callPackage ./writeShellApplication.nix {}; writeScriptBin = callPackage ./writeScriptBin.nix {}; writeShellScript = callPackage ./write-shell-script.nix {}; writeShellScriptBin = callPackage ./writeShellScriptBin.nix {}; - writeStringReferencesToFile = callPackage ./writeStringReferencesToFile.nix {}; + writeStringReferencesToFile = callPackage ./writeStringReferencesToFile.nix { + inherit (references) samples; + }; writeTextFile = callPackage ./write-text-file.nix {}; } diff --git a/pkgs/build-support/trivial-builders/test/invoke-writeDirectReferencesToFile.nix b/pkgs/build-support/trivial-builders/test/invoke-writeDirectReferencesToFile.nix deleted file mode 100644 index ead3f7a2f5712..0000000000000 --- a/pkgs/build-support/trivial-builders/test/invoke-writeDirectReferencesToFile.nix +++ /dev/null @@ -1,4 +0,0 @@ -{ pkgs ? import ../../../.. { config = {}; overlays = []; } }: -pkgs.lib.mapAttrs - (k: v: pkgs.writeDirectReferencesToFile v) - (import ./sample.nix { inherit pkgs; }) diff --git a/pkgs/build-support/trivial-builders/test/invoke-writeReferencesToFile.nix b/pkgs/build-support/trivial-builders/test/invoke-writeReferencesToFile.nix deleted file mode 100644 index 99c6c2f7dcc43..0000000000000 --- a/pkgs/build-support/trivial-builders/test/invoke-writeReferencesToFile.nix +++ /dev/null @@ -1,4 +0,0 @@ -{ pkgs ? import ../../../.. { config = {}; overlays = []; } }: -pkgs.lib.mapAttrs - (k: v: pkgs.writeReferencesToFile v) - (import ./sample.nix { inherit pkgs; }) diff --git a/pkgs/build-support/trivial-builders/test/references-test.sh b/pkgs/build-support/trivial-builders/test/references-test.sh deleted file mode 100755 index 473ca6e107694..0000000000000 --- a/pkgs/build-support/trivial-builders/test/references-test.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -# -------------------------------------------------------------------------- # -# -# trivial-builders test -# -# -------------------------------------------------------------------------- # -# -# This file can be run independently (quick): -# -# $ pkgs/build-support/trivial-builders/references-test.sh -# -# or in the build sandbox with a ~20s VM overhead -# -# $ nix-build -A tests.trivial-builders.references -# -# -------------------------------------------------------------------------- # - -# strict bash -set -euo pipefail - -# debug -# set -x -# PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' - -cd "$(dirname ${BASH_SOURCE[0]})" # nixpkgs root - -if [[ -z ${SAMPLE:-} ]]; then - echo "Running the script directly is currently not supported." - echo "If you need to iterate, remove the raw path, which is not returned by nix-build." - exit 1 -# sample=( `nix-build --no-out-link sample.nix` ) -# directRefs=( `nix-build --no-out-link invoke-writeDirectReferencesToFile.nix` ) -# references=( `nix-build --no-out-link invoke-writeReferencesToFile.nix` ) -# echo "sample: ${#sample[@]}" -# echo "direct: ${#directRefs[@]}" -# echo "indirect: ${#references[@]}" -else - # Injected by Nix (to avoid evaluating in a derivation) - # turn them into arrays - sample=($SAMPLE) - directRefs=($DIRECT_REFS) - references=($REFERENCES) -fi - -echo >&2 Testing direct references... -for i in "${!sample[@]}"; do - echo >&2 Checking '#'$i ${sample[$i]} ${directRefs[$i]} - diff -U3 \ - <(sort <${directRefs[$i]}) \ - <(nix-store -q --references ${sample[$i]} | sort) -done - -echo >&2 Testing closure... -for i in "${!sample[@]}"; do - echo >&2 Checking '#'$i ${sample[$i]} ${references[$i]} - diff -U3 \ - <(sort <${references[$i]}) \ - <(nix-store -q --requisites ${sample[$i]} | sort) -done - -echo 'OK!' diff --git a/pkgs/build-support/trivial-builders/test/references.nix b/pkgs/build-support/trivial-builders/test/references.nix deleted file mode 100644 index 7c8ea83f3c8bf..0000000000000 --- a/pkgs/build-support/trivial-builders/test/references.nix +++ /dev/null @@ -1,52 +0,0 @@ -{ lib, testers, pkgs, writeText, hello, figlet, stdenvNoCC }: - -# -------------------------------------------------------------------------- # -# -# trivial-builders test -# -# -------------------------------------------------------------------------- # -# -# This file can be run independently (quick): -# -# $ pkgs/build-support/trivial-builders/references-test.sh -# -# or in the build sandbox with a ~20s VM overhead -# -# $ nix-build -A tests.trivial-builders.references -# -# -------------------------------------------------------------------------- # - -let - invokeSamples = file: - lib.concatStringsSep " " ( - lib.attrValues (import file { inherit pkgs; }) - ); -in -testers.nixosTest { - name = "nixpkgs-trivial-builders"; - nodes.machine = { ... }: { - virtualisation.writableStore = true; - - # Test runs without network, so we don't substitute and prepare our deps - nix.settings.substituters = lib.mkForce []; - environment.etc."pre-built-paths".source = writeText "pre-built-paths" ( - builtins.toJSON [hello figlet stdenvNoCC] - ); - environment.variables = { - SAMPLE = invokeSamples ./sample.nix; - REFERENCES = invokeSamples ./invoke-writeReferencesToFile.nix; - DIRECT_REFS = invokeSamples ./invoke-writeDirectReferencesToFile.nix; - }; - }; - testScript = - '' - machine.succeed(""" - ${./references-test.sh} 2>/dev/console - """) - ''; - meta = { - maintainers = with lib.maintainers; [ - roberth - ]; - }; -} diff --git a/pkgs/build-support/trivial-builders/test/references/default.nix b/pkgs/build-support/trivial-builders/test/references/default.nix new file mode 100644 index 0000000000000..3e21c905321e4 --- /dev/null +++ b/pkgs/build-support/trivial-builders/test/references/default.nix @@ -0,0 +1,124 @@ +{ lib +, stdenvNoCC +, testers +, callPackage +, writeText + # nativeBuildInputs +, shellcheck-minimal + # Samples +, samples ? cleanSamples (callPackage ./samples.nix { }) + # Filter out the non-string-like attributes such as <pkg>.override added by + # callPackage. +, cleanSamples ? lib.filterAttrs (n: lib.isStringLike) + # Test targets +, writeDirectReferencesToFile +, writeReferencesToFile +}: + +# -------------------------------------------------------------------------- # +# +# trivial-builders test +# +# -------------------------------------------------------------------------- # +# +# Execute this build script directly (quick): +# +# * Classic +# $ NIX_PATH="nixpkgs=$PWD" nix-shell -p tests.trivial-builders.references.testScriptBin --run references-test +# +# * Flake-based +# $ nix run .#tests.trivial-builders.references.testScriptBin +# +# or in the build sandbox with a ~20s VM overhead: +# +# * Classic +# $ nix-build --no-out-link -A tests.trivial-builders.references +# +# * Flake-based +# $ nix build -L --no-link .#tests.trivial-builders.references +# +# -------------------------------------------------------------------------- # + +let + # Map each attribute to an element specification of Bash associative arrary + # and concatenate them with white spaces, to be used to define a + # one-line Bash associative array. + samplesToString = attrs: + lib.concatMapStringsSep " " (name: "[${name}]=${lib.escapeShellArg "${attrs.${name}}"}") (builtins.attrNames attrs); + + references = lib.mapAttrs (n: v: writeReferencesToFile v) samples; + directReferences = lib.mapAttrs (n: v: writeDirectReferencesToFile v) samples; + + testScriptBin = stdenvNoCC.mkDerivation (finalAttrs: { + name = "references-test"; + + src = ./references-test.sh; + dontUnpack = true; + dontBuild = true; + + installPhase = '' + runHook preInstall + mkdir -p "$out/bin" + substitute "$src" "$out/bin/${finalAttrs.meta.mainProgram}" \ + --replace "@SAMPLES@" ${lib.escapeShellArg (samplesToString samples)} \ + --replace "@REFERENCES@" ${lib.escapeShellArg (samplesToString references)} \ + --replace "@DIRECT_REFS@" ${lib.escapeShellArg (samplesToString directReferences)} + runHook postInstall + chmod +x "$out/bin/${finalAttrs.meta.mainProgram}" + ''; + + doInstallCheck = true; + nativeInstallCheckInputs = [ + shellcheck-minimal + ]; + installCheckPhase = '' + runHook preInstallCheck + shellcheck "$out/bin/${finalAttrs.meta.mainProgram}" + runHook postInstallCheck + ''; + + passthru = { + inherit + directReferences + references + samples + ; + }; + + meta = with lib; { + mainProgram = "references-test"; + }; + }); +in +testers.nixosTest { + name = "nixpkgs-trivial-builders"; + nodes.machine = { ... }: { + virtualisation.writableStore = true; + + # Test runs without network, so we don't substitute and prepare our deps + nix.settings.substituters = lib.mkForce [ ]; + environment.etc."pre-built-paths".source = writeText "pre-built-paths" ( + builtins.toJSON [ testScriptBin ] + ); + }; + testScript = + '' + machine.succeed(""" + ${lib.getExe testScriptBin} 2>/dev/console + """) + ''; + passthru = { + inherit + directReferences + references + samples + testScriptBin + ; + }; + meta = { + maintainers = with lib.maintainers; [ + roberth + ShamrockLee + ]; + }; +} diff --git a/pkgs/build-support/trivial-builders/test/references/references-test.sh b/pkgs/build-support/trivial-builders/test/references/references-test.sh new file mode 100755 index 0000000000000..1b8f8e1504ec8 --- /dev/null +++ b/pkgs/build-support/trivial-builders/test/references/references-test.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# -------------------------------------------------------------------------- # +# +# trivial-builders test +# +# -------------------------------------------------------------------------- # +# +# Execute this build script directly (quick): +# +# * Classic +# $ NIX_PATH="nixpkgs=$PWD" nix-shell -p tests.trivial-builders.references.testScriptBin --run references-test +# +# * Flake-based +# $ nix run .#tests.trivial-builders.references.testScriptBin +# +# or in the build sandbox with a ~20s VM overhead: +# +# * Classic +# $ nix-build --no-out-link -A tests.trivial-builders.references +# +# * Flake-based +# $ nix build -L --no-link .#tests.trivial-builders.references +# +# -------------------------------------------------------------------------- # + +# strict bash +set -euo pipefail + +# debug +# set -x +# PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +cd "$(dirname "${BASH_SOURCE[0]}")" # nixpkgs root + + # Injected by Nix (to avoid evaluating in a derivation) + # turn them into arrays + # shellcheck disable=SC2206 # deliberately unquoted + declare -A samples=( @SAMPLES@ ) + # shellcheck disable=SC2206 # deliberately unquoted + declare -A directRefs=( @DIRECT_REFS@ ) + # shellcheck disable=SC2206 # deliberately unquoted + declare -A references=( @REFERENCES@ ) + +echo >&2 Testing direct references... +for i in "${!samples[@]}"; do + echo >&2 Checking "$i" "${samples[$i]}" "${directRefs[$i]}" + diff -U3 \ + <(sort <"${directRefs[$i]}") \ + <(nix-store -q --references "${samples[$i]}" | sort) +done + +echo >&2 Testing closure... +for i in "${!samples[@]}"; do + echo >&2 Checking "$i" "${samples[$i]}" "${references[$i]}" + diff -U3 \ + <(sort <"${references[$i]}") \ + <(nix-store -q --requisites "${samples[$i]}" | sort) +done + +echo 'OK!' diff --git a/pkgs/build-support/trivial-builders/test/references/samples.nix b/pkgs/build-support/trivial-builders/test/references/samples.nix new file mode 100644 index 0000000000000..3afb970c08aeb --- /dev/null +++ b/pkgs/build-support/trivial-builders/test/references/samples.nix @@ -0,0 +1,30 @@ +{ lib +, runCommand +, writeText +, emptyFile +, emptyDirectory +, figlet +, hello +, zlib +}: +{ + inherit + figlet + hello + zlib + ; + zlib-dev = zlib.dev; + norefs = writeText "hi" "hello"; + norefsDup = writeText "hi" "hello"; + helloRef = writeText "hi" "hello ${hello}"; + helloRefDup = writeText "hi" "hello ${hello}"; + path = ./samples.nix; + pathLike.outPath = ./samples.nix; + helloFigletRef = writeText "hi" "hello ${hello} ${figlet}"; + selfRef = runCommand "self-ref-1" { } "echo $out >$out"; + selfRef2 = runCommand "self-ref-2" { } ''echo "${figlet}, $out" >$out''; + inherit + emptyFile + emptyDirectory + ; +} diff --git a/pkgs/build-support/trivial-builders/test/sample.nix b/pkgs/build-support/trivial-builders/test/sample.nix deleted file mode 100644 index a4eedce8417eb..0000000000000 --- a/pkgs/build-support/trivial-builders/test/sample.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ pkgs ? import ../../../.. { config = { }; overlays = [ ]; } }: -let - inherit (pkgs) - figlet - zlib - hello - writeText - runCommand - ; -in -{ - hello = hello; - figlet = figlet; - zlib = zlib; - zlib-dev = zlib.dev; - norefs = writeText "hi" "hello"; - norefsDup = writeText "hi" "hello"; - helloRef = writeText "hi" "hello ${hello}"; - helloRefDup = writeText "hi" "hello ${hello}"; - path = ./invoke-writeReferencesToFile.nix; - pathLike.outPath = ./invoke-writeReferencesToFile.nix; - helloFigletRef = writeText "hi" "hello ${hello} ${figlet}"; - selfRef = runCommand "self-ref-1" {} "echo $out >$out"; - selfRef2 = runCommand "self-ref-2" {} ''echo "${figlet}, $out" >$out''; - inherit (pkgs) - emptyFile - emptyDirectory - ; -} diff --git a/pkgs/build-support/trivial-builders/test/writeShellApplication.nix b/pkgs/build-support/trivial-builders/test/writeShellApplication.nix new file mode 100644 index 0000000000000..c50f5a4d283f9 --- /dev/null +++ b/pkgs/build-support/trivial-builders/test/writeShellApplication.nix @@ -0,0 +1,141 @@ +# Run with: +# nix-build -A tests.trivial-builders.writeShellApplication +{ writeShellApplication +, writeTextFile +, runCommand +, lib +, linkFarm +, diffutils +, hello +}: +let + checkShellApplication = args@{name, expected, ...}: + let + writeShellApplicationArgs = builtins.removeAttrs args ["expected"]; + script = writeShellApplication writeShellApplicationArgs; + executable = lib.getExe script; + expected' = writeTextFile { + name = "${name}-expected"; + text = expected; + }; + actual = "${name}-actual"; + in + runCommand name { } '' + echo "Running test executable ${name}" + ${executable} > ${actual} + echo "Got output from test executable:" + cat ${actual} + echo "Checking test output against expected output:" + ${diffutils}/bin/diff --color --unified ${expected'} ${actual} + touch $out + ''; +in +linkFarm "writeShellApplication-tests" { + test-meta = + let + script = writeShellApplication { + name = "test-meta"; + text = ""; + meta.description = "A test for the `writeShellApplication` `meta` argument."; + }; + in + assert script.meta.mainProgram == "test-meta"; + assert script.meta.description == "A test for the `writeShellApplication` `meta` argument."; + script; + + test-runtime-inputs = + checkShellApplication { + name = "test-runtime-inputs"; + text = '' + hello + ''; + runtimeInputs = [ hello ]; + expected = "Hello, world!\n"; + }; + + test-runtime-env = + checkShellApplication { + name = "test-runtime-env"; + runtimeEnv = { + MY_COOL_ENV_VAR = "my-cool-env-value"; + MY_OTHER_COOL_ENV_VAR = "my-other-cool-env-value"; + # Check that we can serialize a bunch of different types: + BOOL = true; + INT = 1; + LIST = [1 2 3]; + MAP = { + a = "a"; + b = "b"; + }; + }; + text = '' + echo "$MY_COOL_ENV_VAR" + echo "$MY_OTHER_COOL_ENV_VAR" + ''; + expected = '' + my-cool-env-value + my-other-cool-env-value + ''; + }; + + test-check-phase = + checkShellApplication { + name = "test-check-phase"; + text = ""; + checkPhase = '' + echo "echo -n hello" > $target + ''; + expected = "hello"; + }; + + test-argument-forwarding = + checkShellApplication { + name = "test-argument-forwarding"; + text = ""; + derivationArgs.MY_BUILD_TIME_VARIABLE = "puppy"; + derivationArgs.postCheck = '' + if [[ "$MY_BUILD_TIME_VARIABLE" != puppy ]]; then + echo "\$MY_BUILD_TIME_VARIABLE is not set to 'puppy'!" + exit 1 + fi + ''; + meta.description = "A test checking that `writeShellApplication` forwards extra arguments to `stdenv.mkDerivation`."; + expected = ""; + }; + + test-exclude-shell-checks = writeShellApplication { + name = "test-exclude-shell-checks"; + excludeShellChecks = [ "SC2016" ]; + text = '' + # Triggers SC2016: Expressions don't expand in single quotes, use double + # quotes for that. + echo '$SHELL' + ''; + }; + + test-bash-options-pipefail = checkShellApplication { + name = "test-bash-options-pipefail"; + text = '' + touch my-test-file + echo puppy | grep doggy | sed 's/doggy/puppy/g' + # ^^^^^^^^^^ This will fail. + true + ''; + # Don't use `pipefail`: + bashOptions = ["errexit" "nounset"]; + expected = ""; + }; + + test-bash-options-nounset = checkShellApplication { + name = "test-bash-options-nounset"; + text = '' + echo -n "$someUndefinedVariable" + ''; + # Don't use `nounset`: + bashOptions = []; + # Don't warn about the undefined variable at build time: + excludeShellChecks = [ "SC2154" ]; + expected = ""; + }; + +} diff --git a/pkgs/build-support/trivial-builders/test/writeStringReferencesToFile.nix b/pkgs/build-support/trivial-builders/test/writeStringReferencesToFile.nix index b93b43b74aa49..dedd7e183a178 100644 --- a/pkgs/build-support/trivial-builders/test/writeStringReferencesToFile.nix +++ b/pkgs/build-support/trivial-builders/test/writeStringReferencesToFile.nix @@ -1,14 +1,13 @@ -{ callPackage, lib, pkgs, runCommand, writeText, writeStringReferencesToFile }: +{ callPackage, lib, pkgs, runCommand, samples, writeText, writeStringReferencesToFile }: let - sample = import ./sample.nix { inherit pkgs; }; - samplePaths = lib.unique (lib.attrValues sample); + samplePaths = lib.unique (lib.attrValues samples); stri = x: "${x}"; sampleText = writeText "sample-text" (lib.concatStringsSep "\n" (lib.unique (map stri samplePaths))); stringReferencesText = writeStringReferencesToFile ((lib.concatMapStringsSep "fillertext" stri - (lib.attrValues sample)) + '' + (lib.attrValues samples)) + '' STORE=${builtins.storeDir};\nsystemctl start bar-foo.service ''); in diff --git a/pkgs/build-support/vm/default.nix b/pkgs/build-support/vm/default.nix index 4ec5531192dc8..1184d43ccb2cc 100644 --- a/pkgs/build-support/vm/default.nix +++ b/pkgs/build-support/vm/default.nix @@ -1034,24 +1034,46 @@ rec { }; debian11i386 = { - name = "debian-11.6-bullseye-i386"; - fullName = "Debian 11.6 Bullseye (i386)"; + name = "debian-11.8-bullseye-i386"; + fullName = "Debian 11.8 Bullseye (i386)"; packagesList = fetchurl { - url = "https://snapshot.debian.org/archive/debian/20230131T034648Z/dists/bullseye/main/binary-i386/Packages.xz"; - hash = "sha256-z9eG7RlvelEnZAaeCfIO+XxTZVL3d+zTA7ShU43l/pw="; + url = "https://snapshot.debian.org/archive/debian/20231124T031419Z/dists/bullseye/main/binary-i386/Packages.xz"; + hash = "sha256-0bKSLLPhEC7FB5D1NA2jaQP0wTe/Qp1ddiA/NDVjRaI="; }; - urlPrefix = "https://snapshot.debian.org/archive/debian/20230131T034648Z"; + urlPrefix = "https://snapshot.debian.org/archive/debian/20231124T031419Z"; packages = commonDebianPackages; }; debian11x86_64 = { - name = "debian-11.6-bullseye-amd64"; - fullName = "Debian 11.6 Bullseye (amd64)"; + name = "debian-11.8-bullseye-amd64"; + fullName = "Debian 11.8 Bullseye (amd64)"; packagesList = fetchurl { - url = "https://snapshot.debian.org/archive/debian/20230131T034648Z/dists/bullseye/main/binary-amd64/Packages.xz"; - hash = "sha256-mz0eCWdn6uWt40OxsSPheHzEnMeLE52yR/vpb48/VF0="; + url = "https://snapshot.debian.org/archive/debian/20231124T031419Z/dists/bullseye/main/binary-amd64/Packages.xz"; + hash = "sha256-CYPsGgQgJZkh3JmbcAQkYDWP193qrkOADOgrMETZIeo="; }; - urlPrefix = "https://snapshot.debian.org/archive/debian/20230131T034648Z"; + urlPrefix = "https://snapshot.debian.org/archive/debian/20231124T031419Z"; + packages = commonDebianPackages; + }; + + debian12i386 = { + name = "debian-12.2-bookworm-i386"; + fullName = "Debian 12.2 Bookworm (i386)"; + packagesList = fetchurl { + url = "https://snapshot.debian.org/archive/debian/20231124T031419Z/dists/bookworm/main/binary-i386/Packages.xz"; + hash = "sha256-OeN9Q2HFM3GsPNhOa4VhM7qpwT66yUNwC+6Z8SbGEeQ="; + }; + urlPrefix = "https://snapshot.debian.org/archive/debian/20231124T031419Z"; + packages = commonDebianPackages; + }; + + debian12x86_64 = { + name = "debian-12.2-bookworm-amd64"; + fullName = "Debian 12.2 Bookworm (amd64)"; + packagesList = fetchurl { + url = "https://snapshot.debian.org/archive/debian/20231124T031419Z/dists/bookworm/main/binary-amd64/Packages.xz"; + hash = "sha256-SZDElRfe9BlBwDlajQB79Qdn08rv8whYoQDeVCveKVs="; + }; + urlPrefix = "https://snapshot.debian.org/archive/debian/20231124T031419Z"; packages = commonDebianPackages; }; }; diff --git a/pkgs/build-support/writers/data.nix b/pkgs/build-support/writers/data.nix index 48f9bc547ed39..45ed5360eaeba 100644 --- a/pkgs/build-support/writers/data.nix +++ b/pkgs/build-support/writers/data.nix @@ -1,4 +1,4 @@ -{ lib, runCommand, dasel }: +{ lib, pkgs, formats, runCommand, dasel }: let daselBin = lib.getExe dasel; @@ -23,7 +23,7 @@ rec { # writeJSON = makeDataWriter { input = builtins.toJSON; output = "cp $inputPath $out"; }; # myConfig = writeJSON "config.json" { hello = "world"; } # - makeDataWriter = { input ? lib.id, output ? "cp $inputPath $out" }: nameOrPath: data: + makeDataWriter = lib.warn "pkgs.writers.makeDataWriter is deprecated. Use pkgs.writeTextFile." ({ input ? lib.id, output ? "cp $inputPath $out" }: nameOrPath: data: assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null); let name = last (builtins.split "/" nameOrPath); @@ -40,41 +40,25 @@ rec { mkdir -p $out/$(dirname "${nameOrPath}") mv tmp $out/${nameOrPath} ''} - ''; + ''); - # Writes the content to text. - # - # Example: - # writeText "filename.txt" "file content" - writeText = makeDataWriter { - input = toString; - output = "cp $inputPath $out"; - }; + inherit (pkgs) writeText; # Writes the content to a JSON file. # # Example: # writeJSON "data.json" { hello = "world"; } - writeJSON = makeDataWriter { - input = builtins.toJSON; - output = "${daselBin} -f $inputPath -r json -w json > $out"; - }; + writeJSON = (pkgs.formats.json {}).generate; # Writes the content to a TOML file. # # Example: # writeTOML "data.toml" { hello = "world"; } - writeTOML = makeDataWriter { - input = builtins.toJSON; - output = "${daselBin} -f $inputPath -r json -w toml > $out"; - }; + writeTOML = (pkgs.formats.toml {}).generate; # Writes the content to a YAML file. # # Example: # writeYAML "data.yaml" { hello = "world"; } - writeYAML = makeDataWriter { - input = builtins.toJSON; - output = "${daselBin} -f $inputPath -r json -w yaml > $out"; - }; + writeYAML = (pkgs.formats.yaml {}).generate; } diff --git a/pkgs/build-support/writers/default.nix b/pkgs/build-support/writers/default.nix index a161322cd35be..cadb697814815 100644 --- a/pkgs/build-support/writers/default.nix +++ b/pkgs/build-support/writers/default.nix @@ -1,5 +1,6 @@ { config, lib, callPackages }: +# If you are reading this, you can test these writers by running: nix-build . -A tests.writers let aliases = if config.allowAliases then (import ./aliases.nix lib) else prev: {}; diff --git a/pkgs/build-support/writers/scripts.nix b/pkgs/build-support/writers/scripts.nix index 184ecee687770..8a23e5dd4a66d 100644 --- a/pkgs/build-support/writers/scripts.nix +++ b/pkgs/build-support/writers/scripts.nix @@ -13,7 +13,7 @@ let in rec { # Base implementation for non-compiled executables. - # Takes an interpreter, for example `${pkgs.bash}/bin/bash` + # Takes an interpreter, for example `${lib.getExe pkgs.bash}` # # Examples: # writeBash = makeScriptWriter { interpreter = "${pkgs.bash}/bin/bash"; } @@ -116,7 +116,7 @@ rec { # echo hello world # '' writeBash = makeScriptWriter { - interpreter = "${pkgs.bash}/bin/bash"; + interpreter = "${lib.getExe pkgs.bash}"; }; # Like writeScriptBin but the first line is a shebang to bash @@ -130,7 +130,7 @@ rec { # echo hello world # '' writeDash = makeScriptWriter { - interpreter = "${pkgs.dash}/bin/dash"; + interpreter = "${lib.getExe pkgs.dash}"; }; # Like writeScriptBin but the first line is a shebang to dash @@ -144,8 +144,8 @@ rec { # echo hello world # '' writeFish = makeScriptWriter { - interpreter = "${pkgs.fish}/bin/fish --no-config"; - check = "${pkgs.fish}/bin/fish --no-config --no-execute"; # syntax check only + interpreter = "${lib.getExe pkgs.fish} --no-config"; + check = "${lib.getExe pkgs.fish} --no-config --no-execute"; # syntax check only }; # Like writeScriptBin but the first line is a shebang to fish @@ -175,7 +175,7 @@ rec { in makeBinWriter { compileScript = '' cp $contentPath tmp.hs - ${ghc.withPackages (_: libraries )}/bin/ghc ${lib.escapeShellArgs ghcArgs'} tmp.hs + ${(ghc.withPackages (_: libraries ))}/bin/ghc ${lib.escapeShellArgs ghcArgs'} tmp.hs mv tmp $out ''; inherit strip; @@ -185,6 +185,85 @@ rec { writeHaskellBin = name: writeHaskell "/bin/${name}"; + # Like writeScript but the first line is a shebang to nu + # + # Example: + # writeNu "example" '' + # echo hello world + # '' + writeNu = makeScriptWriter { + interpreter = "${lib.getExe pkgs.nushell} --no-config-file"; + }; + + # Like writeScriptBin but the first line is a shebang to nu + writeNuBin = name: + writeNu "/bin/${name}"; + + # makeRubyWriter takes ruby and compatible rubyPackages and produces ruby script writer, + # If any libraries are specified, ruby.withPackages is used as interpreter, otherwise the "bare" ruby is used. + makeRubyWriter = ruby: rubyPackages: buildRubyPackages: name: { libraries ? [], }: + makeScriptWriter { + interpreter = + if libraries == [] + then "${ruby}/bin/ruby" + else "${(ruby.withPackages (ps: libraries))}/bin/ruby"; + # Rubocop doesnt seem to like running in this fashion. + #check = (writeDash "rubocop.sh" '' + # exec ${lib.getExe buildRubyPackages.rubocop} "$1" + #''); + } name; + + # Like writeScript but the first line is a shebang to ruby + # + # Example: + # writeRuby "example" '' + # puts "hello world" + # '' + writeRuby = makeRubyWriter pkgs.ruby pkgs.rubyPackages buildPackages.rubyPackages; + + writeRubyBin = name: + writeRuby "/bin/${name}"; + + # makeLuaWriter takes lua and compatible luaPackages and produces lua script writer, + # which validates the script with luacheck at build time. If any libraries are specified, + # lua.withPackages is used as interpreter, otherwise the "bare" lua is used. + makeLuaWriter = lua: luaPackages: buildLuaPackages: name: { libraries ? [], }: + makeScriptWriter { + interpreter = lua.interpreter; + # if libraries == [] + # then lua.interpreter + # else (lua.withPackages (ps: libraries)).interpreter + # This should support packages! I just cant figure out why some dependency collision happens whenever I try to run this. + check = (writeDash "luacheck.sh" '' + exec ${buildLuaPackages.luacheck}/bin/luacheck "$1" + ''); + } name; + + # writeLua takes a name an attributeset with libraries and some lua source code and + # returns an executable (should also work with luajit) + # + # Example: + # writeLua "test_lua" { libraries = [ pkgs.luaPackages.say ]; } '' + # s = require("say") + # s:set_namespace("en") + # + # s:set('money', 'I have %s dollars') + # s:set('wow', 'So much money!') + # + # print(s('money', {1000})) -- I have 1000 dollars + # + # s:set_namespace("fr") -- switch to french! + # s:set('wow', "Tant d'argent!") + # + # print(s('wow')) -- Tant d'argent! + # s:set_namespace("en") -- switch back to english! + # print(s('wow')) -- So much money! + # '' + writeLua = makeLuaWriter pkgs.lua pkgs.luaPackages buildPackages.luaPackages; + + writeLuaBin = name: + writeLua "/bin/${name}"; + writeRust = name: { rustc ? pkgs.rustc, rustcArgs ? [], @@ -196,7 +275,7 @@ rec { makeBinWriter { compileScript = '' cp "$contentPath" tmp.rs - PATH=${lib.makeBinPath [pkgs.gcc]} ${lib.getBin rustc}/bin/rustc ${lib.escapeShellArgs rustcArgs} ${lib.escapeShellArgs darwinArgs} -o "$out" tmp.rs + PATH=${lib.makeBinPath [pkgs.gcc]} ${rustc}/bin/rustc ${lib.escapeShellArgs rustcArgs} ${lib.escapeShellArgs darwinArgs} -o "$out" tmp.rs ''; inherit strip; } name; @@ -225,7 +304,7 @@ rec { }; in writeDash name '' export NODE_PATH=${node-env}/lib/node_modules - exec ${pkgs.nodejs}/bin/node ${pkgs.writeText "js" content} "$@" + exec ${lib.getExe pkgs.nodejs} ${pkgs.writeText "js" content} "$@" ''; # writeJSBin takes the same arguments as writeJS but outputs a directory (like writeScriptBin) @@ -260,7 +339,7 @@ rec { # '' writePerl = name: { libraries ? [] }: makeScriptWriter { - interpreter = "${pkgs.perl.withPackages (p: libraries)}/bin/perl"; + interpreter = "${lib.getExe (pkgs.perl.withPackages (p: libraries))}"; } name; # writePerlBin takes the same arguments as writePerl but outputs a directory (like writeScriptBin) @@ -276,9 +355,11 @@ rec { in makeScriptWriter { interpreter = - if libraries == [] - then python.interpreter - else (python.withPackages (ps: libraries)).interpreter + if pythonPackages != pkgs.pypy2Packages || pythonPackages != pkgs.pypy3Packages then + if libraries == [] + then python.interpreter + else (python.withPackages (ps: libraries)).interpreter + else python.interpreter ; check = optionalString python.isPy3k (writeDash "pythoncheck.sh" '' exec ${buildPythonPackages.flake8}/bin/flake8 --show-source ${ignoreAttribute} "$1" @@ -358,7 +439,7 @@ rec { export DOTNET_CLI_TELEMETRY_OPTOUT=1 export DOTNET_NOLOGO=1 script="$1"; shift - ${dotnet-sdk}/bin/dotnet fsi --quiet --nologo --readline- ${fsi-flags} "$@" < "$script" + ${lib.getExe dotnet-sdk} fsi --quiet --nologo --readline- ${fsi-flags} "$@" < "$script" ''; in content: makeScriptWriter { diff --git a/pkgs/build-support/writers/test.nix b/pkgs/build-support/writers/test.nix index 2411f8c03a70b..982c550d28e08 100644 --- a/pkgs/build-support/writers/test.nix +++ b/pkgs/build-support/writers/test.nix @@ -6,10 +6,16 @@ , pypy2Packages , python3Packages , pypy3Packages +, luaPackages +, rubyPackages , runCommand +, testers , writers , writeText }: + +# If you are reading this, you can test these writers by running: nix-build . -A tests.writers + with writers; let expectSuccess = test: @@ -36,14 +42,7 @@ let let expectedFile = writeText "${file.name}-expected" expected; in - runCommand "run-${file.name}" {} '' - if ! diff -u ${file} ${expectedFile}; then - echo 'test ${file.name} failed' - exit 1 - fi - - touch $out - ''; + testers.testEqualContents { expected = expectedFile; actual = file; assertion = "${file.name} matches"; }; in lib.recurseIntoAttrs { bin = lib.recurseIntoAttrs { @@ -94,15 +93,6 @@ lib.recurseIntoAttrs { print "success\n" if true; ''); - pypy2 = expectSuccessBin (writePyPy2Bin "test-writers-pypy2-bin" { libraries = [ pypy2Packages.enum ]; } '' - from enum import Enum - - class Test(Enum): - a = "success" - - print Test.a - ''); - python3 = expectSuccessBin (writePython3Bin "test-writers-python3-bin" { libraries = [ python3Packages.pyyaml ]; } '' import yaml @@ -112,14 +102,47 @@ lib.recurseIntoAttrs { print(y[0]['test']) ''); - pypy3 = expectSuccessBin (writePyPy3Bin "test-writers-pypy3-bin" { libraries = [ pypy3Packages.pyyaml ]; } '' - import yaml - - y = yaml.safe_load(""" - - test: success - """) - print(y[0]['test']) - ''); + # Commented out because of this issue: https://github.com/NixOS/nixpkgs/issues/39356 + + #pypy2 = expectSuccessBin (writePyPy2Bin "test-writers-pypy2-bin" { libraries = [ pypy2Packages.enum ]; } '' + # from enum import Enum + # + # class Test(Enum): + # a = "success" + # + # print Test.a + #''); + + #pypy3 = expectSuccessBin (writePyPy3Bin "test-writers-pypy3-bin" { libraries = [ pypy3Packages.pyyaml ]; } '' + # import yaml + # + # y = yaml.safe_load(""" + # - test: success + # """) + # print(y[0]['test']) + #''); + + # Could not test this because of external package issues :( + #lua = writeLuaBin "test-writers-lua-bin" { libraries = [ pkgs.luaPackages.say ]; } '' + # s = require("say") + # s:set_namespace("en") + + # s:set('money', 'I have %s dollars') + # s:set('wow', 'So much money!') + + # print(s('money', {1000})) -- I have 1000 dollars + + # s:set_namespace("fr") -- switch to french! + # s:set('wow', "Tant d'argent!") + + # print(s('wow')) -- Tant d'argent! + # s:set_namespace("en") -- switch back to english! + # print(s('wow')) -- So much money! + #''; + + #ruby = expectSuccessBin (writeRubyBin "test-writers-ruby-bin" { libraries = [ rubyPackages.rubocop ]; } '' + #puts "This should work!" + #''); }; simple = lib.recurseIntoAttrs { @@ -137,6 +160,10 @@ lib.recurseIntoAttrs { end ''); + nu = expectSuccess (writeNu "test-writers-nushell" '' + echo "success" + ''); + haskell = expectSuccess (writeHaskell "test-writers-haskell" { libraries = [ haskellPackages.acme-default ]; } '' import Data.Default @@ -164,15 +191,6 @@ lib.recurseIntoAttrs { print "success\n" if true; ''); - pypy2 = expectSuccess (writePyPy2 "test-writers-pypy2" { libraries = [ pypy2Packages.enum ]; } '' - from enum import Enum - - class Test(Enum): - a = "success" - - print Test.a - ''); - python3 = expectSuccess (writePython3 "test-writers-python3" { libraries = [ python3Packages.pyyaml ]; } '' import yaml @@ -182,20 +200,33 @@ lib.recurseIntoAttrs { print(y[0]['test']) ''); - pypy3 = expectSuccess (writePyPy3 "test-writers-pypy3" { libraries = [ pypy3Packages.pyyaml ]; } '' - import yaml - - y = yaml.safe_load(""" - - test: success - """) - print(y[0]['test']) - ''); + # Commented out because of this issue: https://github.com/NixOS/nixpkgs/issues/39356 + + #pypy2 = expectSuccessBin (writePyPy2Bin "test-writers-pypy2-bin" { libraries = [ pypy2Packages.enum ]; } '' + # from enum import Enum + # + # class Test(Enum): + # a = "success" + # + # print Test.a + #''); + + #pypy3 = expectSuccessBin (writePyPy3Bin "test-writers-pypy3-bin" { libraries = [ pypy3Packages.pyyaml ]; } '' + # import yaml + # + # y = yaml.safe_load(""" + # - test: success + # """) + # print(y[0]['test']) + #''); fsharp = expectSuccess (makeFSharpWriter { libraries = { fetchNuGet }: [ (fetchNuGet { pname = "FSharp.SystemTextJson"; version = "0.17.4"; sha256 = "1bplzc9ybdqspii4q28l8gmfvzpkmgq5l1hlsiyg2h46w881lwg2"; }) + (fetchNuGet { pname = "System.Text.Json"; version = "4.6.0"; sha256 = "0ism236hwi0k6axssfq58s1d8lihplwiz058pdvl8al71hagri39"; }) ]; } "test-writers-fsharp" '' + #r "nuget: FSharp.SystemTextJson, 0.17.4" module Json = @@ -214,9 +245,9 @@ lib.recurseIntoAttrs { |> printfn "%s" ''); - pypy2NoLibs = expectSuccess (writePyPy2 "test-writers-pypy2-no-libs" {} '' - print("success") - ''); + #pypy2NoLibs = expectSuccess (writePyPy2 "test-writers-pypy2-no-libs" {} '' + # print("success") + #''); python3NoLibs = expectSuccess (writePython3 "test-writers-python3-no-libs" {} '' print("success") @@ -228,6 +259,14 @@ lib.recurseIntoAttrs { fsharpNoNugetDeps = expectSuccess (writeFSharp "test-writers-fsharp-no-nuget-deps" '' printfn "success" + ''); + + luaNoLibs = expectSuccess (writeLua "test-writers-lua-no-libs" {} '' + print("success") + ''); + + rubyNoLibs = expectSuccess (writeRuby "test-writers-ruby-no-libs" {} '' + puts "success" ''); }; @@ -261,7 +300,9 @@ lib.recurseIntoAttrs { toml = expectDataEqual { file = writeTOML "data.toml" { hello = "world"; }; - expected = "hello = 'world'\n"; + expected = '' + hello = "world" + ''; }; yaml = expectDataEqual { |