Merge master into staging-next

7 files changed, 138 insertions, 53 deletions

diff --git a/pkgs/build-support/dlang/builddubpackage/default.nix b/pkgs/build-support/dlang/builddubpackage/default.nix
index 9295445b0f7c7..31454d5cd69bf 100644
--- a/pkgs/build-support/dlang/builddubpackage/default.nix
+++ b/pkgs/build-support/dlang/builddubpackage/default.nix
@@ -2,6 +2,7 @@
   lib,
   stdenv,
   fetchurl,
+  fetchgit,
   linkFarm,
   dub,
   ldc,
@@ -43,11 +44,27 @@ let
       };
     };
 
+  makeGitDep =
+    {
+      pname,
+      version,
+      repository,
+      sha256,
+    }:
+    {
+      inherit pname version;
+      src = fetchgit {
+        url = repository;
+        rev = version;
+        inherit sha256;
+      };
+    };
+
   lockJson = if lib.isPath dubLock then lib.importJSON dubLock else dubLock;
+  depsRaw = lib.mapAttrsToList (pname: args: { inherit pname; } // args) lockJson.dependencies;
 
-  lockedDeps = lib.mapAttrsToList (
-    pname: { version, sha256 }: makeDubDep { inherit pname version sha256; }
-  ) lockJson.dependencies;
+  dubDeps = map makeDubDep (lib.filter (args: !(args ? repository)) depsRaw);
+  gitDeps = map makeGitDep (lib.filter (args: args ? repository) depsRaw);
 
   # a directory with multiple single element registries
   # one big directory with all .zip files leads to version parsing errors
@@ -56,7 +73,7 @@ let
     map (dep: {
       name = "${dep.pname}/${dep.pname}-${dep.version}.zip";
       path = dep.src;
-    }) lockedDeps
+    }) dubDeps
   );
 
   combinedFlags = "--skip-registry=all --compiler=${lib.getExe compiler} ${toString dubFlags}";
@@ -79,12 +96,18 @@ stdenv.mkDerivation (
         runHook preConfigure
 
         export DUB_HOME="$NIX_BUILD_TOP/.dub"
-        mkdir -p $DUB_HOME
+        mkdir -p "$DUB_HOME"
 
-        # register dependencies
+        # register dub dependencies
         ${lib.concatMapStringsSep "\n" (dep: ''
           dub fetch ${dep.pname}@${dep.version} --cache=user --skip-registry=standard --registry=file://${dubRegistryBase}/${dep.pname}
-        '') lockedDeps}
+        '') dubDeps}
+
+        # register git dependencies
+        ${lib.concatMapStringsSep "\n" (dep: ''
+          mkdir -p "$DUB_HOME/packages/${dep.pname}/${dep.version}"
+          cp -r --no-preserve=all ${dep.src} "$DUB_HOME/packages/${dep.pname}/${dep.version}/${dep.pname}"
+        '') gitDeps}
 
         runHook postConfigure
       '';
diff --git a/pkgs/build-support/dlang/dub-to-nix/default.nix b/pkgs/build-support/dlang/dub-to-nix/default.nix
index 53a2e99c18df5..87db7eed09331 100644
--- a/pkgs/build-support/dlang/dub-to-nix/default.nix
+++ b/pkgs/build-support/dlang/dub-to-nix/default.nix
@@ -4,8 +4,15 @@
   makeWrapper,
   python3,
   nix,
+  nix-prefetch-git,
 }:
 
+let
+  binPath = lib.makeBinPath [
+    nix
+    nix-prefetch-git
+  ];
+in
 runCommand "dub-to-nix"
   {
     nativeBuildInputs = [ makeWrapper ];
@@ -15,5 +22,5 @@ runCommand "dub-to-nix"
     install -Dm755 ${./dub-to-nix.py} "$out/bin/dub-to-nix"
     patchShebangs "$out/bin/dub-to-nix"
     wrapProgram "$out/bin/dub-to-nix" \
-        --prefix PATH : ${lib.makeBinPath [ nix ]}
+        --prefix PATH : ${binPath}
   ''
diff --git a/pkgs/build-support/dlang/dub-to-nix/dub-to-nix.py b/pkgs/build-support/dlang/dub-to-nix/dub-to-nix.py
index 48a9f241348a4..fbb51960ad7b6 100644
--- a/pkgs/build-support/dlang/dub-to-nix/dub-to-nix.py
+++ b/pkgs/build-support/dlang/dub-to-nix/dub-to-nix.py
@@ -4,10 +4,13 @@ import sys
 import json
 import os
 import subprocess
+import string
+
 
 def eprint(text: str):
     print(text, file=sys.stderr)
 
+
 if not os.path.exists("dub.selections.json"):
     eprint("The file `dub.selections.json` does not exist in the current working directory")
     eprint("run `dub upgrade --annotate` to generate it")
@@ -16,24 +19,53 @@ if not os.path.exists("dub.selections.json"):
 with open("dub.selections.json") as f:
     selectionsJson = json.load(f)
 
-versionDict: dict[str, str] = selectionsJson["versions"]
+depsDict: dict = selectionsJson["versions"]
+
+# For each dependency expand non-expanded version into a dict with a "version" key
+depsDict = {pname: (versionOrDepDict if isinstance(versionOrDepDict, dict) else {"version": versionOrDepDict}) for (pname, versionOrDepDict) in depsDict.items()}
+
+# Don't process path-type selections
+depsDict = {pname: depDict for (pname, depDict) in depsDict.items() if "path" not in depDict}
 
-for pname in versionDict:
-    version = versionDict[pname]
+# Pre-validate selections before trying to fetch
+for pname in depsDict:
+    depDict = depsDict[pname]
+    version = depDict["version"]
     if version.startswith("~"):
-        eprint(f'Package "{pname}" has a branch-type version "{version}", which doesn\'t point to a fixed version')
-        eprint("You can resolve it by manually changing the required version to a fixed one inside `dub.selections.json`")
-        eprint("When packaging, you might need to create a patch for `dub.sdl` or `dub.json` to accept the changed version")
+        eprint(f'Expected version of "{pname}" to be non-branch type')
+        eprint(f'Found: "{version}"')
+        eprint("Please specify a non-branch version inside `dub.selections.json`")
+        eprint("When packaging, you might also need to patch the version value in the appropriate places (`dub.selections.json`, dub.sdl`, `dub.json`)")
         sys.exit(1)
+    if "repository" in depDict:
+        repository = depDict["repository"]
+        if not repository.startswith("git+"):
+            eprint(f'Expected repository field of "{pname}" to begin with "git+"')
+            eprint(f'Found: "{repository}"')
+            sys.exit(1)
+        if (len(version) < 7 or len(version) > 40 or not all(c in string.hexdigits for c in version)):
+            eprint(f'Expected version field of "{pname}" to begin be a valid git revision')
+            eprint(f'Found: "{version}"')
+            sys.exit(1)
 
-lockedDependenciesDict: dict[str, dict[str, str]] = {}
+lockedDepsDict: dict[str, dict[str, str]] = {}
 
-for pname in versionDict:
-    version = versionDict[pname]
-    eprint(f"Fetching {pname}@{version}")
-    url = f"https://code.dlang.org/packages/{pname}/{version}.zip"
-    command = ["nix-prefetch-url", "--type", "sha256", url]
-    sha256 = subprocess.run(command, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout.rstrip()
-    lockedDependenciesDict[pname] = {"version": version, "sha256": sha256}
+for pname in depsDict:
+    depDict = depsDict[pname]
+    version = depDict["version"]
+    if "repository" in depDict:
+        repository = depDict["repository"]
+        strippedRepo = repository[4:]
+        eprint(f"Fetching {pname}@{version} ({strippedRepo})")
+        command = ["nix-prefetch-git", strippedRepo, version]
+        rawRes = subprocess.run(command, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout
+        sha256 = json.loads(rawRes)["sha256"]
+        lockedDepsDict[pname] = {"version": version, "repository": repository, "sha256": sha256}
+    else:
+        eprint(f"Fetching {pname}@{version}")
+        url = f"https://code.dlang.org/packages/{pname}/{version}.zip"
+        command = ["nix-prefetch-url", "--type", "sha256", url]
+        sha256 = subprocess.run(command, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout.rstrip()
+        lockedDepsDict[pname] = {"version": version, "sha256": sha256}
 
-print(json.dumps({"dependencies": lockedDependenciesDict}, indent=2))
+print(json.dumps({"dependencies": lockedDepsDict}, indent=2))
diff --git a/pkgs/build-support/kernel/compress-firmware-xz.nix b/pkgs/build-support/kernel/compress-firmware-xz.nix
deleted file mode 100644
index cb9ce7a713389..0000000000000
--- a/pkgs/build-support/kernel/compress-firmware-xz.nix
+++ /dev/null
@@ -1,29 +0,0 @@
-{ runCommand, lib }:
-
-firmware:
-
-let
-  args = {
-    allowedRequisites = [];
-  } // lib.optionalAttrs (firmware ? meta) { inherit (firmware) meta; };
-in
-
-runCommand "${firmware.name}-xz" args ''
-  mkdir -p $out/lib
-  (cd ${firmware} && find lib/firmware -type d -print0) |
-      (cd $out && xargs -0 mkdir -v --)
-  (cd ${firmware} && find lib/firmware -type f -print0) |
-      (cd $out && xargs -0rtP "$NIX_BUILD_CORES" -n1 \
-          sh -c 'xz -9c -T1 -C crc32 --lzma2=dict=2MiB "${firmware}/$1" > "$1.xz"' --)
-  (cd ${firmware} && find lib/firmware -type l) | while read link; do
-      target="$(readlink "${firmware}/$link")"
-      if [ -f "${firmware}/$link" ]; then
-        ln -vs -- "''${target/^${firmware}/$out}.xz" "$out/$link.xz"
-      else
-        ln -vs -- "''${target/^${firmware}/$out}" "$out/$link"
-      fi
-  done
-
-  echo "Checking for broken symlinks:"
-  find -L $out -type l -print -execdir false -- '{}' '+'
-''
diff --git a/pkgs/build-support/kernel/compress-firmware.nix b/pkgs/build-support/kernel/compress-firmware.nix
new file mode 100644
index 0000000000000..0949036d5127f
--- /dev/null
+++ b/pkgs/build-support/kernel/compress-firmware.nix
@@ -0,0 +1,43 @@
+{ runCommand, lib, type ? "zstd", zstd }:
+
+firmware:
+
+let
+  compressor = {
+    xz = {
+      ext = "xz";
+      nativeBuildInputs = [ ];
+      cmd = file: target: ''xz -9c -T1 -C crc32 --lzma2=dict=2MiB "${file}" > "${target}"'';
+    };
+    zstd = {
+      ext = "zst";
+      nativeBuildInputs = [ zstd ];
+      cmd = file: target: ''zstd -T1 -19 --long --check -f "${file}" -o "${target}"'';
+    };
+  }.${type} or (throw "Unsupported compressor type for firmware.");
+
+  args = {
+    allowedRequisites = [];
+    inherit (compressor) nativeBuildInputs;
+  } // lib.optionalAttrs (firmware ? meta) { inherit (firmware) meta; };
+in
+
+runCommand "${firmware.name}-${type}" args ''
+  mkdir -p $out/lib
+  (cd ${firmware} && find lib/firmware -type d -print0) |
+      (cd $out && xargs -0 mkdir -v --)
+  (cd ${firmware} && find lib/firmware -type f -print0) |
+      (cd $out && xargs -0rtP "$NIX_BUILD_CORES" -n1 \
+          sh -c '${compressor.cmd "${firmware}/$1" "$1.${compressor.ext}"}' --)
+  (cd ${firmware} && find lib/firmware -type l) | while read link; do
+      target="$(readlink "${firmware}/$link")"
+      if [ -f "${firmware}/$link" ]; then
+        ln -vs -- "''${target/^${firmware}/$out}.${compressor.ext}" "$out/$link.${compressor.ext}"
+      else
+        ln -vs -- "''${target/^${firmware}/$out}" "$out/$link"
+      fi
+  done
+
+  echo "Checking for broken symlinks:"
+  find -L $out -type l -print -execdir false -- '{}' '+'
+''
diff --git a/pkgs/build-support/kernel/modules-closure.sh b/pkgs/build-support/kernel/modules-closure.sh
index 5f61bac751af2..06eb5b0d0de12 100644
--- a/pkgs/build-support/kernel/modules-closure.sh
+++ b/pkgs/build-support/kernel/modules-closure.sh
@@ -80,7 +80,7 @@ for module in $(< ~-/closure); do
     # of its output.
     modinfo -b $kernel --set-version "$version" -F firmware $module | grep -v '^name:' | while read -r i; do
         echo "firmware for $module: $i"
-        for name in "$i" "$i.xz" ""; do
+        for name in "$i" "$i.xz" "$i.zst" ""; do
             [ -z "$name" ] && echo "WARNING: missing firmware $i for module $module"
             if cp -v --parents --no-preserve=mode lib/firmware/$name "$out" 2>/dev/null; then
                 break
diff --git a/pkgs/build-support/vm/default.nix b/pkgs/build-support/vm/default.nix
index 56375851783fb..871f81bb5d69f 100644
--- a/pkgs/build-support/vm/default.nix
+++ b/pkgs/build-support/vm/default.nix
@@ -40,10 +40,14 @@ rec {
         ${pkgs.stdenv.cc.libc}/lib/libc.so.* \
         ${pkgs.stdenv.cc.libc}/lib/libm.so.* \
         ${pkgs.stdenv.cc.libc}/lib/libresolv.so.* \
+        ${pkgs.stdenv.cc.libc}/lib/libpthread.so.* \
+        ${pkgs.zstd.out}/lib/libzstd.so.* \
+        ${pkgs.xz.out}/lib/liblzma.so.* \
         $out/lib
 
       # Copy BusyBox.
       cp -pd ${pkgs.busybox}/bin/* $out/bin
+      cp -pd ${pkgs.kmod}/bin/* $out/bin
 
       # Run patchelf to make the programs refer to the copied libraries.
       for i in $out/bin/* $out/lib/*; do if ! test -L $i; then nuke-refs $i; fi; done
@@ -54,6 +58,11 @@ rec {
               patchelf --set-interpreter $out/lib/ld-*.so.? --set-rpath $out/lib $i || true
           fi
       done
+
+      find $out/lib -type f \! -name 'ld*.so.?' | while read i; do
+        echo "patching $i..."
+        patchelf --set-rpath $out/lib $i
+      done
     ''; # */