nuget-to-nix: find sources deterministically

The source used to download a particular package still isn't
deterministic in nuget. Even worse, the hash of the package can vary
between sources. This makes nuget use the first enabled source
containing the package.

The order of the dependencies may be slightly different because it now
uses glob order of the lower-case package names and versions, instead of
sorting the output.

If the package actually downloaded was the first source that contains
the package, then it will be hashed from disk to avoid downloading it
again.

3 files changed, 73 insertions, 31 deletions

diff --git a/pkgs/build-support/dotnet/build-dotnet-module/default.nix b/pkgs/build-support/dotnet/build-dotnet-module/default.nix
index a8754218deceb..5a465b48c1d7e 100644
--- a/pkgs/build-support/dotnet/build-dotnet-module/default.nix
+++ b/pkgs/build-support/dotnet/build-dotnet-module/default.nix
@@ -172,7 +172,7 @@ stdenvNoCC.mkDerivation (args // {
       writeShellScript "fetch-${pname}-deps" ''
         set -euo pipefail
 
-        export PATH="${lib.makeBinPath [ coreutils dotnet-sdk nuget-to-nix ]}"
+        export PATH="${lib.makeBinPath [ coreutils dotnet-sdk (nuget-to-nix.override { inherit dotnet-sdk; }) ]}"
 
         for arg in "$@"; do
             case "$arg" in
@@ -218,6 +218,8 @@ stdenvNoCC.mkDerivation (args // {
                 -p:Deterministic=true \
                 --packages "$tmp/nuget_pkgs" \
                 --runtime "$rid" \
+                --no-cache \
+                --force \
                 ${lib.optionalString (!enableParallelBuilding) "--disable-parallel"} \
                 ${lib.optionalString (flags != []) (toString flags)}
         }
diff --git a/pkgs/build-support/dotnet/nuget-to-nix/default.nix b/pkgs/build-support/dotnet/nuget-to-nix/default.nix
index e3a3e45ac20cf..3fdda4ac68d33 100644
--- a/pkgs/build-support/dotnet/nuget-to-nix/default.nix
+++ b/pkgs/build-support/dotnet/nuget-to-nix/default.nix
@@ -4,11 +4,12 @@
 , substituteAll
 , nix
 , coreutils
-, findutils
-, gnused
 , jq
+, yq
 , curl
 , gnugrep
+, gawk
+, dotnet-sdk
 }:
 
 runCommandLocal "nuget-to-nix" {
@@ -19,11 +20,12 @@ runCommandLocal "nuget-to-nix" {
     binPath = lib.makeBinPath [
       nix
       coreutils
-      findutils
-      gnused
       jq
+      yq
       curl
       gnugrep
+      gawk
+      dotnet-sdk
     ];
   };
 
diff --git a/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh b/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh
index d8c928f5179e1..760350ab05178 100755
--- a/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh
+++ b/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh
@@ -3,6 +3,8 @@
 set -euo pipefail
 
 export PATH="@binPath@"
+# used for glob ordering of package names
+export LC_ALL=C
 
 if [ $# -eq 0 ]; then
   >&2 echo "Usage: $0 <packages directory> [path to excluded package source] > deps.nix"
@@ -10,37 +12,73 @@ if [ $# -eq 0 ]; then
 fi
 
 pkgs=$1
-tmpfile=$(mktemp /tmp/nuget-to-nix.XXXXXX)
-trap "rm -f ${tmpfile}" EXIT
+tmp=$(realpath "$(mktemp -td nuget-to-nix.XXXXXX)")
+trap 'rm -r "$tmp"' EXIT
 excluded_source=$(realpath "${2:-$tmp/empty}")
 
-declare -A nuget_sources_cache
+export DOTNET_NOLOGO=1
+export DOTNET_CLI_TELEMETRY_OPTOUT=1
 
-echo "{ fetchNuGet }: ["
+mapfile -t sources < <(dotnet nuget list source --format short | awk '/^E / { print $2 }')
+
+declare -A base_addresses
 
-while read pkg_spec; do
-  { read pkg_name; read pkg_version; } < <(
-    # Build version part should be ignored: `3.0.0-beta2.20059.3+77df2220` -> `3.0.0-beta2.20059.3`
-    sed -nE 's/.*<id>([^<]*).*/\1/p; s/.*<version>([^<+]*).*/\1/p' "$pkg_spec")
+for index in "${sources[@]}"; do
+  base_addresses[$index]=$(
+    curl --compressed --netrc -fsL "$index" | \
+      jq -r '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"')
+done
 
-  if [[ -e "$excluded_source/${pkg_name}.$pkg_version".nupkg ]]; then
-    continue
-  fi
+echo "{ fetchNuGet }: ["
 
-  pkg_sha256="$(nix-hash --type sha256 --flat --base32 "$(dirname "$pkg_spec")"/*.nupkg)"
+cd "$pkgs"
+for package in *; do
+  cd "$package"
+  for version in *; do
+    id=$(xq -r .package.metadata.id "$version/$package".nuspec)
 
-  pkg_src="$(jq --raw-output '.source' "$(dirname "$pkg_spec")/.nupkg.metadata")"
-  if [[ -d $pkg_src ]]; then
+    if [[ -e "$excluded_source/$id.$version".nupkg ]]; then
       continue
-  elif [[ $pkg_src != https://api.nuget.org/* ]]; then
-    pkg_source_url="${nuget_sources_cache[$pkg_src]:=$(curl -n --fail "$pkg_src" | jq --raw-output '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"')}"
-    pkg_url="$pkg_source_url${pkg_name,,}/${pkg_version,,}/${pkg_name,,}.${pkg_version,,}.nupkg"
-    echo "  (fetchNuGet { pname = \"$pkg_name\"; version = \"$pkg_version\"; sha256 = \"$pkg_sha256\"; url = \"$pkg_url\"; })" >> ${tmpfile}
-  else
-    echo "  (fetchNuGet { pname = \"$pkg_name\"; version = \"$pkg_version\"; sha256 = \"$pkg_sha256\"; })" >> ${tmpfile}
-  fi
-done < <(find $1 -name '*.nuspec')
-
-LC_ALL=C sort --ignore-case ${tmpfile}
-
-echo "]"
+    fi
+
+    used_source="$(jq -r '.source' "$version"/.nupkg.metadata)"
+    for source in "${sources[@]}"; do
+      url="${base_addresses[$source]}$package/$version/$package.$version.nupkg"
+      if [[ "$source" == "$used_source" ]]; then
+        sha256="$(nix-hash --type sha256 --flat --base32 "$version/$package.$version".nupkg)"
+        found=true
+        break
+      else
+        if sha256=$(nix-prefetch-url "$url" 2>"$tmp"/error); then
+          # If multiple remote sources are enabled, nuget will try them all
+          # concurrently and use the one that responds first. We always use the
+          # first source that has the package.
+          echo "$package $version is available on $url, but was downloaded from ${base_addresses[$used_source]}$package/$version/$package.$version.nupkg" 1>&2
+          found=true
+          break
+        else
+          if ! grep -q 'HTTP error 404' "$tmp/error"; then
+            cat "$tmp/error" 1>&2
+            exit 1
+          fi
+        fi
+      fi
+    done
+
+    if ! ${found-false}; then
+      echo "couldn't find $package $version" >&2
+      exit 1
+    fi
+
+    if [[ "$source" != https://api.nuget.org/v3/index.json ]]; then
+      echo "  (fetchNuGet { pname = \"$id\"; version = \"$version\"; sha256 = \"$sha256\"; url = \"$url\"; })"
+    else
+      echo "  (fetchNuGet { pname = \"$id\"; version = \"$version\"; sha256 = \"$sha256\"; })"
+    fi
+  done
+  cd ..
+done
+
+cat << EOL
+]
+EOL