Merge pull request #274591 from tweag/by-name-reproducible

`pkgs/by-name`: Enable gradual migration checks and add `run-local.sh`

3 files changed, 140 insertions, 0 deletions

diff --git a/pkgs/test/nixpkgs-check-by-name/scripts/README.md b/pkgs/test/nixpkgs-check-by-name/scripts/README.md
new file mode 100644
index 0000000000000..41b3012b7d953
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/scripts/README.md
@@ -0,0 +1,26 @@
+# CI-related Scripts
+
+This directory contains scripts used and related to the CI running the `pkgs/by-name` checks in Nixpkgs. See also the [CI GitHub Action](../../../../.github/workflows/check-by-name.yml).
+
+## `./run-local.sh BASE_BRANCH [REPOSITORY]`
+
+Runs the `pkgs/by-name` check on the HEAD commit, closely matching what CI does.
+
+Note that this can't do exactly the same as CI,
+because CI needs to rely on GitHub's server-side Git history to compute the mergeability of PRs before the check can be started.
+In turn when running locally, we don't want to have to push commits to test them,
+and we can also rely on the local Git history to do the mergeability check.
+
+Arguments:
+- `BASE_BRANCH`: The base branch to use, e.g. master or release-23.11
+- `REPOSITORY`: The repository to fetch the base branch from, defaults to https://github.com/NixOS/nixpkgs.git
+
+## `./fetch-tool.sh BASE_BRANCH OUTPUT_PATH`
+
+Fetches the Hydra-prebuilt nixpkgs-check-by-name to use from the NixOS channel corresponding to the given base branch.
+
+This script is used both by [`./run-local.sh`](#run-local-sh-base-branch-repository) and CI.
+
+Arguments:
+- `BASE_BRANCH`: The base branch to use, e.g. master or release-23.11
+- `OUTPUT_PATH`: The output symlink path for the tool
diff --git a/pkgs/test/nixpkgs-check-by-name/scripts/fetch-tool.sh b/pkgs/test/nixpkgs-check-by-name/scripts/fetch-tool.sh
new file mode 100755
index 0000000000000..19a48b6fb1fd5
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/scripts/fetch-tool.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Fetches the prebuilt nixpkgs-check-by-name to use from
+# the NixOS channel corresponding to the given base branch
+
+set -o pipefail -o errexit -o nounset
+
+trace() { echo >&2 "$@"; }
+
+if (( $# < 2 )); then
+    trace "Usage: $0 BASE_BRANCH OUTPUT_PATH"
+    trace "BASE_BRANCH: The base branch to use, e.g. master or release-23.11"
+    trace "OUTPUT_PATH: The output symlink path for the tool"
+    exit 1
+fi
+baseBranch=$1
+output=$2
+
+trace -n "Determining the channel to use for PR base branch $baseBranch.. "
+if [[ "$baseBranch" =~ ^(release|staging|staging-next)-([0-9][0-9]\.[0-9][0-9])$ ]]; then
+  # Use the release channel for all PRs to release-XX.YY, staging-XX.YY and staging-next-XX.YY
+  preferredChannel=nixos-${BASH_REMATCH[2]}
+else
+  # Use the nixos-unstable channel for all other PRs
+  preferredChannel=nixos-unstable
+fi
+
+# Check that the channel exists. It doesn't exist for fresh release branches
+if curl -fSs "https://channels.nixos.org/$preferredChannel"; then
+    channel=$preferredChannel
+    trace "$channel"
+else
+    # Fall back to nixos-unstable, makes sense for fresh release branches
+    channel=nixos-unstable
+    trace -e "\e[33mWarning: Preferred channel $preferredChannel could not be fetched, using fallback: $channel\e[0m"
+fi
+
+trace -n "Fetching latest version of channel $channel.. "
+# This is probably the easiest way to get Nix to output the path to a downloaded channel!
+nixpkgs=$(nix-instantiate --find-file nixpkgs -I nixpkgs=channel:"$channel")
+trace "$nixpkgs"
+
+# This file only exists in channels
+trace -e "Git revision of channel $channel is \e[34m$(<"$nixpkgs/.git-revision")\e[0m"
+
+trace -n "Fetching the prebuilt version of nixpkgs-check-by-name.. "
+nix-build -o "$output" "$nixpkgs" -A tests.nixpkgs-check-by-name -j 0 >/dev/null
+realpath "$output" >&2
diff --git a/pkgs/test/nixpkgs-check-by-name/scripts/run-local.sh b/pkgs/test/nixpkgs-check-by-name/scripts/run-local.sh
new file mode 100755
index 0000000000000..72d3e8dc3de39
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/scripts/run-local.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+set -o pipefail -o errexit -o nounset
+
+trace() { echo >&2 "$@"; }
+
+tmp=$(mktemp -d)
+cleanup() {
+    # Don't exit early if anything fails to cleanup
+    set +o errexit
+
+    trace -n "Cleaning up.. "
+
+    [[ -e "$tmp/base" ]] && git worktree remove --force "$tmp/base"
+    [[ -e "$tmp/merged" ]] && git worktree remove --force "$tmp/merged"
+
+    rm -rf "$tmp"
+
+    trace "Done"
+}
+trap cleanup exit
+
+
+repo=https://github.com/NixOS/nixpkgs.git
+
+if (( $# != 0 )); then
+    baseBranch=$1
+    shift
+else
+    trace "Usage: $0 BASE_BRANCH [REPOSITORY]"
+    trace "BASE_BRANCH: The base branch to use, e.g. master or release-23.11"
+    trace "REPOSITORY: The repository to fetch the base branch from, defaults to $repo"
+    exit 1
+fi
+
+if (( $# != 0 )); then
+    repo=$1
+    shift
+fi
+
+if [[ -n "$(git status --porcelain)" ]]; then
+    trace -e "\e[33mWarning: Dirty tree, uncommitted changes won't be taken into account\e[0m"
+fi
+headSha=$(git rev-parse HEAD)
+trace -e "Using HEAD commit \e[34m$headSha\e[0m"
+
+trace -n "Creating Git worktree for the HEAD commit in $tmp/merged.. "
+git worktree add --detach -q "$tmp/merged" HEAD
+trace "Done"
+
+trace -n "Fetching base branch $baseBranch to compare against.. "
+git fetch -q "$repo" refs/heads/"$baseBranch"
+baseSha=$(git rev-parse FETCH_HEAD)
+trace -e "\e[34m$baseSha\e[0m"
+
+trace -n "Creating Git worktree for the base branch in $tmp/base.. "
+git worktree add -q "$tmp/base" "$baseSha"
+trace "Done"
+
+trace -n "Merging base branch into the HEAD commit in $tmp/merged.. "
+git -C "$tmp/merged" merge -q --no-edit "$baseSha"
+trace -e "\e[34m$(git -C "$tmp/merged" rev-parse HEAD)\e[0m"
+
+"$tmp/merged/pkgs/test/nixpkgs-check-by-name/scripts/fetch-tool.sh" "$baseBranch" "$tmp/tool"
+
+trace "Running nixpkgs-check-by-name.."
+"$tmp/tool/bin/nixpkgs-check-by-name" --base "$tmp/base" "$tmp/merged"