diff options
Diffstat (limited to 'pkgs/applications/networking/cluster/k3s')
4 files changed, 223 insertions, 204 deletions
diff --git a/pkgs/applications/networking/cluster/k3s/default.nix b/pkgs/applications/networking/cluster/k3s/default.nix index f3b5e4cd70c9a..5b5dfece1f1bd 100644 --- a/pkgs/applications/networking/cluster/k3s/default.nix +++ b/pkgs/applications/networking/cluster/k3s/default.nix @@ -5,9 +5,11 @@ , iptables , iproute2 , bridge-utils +, btrfs-progs , conntrack-tools -, buildGoPackage +, buildGoModule , runc +, rsync , kmod , libseccomp , pkg-config @@ -18,6 +20,8 @@ , fetchzip , fetchgit , zstd +, yq-go +, nixosTests }: with lib; @@ -42,27 +46,58 @@ with lib; # Those pieces of software we entirely ignore upstream's handling of, and just # make sure they're in the path if desired. let - k3sVersion = "1.22.3+k3s1"; # k3s git tag - k3sCommit = "61a2aab25eeb97c26fa3f2b177e4355a7654c991"; # k3s git commit at the above version - k3sRepoSha256 = "0lz5hr3c86gxm9w5jy3g26n6a26m8k0y559hv6220rsi709j7ma9"; + k3sVersion = "1.23.3+k3s1"; # k3s git tag + k3sCommit = "6f4217a3405d16a1a51bbb40872d7dcb87207bb9"; # k3s git commit at the above version + k3sRepoSha256 = "sha256-0dRusG1vL+1KbmViIUNCZK1b+FEgV6otcVUyFonHmm4="; - traefikChartVersion = "10.3.0"; # taken from ./manifests/traefik.yaml at spec.version - traefikChartSha256 = "0y6wr64xp7bgx24kqil0x6myr3pnfrg8rw0d1h5zd2n5a8nfd73f"; + # taken from ./manifests/traefik.yaml, extracted from '.spec.chart' https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/download#L9 + # The 'patch' and 'minor' versions are currently hardcoded as single digits only, so ignore the trailing two digits. Weird, I know. + traefikChartVersion = "10.9.1"; + traefikChartSha256 = "sha256-XM1DLofU1zEEFeB5bNQ7cgv102gXsToPP7SFh87QuGQ="; - k3sRootVersion = "0.9.1"; # taken from ./scripts/download at ROOT_VERSION + # taken from ./scripts/version.sh VERSION_ROOT https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L47 + k3sRootVersion = "0.9.1"; k3sRootSha256 = "0r2cj4l50cxkrvszpzxfk36lvbjf9vcmp6d5lvxg8qsah8lki3x8"; - k3sCNIVersion = "0.9.1-k3s1"; # taken from ./scripts/version.sh at VERSION_CNIPLUGINS + # taken from ./scripts/version.sh VERSION_CNIPLUGINS https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L45 + k3sCNIVersion = "0.9.1-k3s1"; k3sCNISha256 = "1327vmfph7b8i14q05c2xdfzk60caflg1zhycx0mrf3d59f4zsz5"; + # taken from go.mod, the 'github.com/containerd/containerd' line + # run `grep github.com/containerd/containerd go.mod | head -n1 | awk '{print $4}'` + containerdVersion = "v1.5.9-k3s1"; + containerdSha256 = "sha256-7xlhBA6KuwFlw+jyThygv4Ow9F3xjjIUtS6x8YHwjic="; + + # run `grep github.com/kubernetes-sigs/cri-tools go.mod | head -n1 | awk '{print $4}'` in the k3s repo at the tag + criCtlVersion = "v1.22.0-k3s1"; + baseMeta = { description = "A lightweight Kubernetes distribution"; license = licenses.asl20; homepage = "https://k3s.io"; - maintainers = with maintainers; [ euank ]; + maintainers = with maintainers; [ euank mic92 ]; platforms = platforms.linux; }; + # https://github.com/k3s-io/k3s/blob/5fb370e53e0014dc96183b8ecb2c25a61e891e76/scripts/build#L19-L40 + versionldflags = [ + "-X github.com/rancher/k3s/pkg/version.Version=v${k3sVersion}" + "-X github.com/rancher/k3s/pkg/version.GitCommit=${lib.substring 0 8 k3sCommit}" + "-X k8s.io/client-go/pkg/version.gitVersion=v${k3sVersion}" + "-X k8s.io/client-go/pkg/version.gitCommit=${k3sCommit}" + "-X k8s.io/client-go/pkg/version.gitTreeState=clean" + "-X k8s.io/client-go/pkg/version.buildDate=1970-01-01T01:01:01Z" + "-X k8s.io/component-base/version.gitVersion=v${k3sVersion}" + "-X k8s.io/component-base/version.gitCommit=${k3sCommit}" + "-X k8s.io/component-base/version.gitTreeState=clean" + "-X k8s.io/component-base/version.buildDate=1970-01-01T01:01:01Z" + "-X github.com/kubernetes-sigs/cri-tools/pkg/version.Version=${criCtlVersion}" + "-X github.com/containerd/containerd/version.Version=${containerdVersion}" + "-X github.com/containerd/containerd/version.Package=github.com/k3s-io/containerd" + "-X github.com/containerd/containerd/version.Version=${containerdVersion}" + "-X github.com/containerd/containerd/version.Package=github.com/k3s-io/containerd" + ]; + # bundled into the k3s binary traefikChart = fetchurl { url = "https://helm.traefik.io/traefik/traefik-${traefikChartVersion}.tgz"; @@ -83,11 +118,11 @@ let sha256 = k3sRootSha256; stripRoot = false; }; - k3sPlugins = buildGoPackage rec { - name = "k3s-cni-plugins"; + k3sCNIPlugins = buildGoModule rec { + pname = "k3s-cni-plugins"; version = k3sCNIVersion; + vendorSha256 = null; - goPackagePath = "github.com/containernetworking/plugins"; subPackages = [ "." ]; src = fetchFromGitHub { @@ -97,6 +132,10 @@ let sha256 = k3sCNISha256; }; + postInstall = '' + mv $out/bin/plugins $out/bin/cni + ''; + meta = baseMeta // { description = "CNI plugins, as patched by rancher for k3s"; }; @@ -123,50 +162,38 @@ let # Then, we bundle those binaries into our thick k3s binary and use that as # the final single output. # This approach was chosen because it ensures the bundled binaries all are - # correctly built to run with nix (we can lean on the existing buildGoPackage + # correctly built to run with nix (we can lean on the existing buildGoModule # stuff), and we can again lean on that tooling for the final k3s binary too. # Other alternatives would be to manually run the # strip/patchelf/remove-references step ourselves in the installPhase of the # derivation when we've built all the binaries, but haven't bundled them in # with generated bindata yet. - k3sBuildStage1 = buildGoPackage rec { - name = "k3s-build-1"; + k3sServer = buildGoModule rec { + pname = "k3s-server"; version = k3sVersion; - goPackagePath = "github.com/rancher/k3s"; - src = k3sRepo; - - # Patch build scripts so that we can use them. - # This makes things more dynamically linked (because nix can deal with - # dynamically linked dependencies just fine), removes the upload at the - # end, and skips building runc + cni, since we have our own derivations for - # those. - patches = [ ./patches/0002-Add-nixpkgs-patches.patch ]; + vendorSha256 = "sha256-9+2k/ipAOhc8JJU+L2dwaM01Dkw+0xyrF5kt6mL19G0="; nativeBuildInputs = [ pkg-config ]; buildInputs = [ libseccomp ]; - # Versioning info for build script - DRONE_TAG = "v${version}"; - DRONE_COMMIT = k3sCommit; - - buildPhase = '' - pushd go/src/${goPackagePath} - - patchShebangs ./scripts/build ./scripts/version.sh - mkdir -p bin - ./scripts/build - - popd - ''; - - installPhase = '' - pushd go/src/${goPackagePath} - - mkdir -p "$out/bin" - install -m 0755 -t "$out/bin" ./bin/* - + subPackages = [ "cmd/server" ]; + ldflags = versionldflags; + + # create the multicall symlinks for k3s + postInstall = '' + mv $out/bin/server $out/bin/k3s + pushd $out + # taken verbatim from https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/build#L105-L113 + ln -s k3s ./bin/k3s-agent + ln -s k3s ./bin/k3s-server + ln -s k3s ./bin/k3s-etcd-snapshot + ln -s k3s ./bin/k3s-secrets-encrypt + ln -s k3s ./bin/k3s-certificate + ln -s k3s ./bin/kubectl + ln -s k3s ./bin/crictl + ln -s k3s ./bin/ctr popd ''; @@ -174,76 +201,33 @@ let description = "The various binaries that get packaged into the final k3s binary"; }; }; - k3sBin = buildGoPackage rec { - name = "k3s-bin"; + k3sContainerd = buildGoModule { + pname = "k3s-containerd"; version = k3sVersion; - - goPackagePath = "github.com/rancher/k3s"; - - src = k3sRepo; - - # See the above comment in k3sBuildStage1 - patches = [ ./patches/0002-Add-nixpkgs-patches.patch ]; - - nativeBuildInputs = [ pkg-config zstd ]; - # These dependencies are embedded as compressed files in k3s at runtime. - # Propagate them to avoid broken runtime references to libraries. - propagatedBuildInputs = [ k3sPlugins k3sBuildStage1 runc ]; - - # k3s appends a suffix to the final distribution binary for some arches - archSuffix = - if stdenv.hostPlatform.system == "x86_64-linux" then "" - else if stdenv.hostPlatform.system == "aarch64-linux" then "-arm64" - else throw "k3s isn't being built for ${stdenv.hostPlatform.system} yet."; - - DRONE_TAG = "v${version}"; - DRONE_COMMIT = k3sCommit; - - # In order to build the thick k3s binary (which is what - # ./scripts/package-cli does), we need to get all the binaries that script - # expects in place. - buildPhase = '' - pushd go/src/${goPackagePath} - - patchShebangs ./scripts/build ./scripts/version.sh ./scripts/package-cli - - mkdir -p bin - - install -m 0755 -t ./bin ${k3sBuildStage1}/bin/* - install -m 0755 -T "${k3sPlugins}/bin/plugins" ./bin/cni - # Note: use the already-nixpkgs-bundled k3s rather than the one bundled - # in k3s because the k3s one is completely unmodified from upstream - # (unlike containerd, cni, etc) - install -m 0755 -T "${runc}/bin/runc" ./bin/runc - cp -R "${k3sRoot}/etc" ./etc - mkdir -p "build/static/charts" - cp "${traefikChart}" "build/static/charts/traefik-${traefikChartVersion}.tgz" - - ./scripts/package-cli - - popd - ''; - - installPhase = '' - pushd go/src/${goPackagePath} - - mkdir -p "$out/bin" - install -m 0755 -T ./dist/artifacts/k3s${archSuffix} "$out/bin/k3s" - - popd - ''; - - meta = baseMeta // { - description = "The k3s go binary which is used by the final wrapped output below"; + src = fetchFromGitHub { + owner = "k3s-io"; + repo = "containerd"; + rev = containerdVersion; + sha256 = containerdSha256; }; + vendorSha256 = null; + buildInputs = [ btrfs-progs ]; + subPackages = [ "cmd/containerd" "cmd/containerd-shim-runc-v2" ]; + ldflags = versionldflags; }; in -stdenv.mkDerivation rec { +buildGoModule rec { pname = "k3s"; version = k3sVersion; - # `src` here is a workaround for the updateScript bot. It couldn't be empty. - src = builtins.filterSource (path: type: false) ./.; + src = k3sRepo; + proxyVendor = true; + vendorSha256 = "sha256-8Yp9csyRNSYi9wo8E8mF8cu92wG1t3l18wJ8Y4L7HEA="; + + patches = [ + ./patches/0001-scrips-download-strip-downloading-just-package-CRD.patch + ./patches/0002-Don-t-build-a-static-binary-in-package-cli.patch + ]; # Important utilities used by the kubelet, see # https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-237202494 @@ -259,35 +243,73 @@ stdenv.mkDerivation rec { conntrack-tools ]; - buildInputs = [ - k3sBin - ] ++ k3sRuntimeDeps; + buildInputs = k3sRuntimeDeps; - nativeBuildInputs = [ makeWrapper ]; + nativeBuildInputs = [ + makeWrapper + rsync + yq-go + zstd + ]; - unpackPhase = "true"; + # embedded in the final k3s cli + propagatedBuildInputs = [ + k3sCNIPlugins + k3sContainerd + k3sServer + runc + ]; + + # We override most of buildPhase due to peculiarities in k3s's build. + # Specifically, it has a 'go generate' which runs part of the package. See + # this comment: + # https://github.com/NixOS/nixpkgs/pull/158089#discussion_r799965694 + # So, why do we use buildGoModule at all? For the `vendorSha256` / `go mod download` stuff primarily. + buildPhase = '' + patchShebangs ./scripts/package-cli ./scripts/download ./scripts/build-upload + + # copy needed 'go generate' inputs into place + mkdir -p ./bin/aux + rsync -a --no-perms ${k3sServer}/bin/ ./bin/ + ln -vsf ${runc}/bin/runc ./bin/runc + ln -vsf ${k3sCNIPlugins}/bin/cni ./bin/cni + ln -vsf ${k3sContainerd}/bin/* ./bin/ + rsync -a --no-perms --chmod u=rwX ${k3sRoot}/etc/ ./etc/ + mkdir -p ./build/static/charts + # Note, upstream's chart has a 00 suffix. This seems to not matter though, so we're ignoring that naming detail. + export TRAEFIK_CHART_FILE=${traefikChart} + # place the traefik chart using their code since it's complicated + # We trim the actual download, see patches + ./scripts/download + + export ARCH=$GOARCH + export DRONE_TAG="v${k3sVersion}" + export DRONE_COMMIT="${k3sCommit}" + # use ./scripts/package-cli to run 'go generate' + 'go build' + + ./scripts/package-cli + mkdir -p $out/bin + ''; + + # Otherwise it depends on 'getGoDirs', which is normally set in buildPhase + doCheck = false; - # And, one final derivation (you thought the last one was it, right?) - # We got the binary we wanted above, but it doesn't have all the runtime - # dependencies k8s wants, including mount utilities for kubelet, networking - # tools for cni/kubelet stuff, etc - # Use a wrapper script to reference all the binaries that k3s tries to - # execute, but that we didn't bundle with it. installPhase = '' - runHook preInstall - mkdir -p "$out/bin" - makeWrapper ${k3sBin}/bin/k3s "$out/bin/k3s" \ + # wildcard to match the arm64 build too + install -m 0755 dist/artifacts/k3s* -D $out/bin/k3s + wrapProgram $out/bin/k3s \ --prefix PATH : ${lib.makeBinPath k3sRuntimeDeps} \ --prefix PATH : "$out/bin" - runHook postInstall ''; doInstallCheck = true; installCheckPhase = '' - $out/bin/k3s --version | grep v${k3sVersion} > /dev/null + $out/bin/k3s --version | grep -F "v${k3sVersion}" >/dev/null ''; passthru.updateScript = ./update.sh; + passthru.tests = { inherit (nixosTests) k3s-single-node k3s-single-node-docker; }; + meta = baseMeta; } diff --git a/pkgs/applications/networking/cluster/k3s/patches/0001-scrips-download-strip-downloading-just-package-CRD.patch b/pkgs/applications/networking/cluster/k3s/patches/0001-scrips-download-strip-downloading-just-package-CRD.patch new file mode 100644 index 0000000000000..115fd6824772b --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/patches/0001-scrips-download-strip-downloading-just-package-CRD.patch @@ -0,0 +1,41 @@ +From 6f53bd36a40da4c71486e3b79f6e32d53d6eea5d Mon Sep 17 00:00:00 2001 +From: Euan Kemp <euank@euank.com> +Date: Thu, 3 Feb 2022 23:50:40 -0800 +Subject: [PATCH 2/2] scrips/download: strip downloading, just package CRD + +The CRD packaging is a complicated set of commands, so let's reuse it. +--- + scripts/download | 10 ++-------- + 1 file changed, 2 insertions(+), 8 deletions(-) + +diff --git a/scripts/download b/scripts/download +index 5effc0562a..82361803ee 100755 +--- a/scripts/download ++++ b/scripts/download +@@ -24,12 +24,6 @@ rm -rf ${CONTAINERD_DIR} + mkdir -p ${CHARTS_DIR} + mkdir -p ${DATA_DIR} + +-curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VERSION_ROOT}/k3s-root-${ARCH}.tar | tar xf - --exclude=bin/socat +- +-git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR} +- +-git clone --single-branch --branch=${VERSION_CONTAINERD} --depth=1 https://github.com/k3s-io/containerd ${CONTAINERD_DIR} +- + setup_tmp() { + TMP_DIR=$(mktemp -d --tmpdir=${CHARTS_DIR}) + cleanup() { +@@ -44,8 +38,8 @@ setup_tmp() { + + download_and_package_traefik () { + echo "Downloading Traefik Helm chart from ${TRAEFIK_URL}" +- curl -sfL ${TRAEFIK_URL} -o ${TMP_DIR}/${TRAEFIK_FILE} +- code=$? ++ # nixpkgs: copy in our known traefik chart instead ++ cp $TRAEFIK_CHART_FILE ${TMP_DIR}/${TRAEFIK_FILE} + + if [ $code -ne 0 ]; then + echo "Error: Failed to download Traefik Helm chart!" +-- +2.34.1 + diff --git a/pkgs/applications/networking/cluster/k3s/patches/0002-Add-nixpkgs-patches.patch b/pkgs/applications/networking/cluster/k3s/patches/0002-Add-nixpkgs-patches.patch deleted file mode 100644 index dd26810a45633..0000000000000 --- a/pkgs/applications/networking/cluster/k3s/patches/0002-Add-nixpkgs-patches.patch +++ /dev/null @@ -1,81 +0,0 @@ --Subject: [PATCH 2/2] Add nixpkgs patches --Original patch by: Euan Kemp <euank@euank.com> --Adapted by: superherointj -- --This patch allows us to re-use upstream build scripts when building for nix. ----- -- 2 files changed: -- scripts/build -- scripts/package-cli -- -diff --git a/scripts/build b/scripts/build -index 2f3d1dc496..4f4e5aa897 100755 ---- a/scripts/build -+++ b/scripts/build -@@ -12,7 +12,8 @@ PKG_CONTAINERD="github.com/containerd/containerd" - PKG_K3S_CONTAINERD="github.com/k3s-io/containerd" - PKG_CRICTL="github.com/kubernetes-sigs/cri-tools" - --buildDate=$(date -u '+%Y-%m-%dT%H:%M:%SZ') -+# nixpkgs: deterministic build date -+buildDate="$(date -d "$(git log -1 --format=%ai)" -u "+%Y-%m-%dT%H:%M:%SZ")" - - VENDOR_PREFIX="${PKG}/vendor/" - VERSIONFLAGS=" -@@ -89,17 +90,7 @@ cleanup() { - } - - INSTALLBIN=$(pwd)/bin --if [ ! -x ${INSTALLBIN}/cni ]; then --( -- echo Building cni -- TMPDIR=$(mktemp -d) -- trap cleanup EXIT -- WORKDIR=$TMPDIR/src/github.com/containernetworking/plugins -- git clone -b $VERSION_CNIPLUGINS https://github.com/rancher/plugins.git $WORKDIR -- cd $WORKDIR -- GOPATH=$TMPDIR CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o $INSTALLBIN/cni --) --fi -+# nixpkgs: skip building cni, we build it separately - # echo Building agent - # CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s-agent ./cmd/agent/main.go - echo Building server -@@ -116,10 +107,7 @@ ln -s containerd ./bin/ctr - #CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC_SQLITE" -o bin/ctr ./cmd/ctr/main.go - # echo Building containerd - # CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd ./cmd/containerd/ --echo Building runc --rm -f ./build/src/github.com/opencontainers/runc/runc --make GOPATH=$(pwd)/build EXTRA_LDFLAGS="-w -s" BUILDTAGS="$RUNC_TAGS" -C ./build/src/github.com/opencontainers/runc $RUNC_STATIC --cp -f ./build/src/github.com/opencontainers/runc/runc ./bin/runc -+# nixpkgs: we build runc separately - - echo Building containerd-shim - rm -f ./vendor/github.com/containerd/containerd/bin/containerd-shim -diff --git a/scripts/package-cli b/scripts/package-cli -index ab4a6dac63..044b5587d0 100755 ---- a/scripts/package-cli -+++ b/scripts/package-cli -@@ -50,15 +50,17 @@ fi - - CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX} - --"${GO}" generate -+CGO_ENABLED=0 env -u GOARCH "${GO}" generate - LDFLAGS=" - -X github.com/rancher/k3s/pkg/version.Version=$VERSION - -X github.com/rancher/k3s/pkg/version.GitCommit=${COMMIT:0:8} - -w -s - " --STATIC="-extldflags '-static'" --CGO_ENABLED=0 "${GO}" build -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go -+# STATIC="-extldflags '-static'" -+# nixpkgs: we can depend on dynamic linking because we have a good package manager -+"${GO}" build -ldflags "$LDFLAGS" -o ${CMD_NAME} ./cmd/k3s/main.go - - stat ${CMD_NAME} - --./scripts/build-upload ${CMD_NAME} ${COMMIT} -+# nixpkgs: skip uploading -+# ./scripts/build-upload ${CMD_NAME} ${COMMIT} diff --git a/pkgs/applications/networking/cluster/k3s/patches/0002-Don-t-build-a-static-binary-in-package-cli.patch b/pkgs/applications/networking/cluster/k3s/patches/0002-Don-t-build-a-static-binary-in-package-cli.patch new file mode 100644 index 0000000000000..886b6ec7409be --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/patches/0002-Don-t-build-a-static-binary-in-package-cli.patch @@ -0,0 +1,37 @@ +From 49c000c7c5dd7a502a2be4c638d2c32b65673c00 Mon Sep 17 00:00:00 2001 +From: Euan Kemp <euank@euank.com> +Date: Sun, 6 Feb 2022 23:13:00 -0800 +Subject: [PATCH] Don't build a static binary in package-cli + +since nixpkgs prefers dynamically linked binaries. + +Also remove "trimpath" for the 'go generate' step because the codegen +they use doesn't work with trimpath set. +--- + scripts/package-cli | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/scripts/package-cli b/scripts/package-cli +index 28927327b7..95dbb469f1 100755 +--- a/scripts/package-cli ++++ b/scripts/package-cli +@@ -48,14 +48,13 @@ fi + + CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX} + +-"${GO}" generate ++GOFLAGS="" "${GO}" generate + LDFLAGS=" + -X github.com/rancher/k3s/pkg/version.Version=$VERSION + -X github.com/rancher/k3s/pkg/version.GitCommit=${COMMIT:0:8} + -w -s + " +-STATIC="-extldflags '-static'" +-CGO_ENABLED=0 "${GO}" build -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go ++CGO_ENABLED=0 "${GO}" build -ldflags "$LDFLAGS" -o ${CMD_NAME} ./cmd/k3s/main.go + + stat ${CMD_NAME} + +-- +2.34.1 + |