diff options
Diffstat (limited to 'pkgs/applications/networking/cluster/k3s')
20 files changed, 647 insertions, 255 deletions
diff --git a/pkgs/applications/networking/cluster/k3s/1_26/chart-versions.nix b/pkgs/applications/networking/cluster/k3s/1_26/chart-versions.nix deleted file mode 100644 index 1acca4d0e101f..0000000000000 --- a/pkgs/applications/networking/cluster/k3s/1_26/chart-versions.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ - traefik-crd = { - url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-25.0.2+up25.0.0.tgz"; - sha256 = "0jygzsn5pxzf7423x5iqfffgx5xvm7c7hfck46y7vpv1fdkiipcq"; - }; - traefik = { - url = "https://k3s.io/k3s-charts/assets/traefik/traefik-25.0.2+up25.0.0.tgz"; - sha256 = "1g9n19lnqdkmbbr3rnbwc854awha0kqqfwyxanyx1lg5ww8ldp89"; - }; -} diff --git a/pkgs/applications/networking/cluster/k3s/1_26/versions.nix b/pkgs/applications/networking/cluster/k3s/1_26/versions.nix deleted file mode 100644 index 6ac3414b1b0e5..0000000000000 --- a/pkgs/applications/networking/cluster/k3s/1_26/versions.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ - k3sVersion = "1.26.15+k3s1"; - k3sCommit = "132972364806998c35d250153e2af245f9ecf18d"; - k3sRepoSha256 = "13iwmjxyf71l2g66kxdivnj21bf9lmr5p4qlp8kmysm23w2badj9"; - k3sVendorHash = "sha256-xoscRchOK4p3d1DAnxbJq7oIvxIn1twePmOBDdfXzw8="; - chartVersions = import ./chart-versions.nix; - k3sRootVersion = "0.12.2"; - k3sRootSha256 = "1gjynvr350qni5mskgm7pcc7alss4gms4jmkiv453vs8mmma9c9k"; - k3sCNIVersion = "1.4.0-k3s2"; - k3sCNISha256 = "17dg6jgjx18nrlyfmkv14dhzxsljz4774zgwz5dchxcf38bvarqa"; - containerdVersion = "1.7.11-k3s2.26"; - containerdSha256 = "0413a81kzb05xkklwyngg8g6a0w4icsi938rim69jmr2sijc89ww"; - criCtlVersion = "1.26.0-rc.0-k3s1"; -} diff --git a/pkgs/applications/networking/cluster/k3s/1_27/chart-versions.nix b/pkgs/applications/networking/cluster/k3s/1_27/chart-versions.nix deleted file mode 100644 index 1acca4d0e101f..0000000000000 --- a/pkgs/applications/networking/cluster/k3s/1_27/chart-versions.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ - traefik-crd = { - url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-25.0.2+up25.0.0.tgz"; - sha256 = "0jygzsn5pxzf7423x5iqfffgx5xvm7c7hfck46y7vpv1fdkiipcq"; - }; - traefik = { - url = "https://k3s.io/k3s-charts/assets/traefik/traefik-25.0.2+up25.0.0.tgz"; - sha256 = "1g9n19lnqdkmbbr3rnbwc854awha0kqqfwyxanyx1lg5ww8ldp89"; - }; -} diff --git a/pkgs/applications/networking/cluster/k3s/1_27/versions.nix b/pkgs/applications/networking/cluster/k3s/1_27/versions.nix deleted file mode 100644 index 928337553966c..0000000000000 --- a/pkgs/applications/networking/cluster/k3s/1_27/versions.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ - k3sVersion = "1.27.12+k3s1"; - k3sCommit = "78ad57567c9eb1fd1831986f5fd7b4024add1767"; - k3sRepoSha256 = "1j6xb3af4ypqq5m6a8x2yc2515zvlgqzfsfindjm9cbmq5iisphq"; - k3sVendorHash = "sha256-65cmpRwD9C+fcbBSv1YpeukO7bfGngsLv/rk6sM59gU="; - chartVersions = import ./chart-versions.nix; - k3sRootVersion = "0.12.2"; - k3sRootSha256 = "1gjynvr350qni5mskgm7pcc7alss4gms4jmkiv453vs8mmma9c9k"; - k3sCNIVersion = "1.4.0-k3s2"; - k3sCNISha256 = "17dg6jgjx18nrlyfmkv14dhzxsljz4774zgwz5dchxcf38bvarqa"; - containerdVersion = "1.7.11-k3s2.27"; - containerdSha256 = "0xjxc5dgh3drk2glvcabd885damjffp9r4cs0cm1zgnrrbhlipra"; - criCtlVersion = "1.26.0-rc.0-k3s1"; -} diff --git a/pkgs/applications/networking/cluster/k3s/1_28/chart-versions.nix b/pkgs/applications/networking/cluster/k3s/1_28/chart-versions.nix index 1acca4d0e101f..aaaa3d4c29700 100644 --- a/pkgs/applications/networking/cluster/k3s/1_28/chart-versions.nix +++ b/pkgs/applications/networking/cluster/k3s/1_28/chart-versions.nix @@ -1,10 +1,10 @@ { - traefik-crd = { - url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-25.0.2+up25.0.0.tgz"; - sha256 = "0jygzsn5pxzf7423x5iqfffgx5xvm7c7hfck46y7vpv1fdkiipcq"; - }; - traefik = { - url = "https://k3s.io/k3s-charts/assets/traefik/traefik-25.0.2+up25.0.0.tgz"; - sha256 = "1g9n19lnqdkmbbr3rnbwc854awha0kqqfwyxanyx1lg5ww8ldp89"; - }; + traefik-crd = { + url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-25.0.3+up25.0.0.tgz"; + sha256 = "1z693i4kd3jyf26ccnb0sxjyxadipl6k13n7jyg5v4y93fv1rpdw"; + }; + traefik = { + url = "https://k3s.io/k3s-charts/assets/traefik/traefik-25.0.3+up25.0.0.tgz"; + sha256 = "1a24qlp7c6iri72ka1i37l1lzn13xibrd26dy295z2wzr55gg7if"; + }; } diff --git a/pkgs/applications/networking/cluster/k3s/1_28/versions.nix b/pkgs/applications/networking/cluster/k3s/1_28/versions.nix index b8e68bb877404..1778aa32efa90 100644 --- a/pkgs/applications/networking/cluster/k3s/1_28/versions.nix +++ b/pkgs/applications/networking/cluster/k3s/1_28/versions.nix @@ -1,14 +1,14 @@ { - k3sVersion = "1.28.8+k3s1"; - k3sCommit = "653dd61aaa2d0ef8bd83ac4dbc6d150dde792efc"; - k3sRepoSha256 = "0pf8xw1m56m2s8i99vxj4i2l7fz7388kiynwzfrck43jb7v7kbbw"; - k3sVendorHash = "sha256-wglwRW2RO9QJI6CRLgkVg5Upt6R0M3gX76zy0kT02ec="; + k3sVersion = "1.28.10+k3s1"; + k3sCommit = "a4c5612ea3dd202135e7c691c534c671a7d43690"; + k3sRepoSha256 = "00r06kc98nvbmaai8m2pbqsl0v6y3kbc3rz3l7lb9wy4qhiyxrww"; + k3sVendorHash = "sha256-8PbpjPVX+Yimhwbydu9YOTIMRTf/iLG21Ee/QMowp5Y="; chartVersions = import ./chart-versions.nix; k3sRootVersion = "0.12.2"; k3sRootSha256 = "1gjynvr350qni5mskgm7pcc7alss4gms4jmkiv453vs8mmma9c9k"; k3sCNIVersion = "1.4.0-k3s2"; k3sCNISha256 = "17dg6jgjx18nrlyfmkv14dhzxsljz4774zgwz5dchxcf38bvarqa"; - containerdVersion = "1.7.11-k3s2"; - containerdSha256 = "0279sil02wz7310xhrgmdbc0r2qibj9lafy0i9k24jdrh74icmib"; + containerdVersion = "1.7.15-k3s1"; + containerdSha256 = "18hlj4ixjk7wvamfd66xyc0cax2hs9s7yjvlx52afxdc73194y0f"; criCtlVersion = "1.26.0-rc.0-k3s1"; } diff --git a/pkgs/applications/networking/cluster/k3s/1_29/chart-versions.nix b/pkgs/applications/networking/cluster/k3s/1_29/chart-versions.nix index 1acca4d0e101f..aaaa3d4c29700 100644 --- a/pkgs/applications/networking/cluster/k3s/1_29/chart-versions.nix +++ b/pkgs/applications/networking/cluster/k3s/1_29/chart-versions.nix @@ -1,10 +1,10 @@ { - traefik-crd = { - url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-25.0.2+up25.0.0.tgz"; - sha256 = "0jygzsn5pxzf7423x5iqfffgx5xvm7c7hfck46y7vpv1fdkiipcq"; - }; - traefik = { - url = "https://k3s.io/k3s-charts/assets/traefik/traefik-25.0.2+up25.0.0.tgz"; - sha256 = "1g9n19lnqdkmbbr3rnbwc854awha0kqqfwyxanyx1lg5ww8ldp89"; - }; + traefik-crd = { + url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-25.0.3+up25.0.0.tgz"; + sha256 = "1z693i4kd3jyf26ccnb0sxjyxadipl6k13n7jyg5v4y93fv1rpdw"; + }; + traefik = { + url = "https://k3s.io/k3s-charts/assets/traefik/traefik-25.0.3+up25.0.0.tgz"; + sha256 = "1a24qlp7c6iri72ka1i37l1lzn13xibrd26dy295z2wzr55gg7if"; + }; } diff --git a/pkgs/applications/networking/cluster/k3s/1_29/versions.nix b/pkgs/applications/networking/cluster/k3s/1_29/versions.nix index cf1f57e0fe601..e06f394df94a0 100644 --- a/pkgs/applications/networking/cluster/k3s/1_29/versions.nix +++ b/pkgs/applications/networking/cluster/k3s/1_29/versions.nix @@ -1,14 +1,14 @@ { - k3sVersion = "1.29.3+k3s1"; - k3sCommit = "8aecc26b0f167d5e9e4e9fbcfd5a471488bf5957"; - k3sRepoSha256 = "12285mhwi6cifsw3gjxxmd1g2i5f7vkdgzdc6a78rkvnx7z1j3p3"; - k3sVendorHash = "sha256-pID2h/rvvKyfHWoglYPbbliAby+9R2zoh7Ajd36qjVQ="; + k3sVersion = "1.29.5+k3s1"; + k3sCommit = "4e53a32306759581f4ed938bcd18b6fa20b83230"; + k3sRepoSha256 = "169hzl23chs4qblicmqj3j10jg1xdq8s9717bd3pzx7wzz9s9mqw"; + k3sVendorHash = "sha256-QreiB4JMtfBjHlkAyflQAW2rnfgay62UD6emx8TgUpM="; chartVersions = import ./chart-versions.nix; - k3sRootVersion = "0.12.2"; - k3sRootSha256 = "1gjynvr350qni5mskgm7pcc7alss4gms4jmkiv453vs8mmma9c9k"; + k3sRootVersion = "0.13.0"; + k3sRootSha256 = "1jq5f0lm08abx5ikarf92z56fvx4kjpy2nmzaazblb34lajw87vj"; k3sCNIVersion = "1.4.0-k3s2"; k3sCNISha256 = "17dg6jgjx18nrlyfmkv14dhzxsljz4774zgwz5dchxcf38bvarqa"; - containerdVersion = "1.7.11-k3s2"; - containerdSha256 = "0279sil02wz7310xhrgmdbc0r2qibj9lafy0i9k24jdrh74icmib"; + containerdVersion = "1.7.15-k3s1"; + containerdSha256 = "18hlj4ixjk7wvamfd66xyc0cax2hs9s7yjvlx52afxdc73194y0f"; criCtlVersion = "1.29.0-k3s1"; } diff --git a/pkgs/applications/networking/cluster/k3s/1_30/chart-versions.nix b/pkgs/applications/networking/cluster/k3s/1_30/chart-versions.nix new file mode 100644 index 0000000000000..aaaa3d4c29700 --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/1_30/chart-versions.nix @@ -0,0 +1,10 @@ +{ + traefik-crd = { + url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-25.0.3+up25.0.0.tgz"; + sha256 = "1z693i4kd3jyf26ccnb0sxjyxadipl6k13n7jyg5v4y93fv1rpdw"; + }; + traefik = { + url = "https://k3s.io/k3s-charts/assets/traefik/traefik-25.0.3+up25.0.0.tgz"; + sha256 = "1a24qlp7c6iri72ka1i37l1lzn13xibrd26dy295z2wzr55gg7if"; + }; +} diff --git a/pkgs/applications/networking/cluster/k3s/1_30/versions.nix b/pkgs/applications/networking/cluster/k3s/1_30/versions.nix new file mode 100644 index 0000000000000..23a3021875752 --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/1_30/versions.nix @@ -0,0 +1,14 @@ +{ + k3sVersion = "1.30.1+k3s1"; + k3sCommit = "80978b5b9a97908c5520c5ee51984e544e168859"; + k3sRepoSha256 = "085dmq49iwvlxpj9c528nfrvd67snkgpm5drj8ahfjv1nkjp0yy1"; + k3sVendorHash = "sha256-XtTahFaWnuHzKDI/U4d/j4C4gRxH163MCGEEM4hu/WM="; + chartVersions = import ./chart-versions.nix; + k3sRootVersion = "0.13.0"; + k3sRootSha256 = "1jq5f0lm08abx5ikarf92z56fvx4kjpy2nmzaazblb34lajw87vj"; + k3sCNIVersion = "1.4.0-k3s2"; + k3sCNISha256 = "17dg6jgjx18nrlyfmkv14dhzxsljz4774zgwz5dchxcf38bvarqa"; + containerdVersion = "1.7.15-k3s1"; + containerdSha256 = "18hlj4ixjk7wvamfd66xyc0cax2hs9s7yjvlx52afxdc73194y0f"; + criCtlVersion = "1.29.0-k3s1"; +} diff --git a/pkgs/applications/networking/cluster/k3s/README.md b/pkgs/applications/networking/cluster/k3s/README.md index c01a9ceab817f..9ea31423c7dbc 100644 --- a/pkgs/applications/networking/cluster/k3s/README.md +++ b/pkgs/applications/networking/cluster/k3s/README.md @@ -1,114 +1,21 @@ -# k3s versions +# K3s -K3s, Kubernetes, and other clustered software has the property of not being able to update atomically. Most software in nixpkgs, like for example bash, can be updated as part of a "nixos-rebuild switch" without having to worry about the old and the new bash interacting in some way. +K3s is a simplified [Kubernetes](https://wiki.nixos.org/wiki/Kubernetes) version that bundles Kubernetes cluster components into a few small binaries optimized for Edge and IoT devices. -K3s/Kubernetes, on the other hand, is typically run across several NixOS machines, and each NixOS machine is updated independently. As such, different versions of the package and NixOS module must maintain compatibility with each other through temporary version skew during updates. +## Usage -The upstream Kubernetes project [documents this in their version-skew policy](https://kubernetes.io/releases/version-skew-policy/#supported-component-upgrade-order). +* [Module Usage](docs/USAGE.md). -Within nixpkgs, we strive to maintain a valid "upgrade path" that does not run -afoul of the upstream version skew policy. +## Configuration Examples -## Upstream release cadence and support +* [Nvidia GPU Passthru](docs/examples/NVIDIA.md) +* [Storage Examples](docs/examples/STORAGE.md) -K3s is built on top of K8s, and typically provides a similar release cadence and support window (simply by cherry-picking over k8s patches). As such, we assume k3s's support lifecycle is identical to upstream K8s. +## Cluster Maintenance and Troubleshooting -This is documented upstream [here](https://kubernetes.io/releases/patch-releases/#support-period). +* [Cluster Upkeep](docs/CLUSTER_UPKEEP.md). -In short, a new Kubernetes version is released roughly every 4 months, and each release is supported for a little over 1 year. +## K3s Package Upkeep -Any version that is not supported by upstream should be dropped from nixpkgs. - -## Versions in NixOS releases - -NixOS releases should avoid having deprecated software, or making major version upgrades, wherever possible. - -As such, we would like to have only the newest K3s version in each NixOS -release at the time the release branch is branched off, which will ensure the -K3s version in that release will receive updates for the longest duration -possible. - -However, this conflicts with another desire: we would like people to be able to upgrade between NixOS stable releases without needing to make a large enough k3s version jump that they violate the Kubernetes version skew policy. - -To give an example, we may have the following timeline for k8s releases: - -(Note, the exact versions and dates may be wrong, this is an illustrative example, reality may differ). - -```mermaid -gitGraph - branch k8s - commit - branch "k8s-1.24" - checkout "k8s-1.24" - commit id: "1.24.0" tag: "2022-05-03" - branch "k8s-1.25" - checkout "k8s-1.25" - commit id: "1.25.0" tag: "2022-08-23" - branch "k8s-1.26" - checkout "k8s-1.26" - commit id: "1.26.0" tag: "2022-12-08" - checkout k8s-1.24 - commit id: "1.24-EOL" tag: "2023-07-28" - checkout k8s-1.25 - commit id: "1.25-EOL" tag: "2023-10-27" - checkout k8s-1.26 - commit id: "1.26-EOL" tag: "2024-02-28" -``` - -(Note: the above graph will render if you view this markdown on GitHub, or when using [mermaid](https://mermaid.js.org/)) - -In this scenario even though k3s 1.24 is still technically supported when the NixOS 23.05 -release is cut, since it goes EOL before the NixOS 23.11 release is made, we would -not want to include it. Similarly, k3s 1.25 would go EOL before NixOS 23.11. - -As such, we should only include k3s 1.26 in the 23.05 release. - -We can then make a similar argument when NixOS 23.11 comes around to not -include k3s 1.26 or 1.27. However, that means someone upgrading from the NixOS -22.05 release to the NixOS 23.11 would not have a supported upgrade path. - -In order to resolve this issue, we propose backporting not just new patch releases to older NixOS releases, but also new k3s versions, up to one version before the first version that is included in the next NixOS release. - -In the above example, where NixOS 23.05 included k3s 1.26, and 23.11 included k3s 1.28, that means we would backport 1.27 to the NixOS 23.05 release, and backport all patches for 1.26 and 1.27. -This would allow someone to upgrade between those NixOS releases in a supported configuration. - - -## K3s upkeep for nixpkgs maintainers - -* After every nixos release, K3s maintainers should remove from `nixos-unstable` all K3s versions that exist in `nixos-stable` except the latest version (to allow decoupling system upgrade from k3s upgrade). - -* Whenever adding a new major/minor K3s version to nixpkgs: - - update `k3s` alias to the latest version. - - add a NixOS release note scheduling the removal of deprecated K3s packages - - include migration information from both Kubernetes and K3s projects - -* For version patch upgrades, use the K3s update script. - - To execute the update script, from nixpkgs git repository, run: - - > ./pkgs/applications/networking/cluster/k3s/update-script.sh "29" - - "29" being the target minor version to be updated. - - On failure, the update script should be fixed. On failing to fix, open an issue reporting the update script breakage. - - RyanTM bot can automatically do patch upgrades. Update logs are available at: https://r.ryantm.com/log/k3s_1_29/ - -* When reviewing upgrades, check: - - - At top-level, every K3s version should have the Go compiler pinned according to `go.mod` file. - - Notice the update script does not automatically pin the Go version. - - - K3s passthru.tests (Currently: single-node, multi-node, etcd) works for all architectures (linux-x86_64, aarch64-linux). - - For GitHub CI, [OfBorg](https://github.com/NixOS/ofborg) can be used to test all platforms. - - To test locally, at nixpkgs repository, run: - > nix build .#k3s_1_29.passthru.tests.{etcd,single-node,multi-node} - - Replace "29" according to the version that you are testing. - - - Read the nix build logs to check for anything unusual. (Obvious but underrated.) - -* Thanks for reading the documentation and your continued contribution. +* [Package Versioning Rationale](docs/VERSIONING.md) +* [Package Maintenance Documentation](docs/PKG_UPKEEP.md) diff --git a/pkgs/applications/networking/cluster/k3s/builder.nix b/pkgs/applications/networking/cluster/k3s/builder.nix index b5f017b85b053..52754219efbfd 100644 --- a/pkgs/applications/networking/cluster/k3s/builder.nix +++ b/pkgs/applications/networking/cluster/k3s/builder.nix @@ -29,33 +29,42 @@ lib: # currently. # It is likely we will have to split out additional builders for additional # versions in the future, or customize this one further. -{ lib -, fetchpatch -, makeWrapper -, socat -, iptables -, iproute2 -, ipset -, bridge-utils -, btrfs-progs -, conntrack-tools -, buildGoModule -, runc -, rsync -, kmod -, libseccomp -, pkg-config -, ethtool -, util-linux -, fetchFromGitHub -, fetchurl -, fetchzip -, fetchgit -, zstd -, yq-go -, sqlite -, nixosTests -, pkgsBuildBuild +{ + lib, + makeWrapper, + socat, + iptables, + iproute2, + ipset, + bridge-utils, + btrfs-progs, + conntrack-tools, + buildGoModule, + runc, + rsync, + kmod, + libseccomp, + pkg-config, + ethtool, + util-linux, + fetchFromGitHub, + fetchurl, + fetchzip, + fetchgit, + zstd, + yq-go, + sqlite, + nixosTests, + pkgsBuildBuild, + go, + runCommand, + bash, + procps, + coreutils, + gnugrep, + findutils, + gnused, + systemd, }: # k3s is a kinda weird derivation. One of the main points of k3s is the @@ -80,10 +89,16 @@ lib: let baseMeta = with lib; { - description = "A lightweight Kubernetes distribution"; + description = "Lightweight Kubernetes distribution"; license = licenses.asl20; homepage = "https://k3s.io"; - maintainers = with maintainers; [ euank mic92 superherointj yajo ]; + maintainers = with maintainers; [ + euank + mic92 + superherointj + wrmilling + yajo + ]; platforms = platforms.linux; # resolves collisions with other installations of kubectl, crictl, ctr @@ -93,8 +108,9 @@ let # https://github.com/k3s-io/k3s/blob/5fb370e53e0014dc96183b8ecb2c25a61e891e76/scripts/build#L19-L40 versionldflags = [ - "-X github.com/rancher/k3s/pkg/version.Version=v${k3sVersion}" - "-X github.com/rancher/k3s/pkg/version.GitCommit=${lib.substring 0 8 k3sCommit}" + "-X github.com/k3s-io/k3s/pkg/version.Version=v${k3sVersion}" + "-X github.com/k3s-io/k3s/pkg/version.GitCommit=${lib.substring 0 8 k3sCommit}" + "-X github.com/k3s-io/k3s/pkg/version.UpstreamGolang=go${go.version}" "-X k8s.io/client-go/pkg/version.gitVersion=v${k3sVersion}" "-X k8s.io/client-go/pkg/version.gitCommit=${k3sCommit}" "-X k8s.io/client-go/pkg/version.gitTreeState=clean" @@ -156,6 +172,42 @@ let rev = "v${k3sVersion}"; sha256 = k3sRepoSha256; }; + + # Modify the k3s installer script so that we can let it install only + # killall.sh + k3sKillallSh = runCommand "k3s-killall.sh" { } '' + # Copy the upstream k3s install script except for the last lines that + # actually run the install process + sed --quiet '/# --- run the install process --/q;p' ${k3sRepo}/install.sh > install.sh + + # Let killall expect "containerd-shim" in the Nix store + to_replace="k3s/data/\[\^/\]\*/bin/containerd-shim" + replacement="/nix/store/.*k3s-containerd.*/bin/containerd-shim" + changes=$(sed -i "s|$to_replace|$replacement| w /dev/stdout" install.sh) + if [ -z "$changes" ]; then + echo "failed to replace \"$to_replace\" in k3s installer script (install.sh)" + exit 1 + fi + + remove_matching_line() { + line_to_delete=$(grep -n "$1" install.sh | cut -d : -f 1 || true) + if [ -z $line_to_delete ]; then + echo "failed to find expression \"$1\" in k3s installer script (install.sh)" + exit 1 + fi + sed -i "''${line_to_delete}d" install.sh + } + + # Don't change mode and owner of killall + remove_matching_line "chmod.*KILLALL_K3S_SH" + remove_matching_line "chown.*KILLALL_K3S_SH" + + # Execute only the "create_killall" function of the installer script + sed -i '$acreate_killall' install.sh + + KILLALL_K3S_SH=$out bash install.sh + ''; + # Stage 1 of the k3s build: # Let's talk about how k3s is structured. # One of the ideas of k3s is that there's the single "k3s" binary which can @@ -185,22 +237,20 @@ let src = k3sRepo; vendorHash = k3sVendorHash; - patches = - # Disable: Add runtime checking of golang version - (fetchpatch { - # https://github.com/k3s-io/k3s/pull/9054 - url = "https://github.com/k3s-io/k3s/commit/b297996b9252b02e56e9425f55f6becbf6bb7832.patch"; - hash = "sha256-xBOY2jnLhT9dtVKtq26V9QUnuX1q6E/9UcO9IaU719U="; - revert = true; - }); - nativeBuildInputs = [ pkg-config ]; - buildInputs = [ libseccomp sqlite.dev ]; + buildInputs = [ + libseccomp + sqlite.dev + ]; subPackages = [ "cmd/server" ]; ldflags = versionldflags; - tags = [ "ctrd" "libsqlite3" "linux" ]; + tags = [ + "ctrd" + "libsqlite3" + "linux" + ]; # create the multicall symlinks for k3s postInstall = '' @@ -222,7 +272,7 @@ let ''; meta = baseMeta // { - description = "The various binaries that get packaged into the final k3s binary"; + description = "Various binaries that get packaged into the final k3s binary"; }; }; # Only used for the shim since @@ -246,7 +296,11 @@ buildGoModule rec { pname = "k3s"; version = k3sVersion; - tags = [ "libsqlite3" "linux" "ctrd" ]; + tags = [ + "libsqlite3" + "linux" + "ctrd" + ]; src = k3sRepo; vendorHash = k3sVendorHash; @@ -284,6 +338,17 @@ buildGoModule rec { util-linux # kubelet wants 'nsenter' from util-linux: https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-705994388 conntrack-tools runc + bash + ]; + + k3sKillallDeps = [ + bash + systemd + procps + coreutils + gnugrep + findutils + gnused ]; buildInputs = k3sRuntimeDeps; @@ -342,6 +407,9 @@ buildGoModule rec { ln -s $out/bin/k3s $out/bin/kubectl ln -s $out/bin/k3s $out/bin/crictl ln -s $out/bin/k3s $out/bin/ctr + install -m 0755 ${k3sKillallSh} -D $out/bin/k3s-killall.sh + wrapProgram $out/bin/k3s-killall.sh \ + --prefix PATH : ${lib.makeBinPath (k3sRuntimeDeps ++ k3sKillallDeps)} ''; doInstallCheck = true; @@ -351,15 +419,17 @@ buildGoModule rec { passthru.updateScript = updateScript; - passthru.mkTests = version: - let k3s_version = "k3s_" + lib.replaceStrings ["."] ["_"] (lib.versions.majorMinor version); - in { + passthru.mkTests = + version: + let + k3s_version = "k3s_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor version); + in + { etcd = nixosTests.k3s.etcd.${k3s_version}; single-node = nixosTests.k3s.single-node.${k3s_version}; multi-node = nixosTests.k3s.multi-node.${k3s_version}; }; passthru.tests = passthru.mkTests k3sVersion; - meta = baseMeta; } diff --git a/pkgs/applications/networking/cluster/k3s/default.nix b/pkgs/applications/networking/cluster/k3s/default.nix index 934f5a3691cde..c8e3fbc926961 100644 --- a/pkgs/applications/networking/cluster/k3s/default.nix +++ b/pkgs/applications/networking/cluster/k3s/default.nix @@ -12,22 +12,35 @@ let extraArgs = builtins.removeAttrs args [ "callPackage" ]; in { - k3s_1_26 = common ((import ./1_26/versions.nix) // { - updateScript = [ ./update-script.sh "26" ]; - }) extraArgs; + # 1_28 can be built with the same builder as 1_30 + k3s_1_28 = common ( + (import ./1_28/versions.nix) + // { + updateScript = [ + ./update-script.sh + "28" + ]; + } + ) extraArgs; - # 1_27 can be built with the same builder as 1_26 - k3s_1_27 = common ((import ./1_27/versions.nix) // { - updateScript = [ ./update-script.sh "27" ]; - }) extraArgs; + # 1_29 can be built with the same builder as 1_30 + k3s_1_29 = common ( + (import ./1_29/versions.nix) + // { + updateScript = [ + ./update-script.sh + "29" + ]; + } + ) extraArgs; - # 1_28 can be built with the same builder as 1_26 - k3s_1_28 = common ((import ./1_28/versions.nix) // { - updateScript = [ ./update-script.sh "28" ]; - }) extraArgs; - - # 1_29 can be built with the same builder as 1_26 - k3s_1_29 = common ((import ./1_29/versions.nix) // { - updateScript = [ ./update-script.sh "29" ]; - }) extraArgs; + k3s_1_30 = common ( + (import ./1_30/versions.nix) + // { + updateScript = [ + ./update-script.sh + "30" + ]; + } + ) extraArgs; } diff --git a/pkgs/applications/networking/cluster/k3s/docs/CLUSTER_UPKEEP.md b/pkgs/applications/networking/cluster/k3s/docs/CLUSTER_UPKEEP.md new file mode 100644 index 0000000000000..2542fd78655e5 --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/docs/CLUSTER_UPKEEP.md @@ -0,0 +1,86 @@ + +# K3s Upkeep for Users + +General documentation for the K3s user for cluster tasks and troubleshooting steps. + +## Upkeep + +### Changing K3s Token + +Changing the K3s token requires resetting cluster. To reset the cluster, you must do the following: + +#### Stopping K3s + +Disabling K3s NixOS module won't stop K3s related dependencies, such as containerd or networking. For stopping everything, either run "k3s-killall.sh" script (available on $PATH under `/run/current-system/sw/bin/k3s-killall.sh`) or reboot host. + +### Syncing K3s in multiple hosts + +Nix automatically syncs hosts to `configuration.nix`, for syncing configuration.nix's git repository and triggering `nixos-rebuild switch` in multiple hosts, it is commonly used `ansible`, which enables automation of cluster provisioning, upgrade and reset. + +### Cluster Reset + +As upstream "k3s-uninstall.sh" is yet to be packaged for NixOS, it's necessary to run manual steps for resetting cluster. + +Disable K3s instances in **all** hosts: + +In NixOS configuration, set: +``` + services.k3s.enable = false; +``` +Rebuild NixOS. This is going to remove K3s service files. But it won't delete K3s data. + +To delete K3s files: + +Dismount kubelet: +``` + KUBELET_PATH=$(mount | grep kubelet | cut -d' ' -f3); + ${KUBELET_PATH:+umount $KUBELET_PATH} +``` +Delete k3s data: +``` + rm -rf /etc/rancher/{k3s,node}; + rm -rf /var/lib/{rancher/k3s,kubelet,longhorn,etcd,cni} +``` +When using Etcd, Reset Etcd: + +Certify **all** K3s instances are stopped, because a single instance can re-seed etcd database with previous cryptographic key. + +Disable etcd database in NixOS configuration: +``` + services.etcd.enable = false; +``` +Rebuild NixOS. + +Delete etcd files: +``` + rm -rf /var/lib/etcd/ +``` +Reboot hosts. + +In NixOS configuration: +``` + Re-enable Etcd first. Rebuild NixOS. Certify service health. (systemctl status etcd) + Re-enable K3s second. Rebuild NixOS. Certify service health. (systemctl status k3s) +``` +Etcd & K3s cluster will be provisioned new. + +Tip: Use Ansible to automate reset routine, like this. + +## Troubleshooting + +### Raspberry Pi not working + +If the k3s.service/k3s server does not start and gives you the error FATA[0000] failed to find memory cgroup (v2) Here's the github issue: https://github.com/k3s-io/k3s/issues/2067 . + +To fix the problem, you can add these things to your configuration.nix. +``` + boot.kernelParams = [ + "cgroup_enable=cpuset" "cgroup_memory=1" "cgroup_enable=memory" + ]; +``` + +### FailedKillPod: failed to get network "cbr0" cached result + +> KillPodSandboxError: failed to get network "cbr0" cached result: decoding version from network config: unexpected end of JSON input + +Workaround: https://github.com/k3s-io/k3s/issues/6185#issuecomment-1581245331 diff --git a/pkgs/applications/networking/cluster/k3s/docs/PKG_UPKEEP.md b/pkgs/applications/networking/cluster/k3s/docs/PKG_UPKEEP.md new file mode 100644 index 0000000000000..d41ef357da008 --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/docs/PKG_UPKEEP.md @@ -0,0 +1,60 @@ + +# K3s Upkeep for Maintainers + +General documentation for the K3s maintainer and reviewer use for consistency in maintenance processes. + +## NixOS Release Maintenance + +This process split into two sections and adheres to the versioning policy outlined in [VERSIONING.md](VERSIONING.md). + +### Pre-Release + +* Prior to the breaking change window of the next release being closed: + * `nixos-unstable`: Ensure k3s points to latest versioned release + * `nixos-unstable`: Ensure release notes are up to date + * `nixos-unstable`: Remove k3s releases which will be end of life upstream prior to end-of-life for the next NixOS stable release are removed with proper deprecation notice (process listed below) + +### Post-Release + +* For major/minor releases of k3s: + * `nixos-unstable`: Create a new versioned k3s package + * `nixos-unstable`: Update k3s alias to point to new versioned k3s package + * `nixos-unstable`: Add NixOS Release note denoting: + * Removal of deprecated K3s packages + * Migration information from the Kubernetes and K3s projects + * `nixos-stable`: Backport the versioned package +* For patch releases of existing packages: + * `nixos-unstable`: Update package version (process listed below) + * `nixos-stable`: Backport package update done to nixos-unstable + +## Patch Upgrade Process + +Patch upgrades can use the [update script](../update-script.sh) in the root of the package. To update k3s 1.30.x, for example, you can run the following from the root of the nixpkgs git repo: + +> ./pkgs/applications/networking/cluster/k3s/update-script.sh "30" + +To update another version, just replace the `"30"` with the appropriate minor revision. + +If the script should fail, the first goal would be to fix the script. If you are unable to fix the script, open an issue reporting the update script failure with the exact command used and the failure observed. + +RyanTM bot can automatically do patch upgrades. Update logs are available at versioned urls, e.g. for 1.30.x: https://r.ryantm.com/log/k3s_1_30 + +## Package Removal Process + +Package removal policy and timelines follow our reasoning in the [versioning documentation](VERSIONING.md#patch-release-support-lifecycle). In order to remove a versioned k3s package, create a PR achieving the following: + +* Remove the versioned folder containing the chart and package version files (e.g. `./1_30/`) +* Remove the package block from [default.nix](../default.nix) (e.g. `k3s_1_30 = ...`) +* Remove the package reference from [pkgs/top-level/all-packages.nix](/pkgs/top-level/all-packages.nix) +* Add a deprecation notice in [pkgs/top-level/aliases.nix](/pkgs/top-level/aliases.nix), such as `k3s_1_26 = throw "'k3s_1_26' has been removed from nixpkgs as it has reached end of life"; # Added 2024-05-20`. + +## Change Request Review Process + +Quick checklist for reviewers of the k3s package: + +* Is the version of the Go compiler pinned according to the go.mod file for the release? + * Update script will not pin nor change the go version. +* Do the K3s passthru.tests work for all architectures supported? (linux-x86_64, aarch64-linux) + * For GitHub CI, [OfBorg](https://github.com/NixOS/ofborg) can be used to test all platforms. + * For Local testing, the following can be run in nixpkgs root on the upgrade branch: `nix build .#k3s_1_29.passthru.tests.{etcd,single-node,multi-node}` (Replace "29" to the version tested) +* Anything unusual in the nix build logs or test logs? diff --git a/pkgs/applications/networking/cluster/k3s/docs/USAGE.md b/pkgs/applications/networking/cluster/k3s/docs/USAGE.md new file mode 100644 index 0000000000000..27c5963bd0b8f --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/docs/USAGE.md @@ -0,0 +1,57 @@ +# K3s Usage + +## Single Node + +``` +{ + networking.firewall.allowedTCPPorts = [ + 6443 # k3s: required so that pods can reach the API server (running on port 6443 by default) + # 2379 # k3s, etcd clients: required if using a "High Availability Embedded etcd" configuration + # 2380 # k3s, etcd peers: required if using a "High Availability Embedded etcd" configuration + ]; + networking.firewall.allowedUDPPorts = [ + # 8472 # k3s, flannel: required if using multi-node for inter-node networking + ]; + services.k3s.enable = true; + services.k3s.role = "server"; + services.k3s.extraFlags = toString [ + # "--kubelet-arg=v=4" # Optionally add additional args to k3s + ]; +} +``` + +Once the above changes are active, you can access your cluster through `sudo k3s kubectl` (e.g. `sudo k3s kubectl cluster-info`) or by using the generated kubeconfig file in `/etc/rancher/k3s/k3s.yaml`. +Multi-node setup + +## Multi-Node + +it is simple to create a cluster of multiple nodes in a highly available setup (all nodes are in the control-plane and are a part of the etcd cluster). + +The first node is configured like this: +``` +{ + services.k3s = { + enable = true; + role = "server"; + token = "<randomized common secret>"; + clusterInit = true; + }; +} +``` + +Any other subsequent nodes can be added with a slightly different config: + +``` +{ + services.k3s = { + enable = true; + role = "server"; # Or "agent" for worker only nodes + token = "<randomized common secret>"; + serverAddr = "https://<ip of first node>:6443"; + }; +} +``` + +For this to work you need to open the aforementioned API, etcd, and flannel ports in the firewall. Official documentation on what ports need to be opened for specific use cases can be found on [k3s' documentation site](https://docs.k3s.io/installation/requirements#inbound-rules-for-k3s-nodes). Note that it is [recommended](https://etcd.io/docs/v3.3/faq/#why-an-odd-number-of-cluster-members) to use an odd number of nodes in such a cluster. + +Tip: If you run into connectivity issues between nodes for specific applications (e.g. ingress controller), please verify the firewall settings you have enabled (example under [Single Node](#single-node)) against the documentation for that specific application. In the ingress controller example, you may want to open 443 or 80 depending on your use case. diff --git a/pkgs/applications/networking/cluster/k3s/docs/VERSIONING.md b/pkgs/applications/networking/cluster/k3s/docs/VERSIONING.md new file mode 100644 index 0000000000000..c1347b1f861e3 --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/docs/VERSIONING.md @@ -0,0 +1,46 @@ +# Versioning + +K3s, Kubernetes, and other clustered software has the property of not being able to update atomically. Most software in nixpkgs, like for example bash, can be updated as part of a "nixos-rebuild switch" without having to worry about the old and the new bash interacting in some way. + +K3s/Kubernetes, on the other hand, is typically run across several NixOS machines, and each NixOS machine is updated independently. As such, different versions of the package and NixOS module must maintain compatibility with each other through temporary version skew during updates. + +The upstream Kubernetes project [documents this in their version-skew policy](https://kubernetes.io/releases/version-skew-policy/#supported-component-upgrade-order). + +Within nixpkgs, we strive to maintain a valid "upgrade path" that does not run +afoul of the upstream version skew policy. + +## Patch Release Support Lifecycle + +K3s is built on top of K8s and typically provides a similar release cadence and support window (simply by cherry-picking over k8s patches). As such, we assume k3s's support lifecycle is identical to upstream K8s. The upstream K8s release and support lifecycle, including maintenance and end-of-life dates for current releases, is documented [on their suppport site](https://kubernetes.io/releases/patch-releases/#support-period). A more tabular view of the current support timeline can also be found on [endoflife.date](https://endoflife.date/kubernetes). + +In short, a new Kubernetes version is released roughly every 4 months and each release is supported for a little over 1 year. + +## Versioning in nixpkgs + +There are two package types that are maintained within nixpkgs when we are looking at the `nixos-unstable` branch. A standard `k3s` package and versioned releases such as `k3s_1_28`, `k3s_1_29`, and `k3s_1_30`. + +The standard `k3s` package will be updated as new versions of k3s are released upstream. Versioned releases, on the other hand, will follow the path release support lifecycle as detailed in the previous section and be removed from `nixos-unstable` when they are either end-of-life upstream or older than the current `k3s` package in `nixos-stable`. + +## Versioning in NixOS Releases + +Those same package types are also maintained on the release branches of NixOS, but have some special considerations within a release. + +NixOS releases (24.05, 24.11, etc) should avoid having deprecated software or major version upgrades during the support lifecycle of that release wherever possible. As such, each NixOS release should only ever have one version of `k3s` when it is released. An example for the NixOS 24.05 release would be that `k3s` package points to `k3s_1_30` for the full lifecycle of its release with no other versions present at release. + +However, this conflicts with our desire for users to be able to upgrade between stable NixOS releases without needing to make a large enough k3s version jump as to violate the skew policy listed previously. Given NixOS 24.05 has 1.30.x as its k3s version and the NixOS 24.11 release would have 1.32.x as its k3s version, we need to provide a way for users to upgrade k3s to 1.32.x before upgrading to the next NixOS stable release. + +To be able to achieve the goal above, the k3s maintainers would backport `k3s_1_31` and `k3s_1_32` from `nixos-unstable` to NixOS 24.05 as they release. This means that when NixOS 24.11 is released with only the `k3s` package pointing to `k3s_1_32`, users will have an upgrade path on 24.05 to first upgrade locally to `k3s_1_31` and then to `k3s_1_32` (e.g. pointing `services.k3s.package` from `k3s` to `k3s_1_31`, upgrading the cluster, and repeating the process through versions). + +Using the above as the example, a three NixOS release example would look like: + +* NixOS 23.11 + * k3s/k3s_1_27 (Release Version, patches backported) + * k3s_1_28 (Backported) + * k3s_1_29 (Backported) + * k3s_1_30 (Backported) +* NixOS 24.05 + * k3s/k3s_1_30 (Release Version, patches backported) + * k3s_1_31 (Backported) + * k3s_1_32 (Backported) +* NixOS 24.11 + * k3s/k3s_1_32 (Release Version, patches backported) diff --git a/pkgs/applications/networking/cluster/k3s/docs/examples/NVIDIA.md b/pkgs/applications/networking/cluster/k3s/docs/examples/NVIDIA.md new file mode 100644 index 0000000000000..26ec48e0273c9 --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/docs/examples/NVIDIA.md @@ -0,0 +1,55 @@ +# Nvidia GPU Support + +To use Nvidia GPU in the cluster the nvidia-container-runtime and runc are needed. To get the two components it suffices to add the following to the configuration + +``` +virtualisation.docker = { + enable = true; + enableNvidia = true; +}; +environment.systemPackages = with pkgs; [ docker runc ]; +``` + +Note, using docker here is a workaround, it will install nvidia-container-runtime and that will cause it to be accessible via /run/current-system/sw/bin/nvidia-container-runtime, currently its not directly accessible in nixpkgs. + +You now need to create a new file in `/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl` with the following + +``` +{{ template "base" . }} + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia] + privileged_without_host_devices = false + runtime_engine = "" + runtime_root = "" + runtime_type = "io.containerd.runc.v2" + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options] + BinaryName = "/run/current-system/sw/bin/nvidia-container-runtime" +``` + +Update: As of 12/03/2024 It appears that the last two lines above are added by default, and if the two lines are present (as shown above) it will refuse to start the server. You will need to remove the two lines from that point onward. + +Note here we are pointing the nvidia runtime to "/run/current-system/sw/bin/nvidia-container-runtime". + +Now apply the following runtime class to k3s cluster: + +``` +apiVersion: node.k8s.io/v1 +handler: nvidia +kind: RuntimeClass +metadata: + labels: + app.kubernetes.io/component: gpu-operator + name: nvidia +``` + +Following [k8s-device-plugin](https://github.com/NVIDIA/k8s-device-plugin#deployment-via-helm) install the helm chart with `runtimeClassName: nvidia` set. In order to passthrough the nvidia card into the container, your deployments spec must contain - runtimeClassName: nvidia - env: + +``` + - name: NVIDIA_VISIBLE_DEVICES + value: all + - name: NVIDIA_DRIVER_CAPABILITIES + value: all +``` + +to test its working exec onto a pod and run nvidia-smi. For more configurability of nvidia related matters in k3s look in [k3s-docs](https://docs.k3s.io/advanced#nvidia-container-runtime-support). diff --git a/pkgs/applications/networking/cluster/k3s/docs/examples/STORAGE.md b/pkgs/applications/networking/cluster/k3s/docs/examples/STORAGE.md new file mode 100644 index 0000000000000..fc9a6440ff9ae --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/docs/examples/STORAGE.md @@ -0,0 +1,122 @@ +# Storage Examples + +The following are some NixOS specific considerations for specific storage mechanisms with kubernetes/k3s. + +## Longhorn + +NixOS configuration required for Longhorn: + +``` +environment.systemPackages = [ pkgs.nfs-utils ]; +services.openiscsi = { + enable = true; + name = "${config.networking.hostName}-initiatorhost"; +}; +``` + +Longhorn container has trouble with NixOS path. Solution is to override PATH environment variable, such as: + +``` +PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/run/wrappers/bin:/nix/var/nix/profiles/default/bin:/run/current-system/sw/bin +``` + +**Kyverno Policy for Fixing Longhorn Container for NixOS** + +``` +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-nixos-path + namespace: longhorn-system +data: + PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/run/wrappers/bin:/nix/var/nix/profiles/default/bin:/run/current-system/sw/bin +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: longhorn-add-nixos-path + annotations: + policies.kyverno.io/title: Add Environment Variables from ConfigMap + policies.kyverno.io/subject: Pod + policies.kyverno.io/category: Other + policies.kyverno.io/description: >- + Longhorn invokes executables on the host system, and needs + to be aware of the host systems PATH. This modifies all + deployments such that the PATH is explicitly set to support + NixOS based systems. +spec: + rules: + - name: add-env-vars + match: + resources: + kinds: + - Pod + namespaces: + - longhorn-system + mutate: + patchStrategicMerge: + spec: + initContainers: + - (name): "*" + envFrom: + - configMapRef: + name: longhorn-nixos-path + containers: + - (name): "*" + envFrom: + - configMapRef: + name: longhorn-nixos-path +--- +``` + +## NFS + +NixOS configuration required for NFS: + +``` +boot.supportedFilesystems = [ "nfs" ]; +services.rpcbind.enable = true; +``` + +## Rook/Ceph + +In order to support Rook/Ceph, the following NixOS kernelModule configuration is required: + +``` + boot.kernelModules = [ "rbd" ]; +``` + +## ZFS Snapshot Support + +K3s's builtin containerd does not support the zfs snapshotter. However, it is possible to configure it to use an external containerd: + +``` +virtualisation.containerd = { + enable = true; + settings = + let + fullCNIPlugins = pkgs.buildEnv { + name = "full-cni"; + paths = with pkgs;[ + cni-plugins + cni-plugin-flannel + ]; + }; + in { + plugins."io.containerd.grpc.v1.cri".cni = { + bin_dir = "${fullCNIPlugins}/bin"; + conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/"; + }; + # Optionally set private registry credentials here instead of using /etc/rancher/k3s/registries.yaml + # plugins."io.containerd.grpc.v1.cri".registry.configs."registry.example.com".auth = { + # username = ""; + # password = ""; + # }; + }; +}; +# TODO describe how to enable zfs snapshotter in containerd +services.k3s.extraFlags = toString [ + "--container-runtime-endpoint unix:///run/containerd/containerd.sock" +]; +``` diff --git a/pkgs/applications/networking/cluster/k3s/update-script.sh b/pkgs/applications/networking/cluster/k3s/update-script.sh index 9f534f4d179a7..ca7d21bff4781 100755 --- a/pkgs/applications/networking/cluster/k3s/update-script.sh +++ b/pkgs/applications/networking/cluster/k3s/update-script.sh @@ -57,14 +57,14 @@ CHARTS_URL=https://k3s.io/k3s-charts/assets rm -f chart-versions.nix.update cat > chart-versions.nix.update <<EOF { - traefik-crd = { - url = "${CHARTS_URL}/traefik-crd/${CHART_FILES[0]}"; - sha256 = "$(nix-prefetch-url --quiet "${CHARTS_URL}/traefik-crd/${CHART_FILES[0]}")"; - }; - traefik = { - url = "${CHARTS_URL}/traefik/${CHART_FILES[1]}"; - sha256 = "$(nix-prefetch-url --quiet "${CHARTS_URL}/traefik/${CHART_FILES[1]}")"; - }; + traefik-crd = { + url = "${CHARTS_URL}/traefik-crd/${CHART_FILES[0]}"; + sha256 = "$(nix-prefetch-url --quiet "${CHARTS_URL}/traefik-crd/${CHART_FILES[0]}")"; + }; + traefik = { + url = "${CHARTS_URL}/traefik/${CHART_FILES[1]}"; + sha256 = "$(nix-prefetch-url --quiet "${CHARTS_URL}/traefik/${CHART_FILES[1]}")"; + }; } EOF mv chart-versions.nix.update chart-versions.nix |