1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
|
{ stdenv
, lib
, makeWrapper
, socat
, iptables
, iproute2
, bridge-utils
, btrfs-progs
, conntrack-tools
, buildGoModule
, runc
, rsync
, kmod
, libseccomp
, pkg-config
, ethtool
, util-linux
, fetchFromGitHub
, fetchurl
, fetchzip
, fetchgit
, zstd
, yq-go
, nixosTests
}:
with lib;
# k3s is a kinda weird derivation. One of the main points of k3s is the
# simplicity of it being one binary that can perform several tasks.
# However, when you have a good package manager (like nix), that doesn't
# actually make much of a difference; you don't really care if it's one binary
# or 10 since with a good package manager, installing and running it is
# identical.
# Since upstream k3s packages itself as one large binary with several
# "personalities" (in the form of subcommands like 'k3s agent' and 'k3s
# kubectl'), it ends up being easiest to mostly mimic upstream packaging, with
# some exceptions.
# K3s also carries patches to some packages (such as containerd and cni
# plugins), so we intentionally use the k3s versions of those binaries for k3s,
# even if the upstream version of those binaries exist in nixpkgs already. In
# the end, that means we have a thick k3s binary that behaves like the upstream
# one for the most part.
# However, k3s also bundles several pieces of unpatched software, from the
# strongswan vpn software, to iptables, to socat, conntrack, busybox, etc.
# Those pieces of software we entirely ignore upstream's handling of, and just
# make sure they're in the path if desired.
let
k3sVersion = "1.23.5+k3s1"; # k3s git tag
k3sCommit = "313aaca547f030752788dce696fdf8c9568bc035"; # k3s git commit at the above version
k3sRepoSha256 = "0vk72609cyyh64irp14jp2zspnxw34jm710cbwgklx0ch6kiz88d";
k3sVendorSha256 = "sha256-d7kQsJi/eQbaTUDglp3gFpc5Im6CyD9coKeM3kMrbjI=";
k3sServerVendorSha256 = "sha256-E3USXNuXY0lzZH+t3O7BOQ8rKNNQ6avOMItgOEi1cEg=";
# taken from ./manifests/traefik.yaml, extracted from '.spec.chart' https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/download#L9
# The 'patch' and 'minor' versions are currently hardcoded as single digits only, so ignore the trailing two digits. Weird, I know.
traefikChartVersion = "10.14.1";
traefikChartSha256 = "09a6cialx7nrh7nwi1gkkh8zcsasxcgb52dyx0r8bjq9ng29simj";
# taken from ./scripts/version.sh VERSION_ROOT https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L47
k3sRootVersion = "0.11.0";
k3sRootSha256 = "016n56vi09xkvjph7wgzb2m86mhd5x65fs4d11pmh20hl249r620";
# taken from ./scripts/version.sh VERSION_CNIPLUGINS https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L45
k3sCNIVersion = "1.0.1-k3s1";
k3sCNISha256 = "11ihlzzdnqf9p21y0a4ckpbxac016nm7746dcykhj26ym9zxyv92";
# taken from go.mod, the 'github.com/containerd/containerd' line
# run `grep github.com/containerd/containerd go.mod | head -n1 | awk '{print $4}'`
containerdVersion = "1.5.10-k3s1";
containerdSha256 = "1ff2sfaqpjimq7w0lprci6ibyi6v65ap6b9sr6b0j12gqr2sqwa5";
# run `grep github.com/kubernetes-sigs/cri-tools go.mod | head -n1 | awk '{print $4}'` in the k3s repo at the tag
criCtlVersion = "1.22.0-k3s1";
baseMeta = {
description = "A lightweight Kubernetes distribution";
license = licenses.asl20;
homepage = "https://k3s.io";
maintainers = with maintainers; [ euank mic92 superherointj ];
platforms = platforms.linux;
};
# https://github.com/k3s-io/k3s/blob/5fb370e53e0014dc96183b8ecb2c25a61e891e76/scripts/build#L19-L40
versionldflags = [
"-X github.com/rancher/k3s/pkg/version.Version=v${k3sVersion}"
"-X github.com/rancher/k3s/pkg/version.GitCommit=${lib.substring 0 8 k3sCommit}"
"-X k8s.io/client-go/pkg/version.gitVersion=v${k3sVersion}"
"-X k8s.io/client-go/pkg/version.gitCommit=${k3sCommit}"
"-X k8s.io/client-go/pkg/version.gitTreeState=clean"
"-X k8s.io/client-go/pkg/version.buildDate=1970-01-01T01:01:01Z"
"-X k8s.io/component-base/version.gitVersion=v${k3sVersion}"
"-X k8s.io/component-base/version.gitCommit=${k3sCommit}"
"-X k8s.io/component-base/version.gitTreeState=clean"
"-X k8s.io/component-base/version.buildDate=1970-01-01T01:01:01Z"
"-X github.com/kubernetes-sigs/cri-tools/pkg/version.Version=v${criCtlVersion}"
"-X github.com/containerd/containerd/version.Version=v${containerdVersion}"
"-X github.com/containerd/containerd/version.Package=github.com/k3s-io/containerd"
];
# bundled into the k3s binary
traefikChart = fetchurl {
url = "https://helm.traefik.io/traefik/traefik-${traefikChartVersion}.tgz";
sha256 = traefikChartSha256;
};
# so, k3s is a complicated thing to package
# This derivation attempts to avoid including any random binaries from the
# internet. k3s-root is _mostly_ binaries built to be bundled in k3s (which
# we don't care about doing, we can add those as build or runtime
# dependencies using a real package manager).
# In addition to those binaries, it's also configuration though (right now
# mostly strongswan configuration), and k3s does use those files.
# As such, we download it in order to grab 'etc' and bundle it into the final
# k3s binary.
k3sRoot = fetchzip {
# Note: marked as apache 2.0 license
url = "https://github.com/k3s-io/k3s-root/releases/download/v${k3sRootVersion}/k3s-root-amd64.tar";
sha256 = k3sRootSha256;
stripRoot = false;
};
k3sCNIPlugins = buildGoModule rec {
pname = "k3s-cni-plugins";
version = k3sCNIVersion;
vendorSha256 = null;
subPackages = [ "." ];
src = fetchFromGitHub {
owner = "rancher";
repo = "plugins";
rev = "v${version}";
sha256 = k3sCNISha256;
};
postInstall = ''
mv $out/bin/plugins $out/bin/cni
'';
meta = baseMeta // {
description = "CNI plugins, as patched by rancher for k3s";
};
};
# Grab this separately from a build because it's used by both stages of the
# k3s build.
k3sRepo = fetchgit {
url = "https://github.com/k3s-io/k3s";
rev = "v${k3sVersion}";
sha256 = k3sRepoSha256;
};
# Stage 1 of the k3s build:
# Let's talk about how k3s is structured.
# One of the ideas of k3s is that there's the single "k3s" binary which can
# do everything you need, from running a k3s server, to being a worker node,
# to running kubectl.
# The way that actually works is that k3s is a single go binary that contains
# a bunch of bindata that it unpacks at runtime into directories (either the
# user's home directory or /var/lib/rancher if run as root).
# This bindata includes both binaries and configuration.
# In order to let nixpkgs do all its autostripping/patching/etc, we split this into two derivations.
# First, we build all the binaries that get packed into the thick k3s binary
# (and output them from one derivation so they'll all be suitably patched up).
# Then, we bundle those binaries into our thick k3s binary and use that as
# the final single output.
# This approach was chosen because it ensures the bundled binaries all are
# correctly built to run with nix (we can lean on the existing buildGoModule
# stuff), and we can again lean on that tooling for the final k3s binary too.
# Other alternatives would be to manually run the
# strip/patchelf/remove-references step ourselves in the installPhase of the
# derivation when we've built all the binaries, but haven't bundled them in
# with generated bindata yet.
k3sServer = buildGoModule rec {
pname = "k3s-server";
version = k3sVersion;
src = k3sRepo;
vendorSha256 = k3sServerVendorSha256;
nativeBuildInputs = [ pkg-config ];
buildInputs = [ libseccomp ];
subPackages = [ "cmd/server" ];
ldflags = versionldflags;
# create the multicall symlinks for k3s
postInstall = ''
mv $out/bin/server $out/bin/k3s
pushd $out
# taken verbatim from https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/build#L105-L113
ln -s k3s ./bin/k3s-agent
ln -s k3s ./bin/k3s-server
ln -s k3s ./bin/k3s-etcd-snapshot
ln -s k3s ./bin/k3s-secrets-encrypt
ln -s k3s ./bin/k3s-certificate
ln -s k3s ./bin/kubectl
ln -s k3s ./bin/crictl
ln -s k3s ./bin/ctr
popd
'';
meta = baseMeta // {
description = "The various binaries that get packaged into the final k3s binary";
};
};
k3sContainerd = buildGoModule {
pname = "k3s-containerd";
version = containerdVersion;
src = fetchFromGitHub {
owner = "k3s-io";
repo = "containerd";
rev = "v${containerdVersion}";
sha256 = containerdSha256;
};
vendorSha256 = null;
buildInputs = [ btrfs-progs ];
subPackages = [ "cmd/containerd" "cmd/containerd-shim-runc-v2" ];
ldflags = versionldflags;
};
in
buildGoModule rec {
pname = "k3s";
version = k3sVersion;
src = k3sRepo;
proxyVendor = true;
vendorSha256 = k3sVendorSha256;
patches = [
./patches/0001-scrips-download-strip-downloading-just-package-CRD.patch
];
postPatch = ''
# Nix prefers dynamically linked binaries over static binary.
substituteInPlace scripts/package-cli \
--replace '"$LDFLAGS $STATIC" -o' \
'"$LDFLAGS" -o' \
--replace "STATIC=\"-extldflags \'-static\'\"" \
""
# Upstream codegen fails with trimpath set. Removes "trimpath" for 'go generate':
substituteInPlace scripts/package-cli \
--replace '"''${GO}" generate' \
'GOFLAGS="" "''${GO}" generate'
'';
# Important utilities used by the kubelet, see
# https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-237202494
# Note the list in that issue is stale and some aren't relevant for k3s.
k3sRuntimeDeps = [
kmod
socat
iptables
iproute2
bridge-utils
ethtool
util-linux # kubelet wants 'nsenter' from util-linux: https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-705994388
conntrack-tools
];
buildInputs = k3sRuntimeDeps;
nativeBuildInputs = [
makeWrapper
rsync
yq-go
zstd
];
# embedded in the final k3s cli
propagatedBuildInputs = [
k3sCNIPlugins
k3sContainerd
k3sServer
runc
];
# We override most of buildPhase due to peculiarities in k3s's build.
# Specifically, it has a 'go generate' which runs part of the package. See
# this comment:
# https://github.com/NixOS/nixpkgs/pull/158089#discussion_r799965694
# So, why do we use buildGoModule at all? For the `vendorSha256` / `go mod download` stuff primarily.
buildPhase = ''
patchShebangs ./scripts/package-cli ./scripts/download ./scripts/build-upload
# copy needed 'go generate' inputs into place
mkdir -p ./bin/aux
rsync -a --no-perms ${k3sServer}/bin/ ./bin/
ln -vsf ${runc}/bin/runc ./bin/runc
ln -vsf ${k3sCNIPlugins}/bin/cni ./bin/cni
ln -vsf ${k3sContainerd}/bin/* ./bin/
rsync -a --no-perms --chmod u=rwX ${k3sRoot}/etc/ ./etc/
mkdir -p ./build/static/charts
# Note, upstream's chart has a 00 suffix. This seems to not matter though, so we're ignoring that naming detail.
export TRAEFIK_CHART_FILE=${traefikChart}
# place the traefik chart using their code since it's complicated
# We trim the actual download, see patches
./scripts/download
export ARCH=$GOARCH
export DRONE_TAG="v${k3sVersion}"
export DRONE_COMMIT="${k3sCommit}"
# use ./scripts/package-cli to run 'go generate' + 'go build'
./scripts/package-cli
mkdir -p $out/bin
'';
# Otherwise it depends on 'getGoDirs', which is normally set in buildPhase
doCheck = false;
installPhase = ''
# wildcard to match the arm64 build too
install -m 0755 dist/artifacts/k3s* -D $out/bin/k3s
wrapProgram $out/bin/k3s \
--prefix PATH : ${lib.makeBinPath k3sRuntimeDeps} \
--prefix PATH : "$out/bin"
'';
doInstallCheck = true;
installCheckPhase = ''
$out/bin/k3s --version | grep -F "v${k3sVersion}" >/dev/null
'';
passthru.updateScript = ./update.sh;
passthru.tests = { inherit (nixosTests) k3s-single-node k3s-single-node-docker; };
meta = baseMeta;
}
|