diff options
Diffstat (limited to 'nixos/tests/k3s')
| -rw-r--r-- | nixos/tests/k3s/default.nix | 18 | ||||
| -rw-r--r-- | nixos/tests/k3s/etcd.nix | 200 | ||||
| -rw-r--r-- | nixos/tests/k3s/multi-node.nix | 226 | ||||
| -rw-r--r-- | nixos/tests/k3s/single-node.nix | 131 |
4 files changed, 356 insertions, 219 deletions
diff --git a/nixos/tests/k3s/default.nix b/nixos/tests/k3s/default.nix index 512dc06ee77ec..297b05a4e4a74 100644 --- a/nixos/tests/k3s/default.nix +++ b/nixos/tests/k3s/default.nix @@ -1,16 +1,20 @@ -{ system ? builtins.currentSystem -, pkgs ? import ../../.. { inherit system; } -, lib ? pkgs.lib +{ + system ? builtins.currentSystem, + pkgs ? import ../../.. { inherit system; }, + lib ? pkgs.lib, }: let allK3s = lib.filterAttrs (n: _: lib.strings.hasPrefix "k3s_" n) pkgs; in { # Testing K3s with Etcd backend - etcd = lib.mapAttrs (_: k3s: import ./etcd.nix { - inherit system pkgs k3s; - inherit (pkgs) etcd; - }) allK3s; + etcd = lib.mapAttrs ( + _: k3s: + import ./etcd.nix { + inherit system pkgs k3s; + inherit (pkgs) etcd; + } + ) allK3s; # Run a single node k3s cluster and verify a pod can run single-node = lib.mapAttrs (_: k3s: import ./single-node.nix { inherit system pkgs k3s; }) allK3s; # Run a multi-node k3s cluster and verify pod networking works across nodes diff --git a/nixos/tests/k3s/etcd.nix b/nixos/tests/k3s/etcd.nix index d6e9a294adb13..ac0aa90472516 100644 --- a/nixos/tests/k3s/etcd.nix +++ b/nixos/tests/k3s/etcd.nix @@ -1,100 +1,130 @@ -import ../make-test-python.nix ({ pkgs, lib, k3s, etcd, ... }: - -{ - name = "${k3s.name}-etcd"; - - nodes = { - - etcd = { ... }: { - services.etcd = { - enable = true; - openFirewall = true; - listenClientUrls = [ "http://192.168.1.1:2379" "http://127.0.0.1:2379" ]; - listenPeerUrls = [ "http://192.168.1.1:2380" ]; - initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ]; - initialCluster = [ "etcd=http://192.168.1.1:2380" ]; - }; - networking = { - useDHCP = false; - defaultGateway = "192.168.1.1"; - interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ - { address = "192.168.1.1"; prefixLength = 24; } - ]; - }; - }; +import ../make-test-python.nix ( + { + pkgs, + lib, + k3s, + etcd, + ... + }: + + { + name = "${k3s.name}-etcd"; + + nodes = { + + etcd = + { ... }: + { + services.etcd = { + enable = true; + openFirewall = true; + listenClientUrls = [ + "http://192.168.1.1:2379" + "http://127.0.0.1:2379" + ]; + listenPeerUrls = [ "http://192.168.1.1:2380" ]; + initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ]; + initialCluster = [ "etcd=http://192.168.1.1:2380" ]; + }; + networking = { + useDHCP = false; + defaultGateway = "192.168.1.1"; + interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { + address = "192.168.1.1"; + prefixLength = 24; + } + ]; + }; + }; - k3s = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ jq ]; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s = { - enable = true; - role = "server"; - extraFlags = builtins.toString [ - "--datastore-endpoint=\"http://192.168.1.1:2379\"" - "--disable" "coredns" - "--disable" "local-storage" - "--disable" "metrics-server" - "--disable" "servicelb" - "--disable" "traefik" - "--node-ip" "192.168.1.2" - ]; - }; - - networking = { - firewall = { - allowedTCPPorts = [ 2379 2380 6443 ]; - allowedUDPPorts = [ 8472 ]; + k3s = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ jq ]; + # k3s uses enough resources the default vm fails. + virtualisation.memorySize = 1536; + virtualisation.diskSize = 4096; + + services.k3s = { + enable = true; + role = "server"; + extraFlags = builtins.toString [ + "--datastore-endpoint=\"http://192.168.1.1:2379\"" + "--disable" + "coredns" + "--disable" + "local-storage" + "--disable" + "metrics-server" + "--disable" + "servicelb" + "--disable" + "traefik" + "--node-ip" + "192.168.1.2" + ]; + }; + + networking = { + firewall = { + allowedTCPPorts = [ + 2379 + 2380 + 6443 + ]; + allowedUDPPorts = [ 8472 ]; + }; + useDHCP = false; + defaultGateway = "192.168.1.2"; + interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { + address = "192.168.1.2"; + prefixLength = 24; + } + ]; + }; }; - useDHCP = false; - defaultGateway = "192.168.1.2"; - interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ - { address = "192.168.1.2"; prefixLength = 24; } - ]; - }; }; - }; - - testScript = '' - with subtest("should start etcd"): - etcd.start() - etcd.wait_for_unit("etcd.service") + testScript = '' + with subtest("should start etcd"): + etcd.start() + etcd.wait_for_unit("etcd.service") - with subtest("should wait for etcdctl endpoint status to succeed"): - etcd.wait_until_succeeds("etcdctl endpoint status") + with subtest("should wait for etcdctl endpoint status to succeed"): + etcd.wait_until_succeeds("etcdctl endpoint status") - with subtest("should start k3s"): - k3s.start() - k3s.wait_for_unit("k3s") + with subtest("should start k3s"): + k3s.start() + k3s.wait_for_unit("k3s") - with subtest("should test if kubectl works"): - k3s.wait_until_succeeds("k3s kubectl get node") + with subtest("should test if kubectl works"): + k3s.wait_until_succeeds("k3s kubectl get node") - with subtest("should wait for service account to show up; takes a sec"): - k3s.wait_until_succeeds("k3s kubectl get serviceaccount default") + with subtest("should wait for service account to show up; takes a sec"): + k3s.wait_until_succeeds("k3s kubectl get serviceaccount default") - with subtest("should create a sample secret object"): - k3s.succeed("k3s kubectl create secret generic nixossecret --from-literal thesecret=abacadabra") + with subtest("should create a sample secret object"): + k3s.succeed("k3s kubectl create secret generic nixossecret --from-literal thesecret=abacadabra") - with subtest("should check if secret is correct"): - k3s.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]") + with subtest("should check if secret is correct"): + k3s.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]") - with subtest("should have a secret in database"): - etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") + with subtest("should have a secret in database"): + etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") - with subtest("should delete the secret"): - k3s.succeed("k3s kubectl delete secret nixossecret") + with subtest("should delete the secret"): + k3s.succeed("k3s kubectl delete secret nixossecret") - with subtest("should not have a secret in database"): - etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") + with subtest("should not have a secret in database"): + etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") - with subtest("should shutdown k3s and etcd"): - k3s.shutdown() - etcd.shutdown() - ''; + with subtest("should shutdown k3s and etcd"): + k3s.shutdown() + etcd.shutdown() + ''; - meta.maintainers = etcd.meta.maintainers ++ k3s.meta.maintainers; -}) + meta.maintainers = etcd.meta.maintainers ++ k3s.meta.maintainers; + } +) diff --git a/nixos/tests/k3s/multi-node.nix b/nixos/tests/k3s/multi-node.nix index 20279f3ca4b93..b618d2aff34c4 100644 --- a/nixos/tests/k3s/multi-node.nix +++ b/nixos/tests/k3s/multi-node.nix @@ -1,14 +1,30 @@ -import ../make-test-python.nix ({ pkgs, lib, k3s, ... }: +import ../make-test-python.nix ( + { + pkgs, + lib, + k3s, + ... + }: let imageEnv = pkgs.buildEnv { name = "k3s-pause-image-env"; - paths = with pkgs; [ tini bashInteractive coreutils socat ]; + paths = with pkgs; [ + tini + bashInteractive + coreutils + socat + ]; }; pauseImage = pkgs.dockerTools.streamLayeredImage { name = "test.local/pause"; tag = "local"; contents = imageEnv; - config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ]; + config.Entrypoint = [ + "/bin/tini" + "--" + "/bin/sleep" + "inf" + ]; }; # A daemonset that responds 'server' on port 8000 networkTestDaemonset = pkgs.writeText "test.yml" '' @@ -42,90 +58,135 @@ import ../make-test-python.nix ({ pkgs, lib, k3s, ... }: name = "${k3s.name}-multi-node"; nodes = { - server = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ gzip jq ]; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s = { - inherit tokenFile; - enable = true; - role = "server"; - package = k3s; - clusterInit = true; - extraFlags = builtins.toString [ - "--disable" "coredns" - "--disable" "local-storage" - "--disable" "metrics-server" - "--disable" "servicelb" - "--disable" "traefik" - "--node-ip" "192.168.1.1" - "--pause-image" "test.local/pause:local" + server = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + gzip + jq + ]; + # k3s uses enough resources the default vm fails. + virtualisation.memorySize = 1536; + virtualisation.diskSize = 4096; + + services.k3s = { + inherit tokenFile; + enable = true; + role = "server"; + package = k3s; + clusterInit = true; + extraFlags = builtins.toString [ + "--disable" + "coredns" + "--disable" + "local-storage" + "--disable" + "metrics-server" + "--disable" + "servicelb" + "--disable" + "traefik" + "--node-ip" + "192.168.1.1" + "--pause-image" + "test.local/pause:local" + ]; + }; + networking.firewall.allowedTCPPorts = [ + 2379 + 2380 + 6443 + ]; + networking.firewall.allowedUDPPorts = [ 8472 ]; + networking.firewall.trustedInterfaces = [ "flannel.1" ]; + networking.useDHCP = false; + networking.defaultGateway = "192.168.1.1"; + networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { + address = "192.168.1.1"; + prefixLength = 24; + } ]; }; - networking.firewall.allowedTCPPorts = [ 2379 2380 6443 ]; - networking.firewall.allowedUDPPorts = [ 8472 ]; - networking.firewall.trustedInterfaces = [ "flannel.1" ]; - networking.useDHCP = false; - networking.defaultGateway = "192.168.1.1"; - networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ - { address = "192.168.1.1"; prefixLength = 24; } - ]; - }; - - server2 = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ gzip jq ]; - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s = { - inherit tokenFile; - enable = true; - serverAddr = "https://192.168.1.1:6443"; - clusterInit = false; - extraFlags = builtins.toString [ - "--disable" "coredns" - "--disable" "local-storage" - "--disable" "metrics-server" - "--disable" "servicelb" - "--disable" "traefik" - "--node-ip" "192.168.1.3" - "--pause-image" "test.local/pause:local" + + server2 = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + gzip + jq + ]; + virtualisation.memorySize = 1536; + virtualisation.diskSize = 4096; + + services.k3s = { + inherit tokenFile; + enable = true; + serverAddr = "https://192.168.1.1:6443"; + clusterInit = false; + extraFlags = builtins.toString [ + "--disable" + "coredns" + "--disable" + "local-storage" + "--disable" + "metrics-server" + "--disable" + "servicelb" + "--disable" + "traefik" + "--node-ip" + "192.168.1.3" + "--pause-image" + "test.local/pause:local" + ]; + }; + networking.firewall.allowedTCPPorts = [ + 2379 + 2380 + 6443 + ]; + networking.firewall.allowedUDPPorts = [ 8472 ]; + networking.firewall.trustedInterfaces = [ "flannel.1" ]; + networking.useDHCP = false; + networking.defaultGateway = "192.168.1.3"; + networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { + address = "192.168.1.3"; + prefixLength = 24; + } ]; }; - networking.firewall.allowedTCPPorts = [ 2379 2380 6443 ]; - networking.firewall.allowedUDPPorts = [ 8472 ]; - networking.firewall.trustedInterfaces = [ "flannel.1" ]; - networking.useDHCP = false; - networking.defaultGateway = "192.168.1.3"; - networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ - { address = "192.168.1.3"; prefixLength = 24; } - ]; - }; - - agent = { pkgs, ... }: { - virtualisation.memorySize = 1024; - virtualisation.diskSize = 2048; - services.k3s = { - inherit tokenFile; - enable = true; - role = "agent"; - serverAddr = "https://192.168.1.3:6443"; - extraFlags = lib.concatStringsSep " " [ - "--pause-image" "test.local/pause:local" - "--node-ip" "192.168.1.2" + + agent = + { pkgs, ... }: + { + virtualisation.memorySize = 1024; + virtualisation.diskSize = 2048; + services.k3s = { + inherit tokenFile; + enable = true; + role = "agent"; + serverAddr = "https://192.168.1.3:6443"; + extraFlags = lib.concatStringsSep " " [ + "--pause-image" + "test.local/pause:local" + "--node-ip" + "192.168.1.2" + ]; + }; + networking.firewall.allowedTCPPorts = [ 6443 ]; + networking.firewall.allowedUDPPorts = [ 8472 ]; + networking.firewall.trustedInterfaces = [ "flannel.1" ]; + networking.useDHCP = false; + networking.defaultGateway = "192.168.1.2"; + networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { + address = "192.168.1.2"; + prefixLength = 24; + } ]; }; - networking.firewall.allowedTCPPorts = [ 6443 ]; - networking.firewall.allowedUDPPorts = [ 8472 ]; - networking.firewall.trustedInterfaces = [ "flannel.1" ]; - networking.useDHCP = false; - networking.defaultGateway = "192.168.1.2"; - networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ - { address = "192.168.1.2"; prefixLength = 24; } - ]; - }; }; meta.maintainers = k3s.meta.maintainers; @@ -178,4 +239,5 @@ import ../make-test-python.nix ({ pkgs, lib, k3s, ... }: for m in machines: m.shutdown() ''; - }) + } +) diff --git a/nixos/tests/k3s/single-node.nix b/nixos/tests/k3s/single-node.nix index fd64a050e61ef..80d80a55ddf41 100644 --- a/nixos/tests/k3s/single-node.nix +++ b/nixos/tests/k3s/single-node.nix @@ -1,14 +1,29 @@ -import ../make-test-python.nix ({ pkgs, lib, k3s, ... }: +import ../make-test-python.nix ( + { + pkgs, + lib, + k3s, + ... + }: let imageEnv = pkgs.buildEnv { name = "k3s-pause-image-env"; - paths = with pkgs; [ tini (hiPrio coreutils) busybox ]; + paths = with pkgs; [ + tini + (hiPrio coreutils) + busybox + ]; }; pauseImage = pkgs.dockerTools.streamLayeredImage { name = "test.local/pause"; tag = "local"; contents = imageEnv; - config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ]; + config.Entrypoint = [ + "/bin/tini" + "--" + "/bin/sleep" + "inf" + ]; }; testPodYaml = pkgs.writeText "test.yml" '' apiVersion: v1 @@ -27,57 +42,83 @@ import ../make-test-python.nix ({ pkgs, lib, k3s, ... }: name = "${k3s.name}-single-node"; meta.maintainers = k3s.meta.maintainers; - nodes.machine = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ k3s gzip ]; + nodes.machine = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + k3s + gzip + ]; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + # k3s uses enough resources the default vm fails. + virtualisation.memorySize = 1536; + virtualisation.diskSize = 4096; - services.k3s.enable = true; - services.k3s.role = "server"; - services.k3s.package = k3s; - # Slightly reduce resource usage - services.k3s.extraFlags = builtins.toString [ - "--disable" "coredns" - "--disable" "local-storage" - "--disable" "metrics-server" - "--disable" "servicelb" - "--disable" "traefik" - "--pause-image" "test.local/pause:local" - ]; + services.k3s.enable = true; + services.k3s.role = "server"; + services.k3s.package = k3s; + # Slightly reduce resource usage + services.k3s.extraFlags = builtins.toString [ + "--disable" + "coredns" + "--disable" + "local-storage" + "--disable" + "metrics-server" + "--disable" + "servicelb" + "--disable" + "traefik" + "--pause-image" + "test.local/pause:local" + ]; - users.users = { - noprivs = { - isNormalUser = true; - description = "Can't access k3s by default"; - password = "*"; + users.users = { + noprivs = { + isNormalUser = true; + description = "Can't access k3s by default"; + password = "*"; + }; }; }; - }; - testScript = '' - start_all() + testScript = + '' + start_all() - machine.wait_for_unit("k3s") - machine.succeed("kubectl cluster-info") - machine.fail("sudo -u noprivs kubectl cluster-info") + machine.wait_for_unit("k3s") + machine.succeed("kubectl cluster-info") + machine.fail("sudo -u noprivs kubectl cluster-info") '' # Fix-Me: Tests fail for 'aarch64-linux' as: "CONFIG_CGROUP_FREEZER: missing (fail)" - + lib.optionalString (!pkgs.stdenv.isAarch64) ''machine.succeed("k3s check-config")'' + '' + + lib.optionalString (!pkgs.stdenv.isAarch64) ''machine.succeed("k3s check-config")'' + + '' - machine.succeed( - "${pauseImage} | ctr image import -" - ) + machine.succeed( + "${pauseImage} | ctr image import -" + ) - # Also wait for our service account to show up; it takes a sec - machine.wait_until_succeeds("kubectl get serviceaccount default") - machine.succeed("kubectl apply -f ${testPodYaml}") - machine.succeed("kubectl wait --for 'condition=Ready' pod/test") - machine.succeed("kubectl delete -f ${testPodYaml}") + # Also wait for our service account to show up; it takes a sec + machine.wait_until_succeeds("kubectl get serviceaccount default") + machine.succeed("kubectl apply -f ${testPodYaml}") + machine.succeed("kubectl wait --for 'condition=Ready' pod/test") + machine.succeed("kubectl delete -f ${testPodYaml}") - # regression test for #176445 - machine.fail("journalctl -o cat -u k3s.service | grep 'ipset utility not found'") + # regression test for #176445 + machine.fail("journalctl -o cat -u k3s.service | grep 'ipset utility not found'") - machine.shutdown() - ''; - }) + with subtest("Run k3s-killall"): + # Call the killall script with a clean path to assert that + # all required commands are wrapped + output = machine.succeed("PATH= ${k3s}/bin/k3s-killall.sh 2>&1 | tee /dev/stderr") + assert "command not found" not in output, "killall script contains unknown command" + + # Check that killall cleaned up properly + machine.fail("systemctl is-active k3s.service") + machine.fail("systemctl list-units | grep containerd") + machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0") + machine.fail("ip netns show | grep cni-") + + machine.shutdown() + ''; + } +) |