nixos: kubernetes fixes

* Fix reference CNI plugins
  * The plugins were split out of the upstream cni repo around version
    0.6.0

* Fix RBAC and DNS tests
  * Fix broken apiVersion fields
  * Change plugin linking to look in ${package}/bin rather than
    ${package.plugins}

* Initial work towards a working e2e test
  * Test still fails, but at least the expression evaluates now

Continues @srhb's work in #37199

Fixes #37199

3 files changed, 55 insertions, 22 deletions

diff --git a/nixos/tests/kubernetes/certs.nix b/nixos/tests/kubernetes/certs.nix
index d3eff910c467f..520c728b65ee2 100644
--- a/nixos/tests/kubernetes/certs.nix
+++ b/nixos/tests/kubernetes/certs.nix
@@ -6,29 +6,62 @@
   kubelets
 }:
 let
-  runWithCFSSL = name: cmd:
-    builtins.fromJSON (builtins.readFile (
-      pkgs.runCommand "${name}-cfss.json" {
-        buildInputs = [ pkgs.cfssl ];
-      } "cfssl ${cmd} > $out"
-    ));
-
-  writeCFSSL = content:
-    pkgs.runCommand content.name {
-      buildInputs = [ pkgs.cfssl ];
-    } ''
-      mkdir -p $out
-      cd $out
-      cat ${writeFile content} | cfssljson -bare ${content.name}
-    '';
+   runWithCFSSL = name: cmd:
+     let secrets = pkgs.runCommand "${name}-cfss.json" {
+         buildInputs = [ pkgs.cfssl pkgs.jq ];
+         outputs = [ "out" "cert" "key" "csr" ];
+       }
+       ''
+         (
+           echo "${cmd}"
+           cfssl ${cmd} > tmp
+           cat tmp | jq -r .key > $key
+           cat tmp | jq -r .cert > $cert
+           cat tmp | jq -r .csr > $csr
+
+           touch $out
+         ) 2>&1 | fold -w 80 -s
+       '';
+     in {
+       key = secrets.key;
+       cert = secrets.cert;
+       csr = secrets.csr;
+     };
+
+   writeCFSSL = content:
+     pkgs.runCommand content.name {
+      buildInputs = [ pkgs.cfssl pkgs.jq ];
+     } ''
+       mkdir -p $out
+       cd $out
+
+       json=${pkgs.lib.escapeShellArg (builtins.toJSON content)}
+
+       # for a given $field in the $json, treat the associated value as a
+       # file path and substitute the contents thereof into the $json
+       # object.
+       expandFileField() {
+         local field=$1
+         if jq -e --arg field "$field" 'has($field)'; then
+           local path="$(echo "$json" | jq -r ".$field")"
+           json="$(echo "$json" | jq --arg val "$(cat "$path")" ".$field = \$val")"
+         fi
+       }
+
+       expandFileField key
+       expandFileField ca
+       expandFileField cert
+
+       echo "$json" | cfssljson -bare ${content.name}
+     '';
 
   noCSR = content: pkgs.lib.filterAttrs (n: v: n != "csr") content;
   noKey = content: pkgs.lib.filterAttrs (n: v: n != "key") content;
 
-  writeFile = content: pkgs.writeText "content" (
-    if pkgs.lib.isAttrs content then builtins.toJSON content
-    else toString content
-  );
+  writeFile = content:
+    if pkgs.lib.isDerivation content
+    then content
+    else pkgs.writeText "content" (builtins.toJSON content);
 
   createServingCertKey = { ca, cn, hosts? [], size ? 2048, name ? cn }:
     noCSR (
diff --git a/nixos/tests/kubernetes/e2e.nix b/nixos/tests/kubernetes/e2e.nix
index d9d7ba9bb2cc5..175d8413045ed 100644
--- a/nixos/tests/kubernetes/e2e.nix
+++ b/nixos/tests/kubernetes/e2e.nix
@@ -2,7 +2,7 @@
 with import ./base.nix { inherit system; };
 let
   domain = "my.zyx";
-  certs = import ./certs.nix { externalDomain = domain; };
+  certs = import ./certs.nix { externalDomain = domain; kubelets = ["machine1" "machine2"]; };
   kubeconfig = pkgs.writeText "kubeconfig.json" (builtins.toJSON {
     apiVersion = "v1";
     kind = "Config";
diff --git a/nixos/tests/kubernetes/rbac.nix b/nixos/tests/kubernetes/rbac.nix
index 1966fed3a5fb9..226808c4b2632 100644
--- a/nixos/tests/kubernetes/rbac.nix
+++ b/nixos/tests/kubernetes/rbac.nix
@@ -12,7 +12,7 @@ let
   });
 
   roRoleBinding = pkgs.writeText "ro-role-binding.json" (builtins.toJSON {
-    apiVersion = "rbac.authorization.k8s.io/v1beta1";
+    apiVersion = "rbac.authorization.k8s.io/v1";
     kind = "RoleBinding";
     metadata = {
       name = "read-pods";
@@ -31,7 +31,7 @@ let
   });
 
   roRole = pkgs.writeText "ro-role.json" (builtins.toJSON {
-    apiVersion = "rbac.authorization.k8s.io/v1beta1";
+    apiVersion = "rbac.authorization.k8s.io/v1";
     kind = "Role";
     metadata = {
       name = "pod-reader";