nixos/tests/custom-ca: Split

Run each browser check as a separate NixOS test.

This fixes a problem in which one browser starts up before the previous
browser is finished exiting, exhausting a resource and causing a
spurious test failure.

As a bonus, splitting the test
  * Gives more signal about exactly what's broken in the pass/fail status,
  * Makes it easier to quickly diagnose test failures,
  * Makes development iteration faster,
  * Allows concurrent test execution, which makes the test finish sooner
    when parallel builds are enabled.
  * Would allow each browser's test to be included in its nixpkgs
    passthru.tests, if desired (not done in this commit).

Reviewed-by: rnhmjoj <rnhmjoj@inventati.org>

1 files changed, 102 insertions, 90 deletions

diff --git a/nixos/tests/custom-ca.nix b/nixos/tests/custom-ca.nix
index 60c6c82223a93..91d80dc0dea16 100644
--- a/nixos/tests/custom-ca.nix
+++ b/nixos/tests/custom-ca.nix
@@ -1,9 +1,14 @@
 # Checks that `security.pki` options are working in curl and the main browser
-# engines: Gecko (via Firefox), Chromium, QtWebEngine (Falkon) and WebKitGTK
-# (via Midori). The test checks that certificates issued by a custom trusted
-# CA are accepted but those from an unknown CA are rejected.
+# engines: Gecko (via Firefox), Chromium, QtWebEngine (via qutebrowser) and
+# WebKitGTK (via Midori). The test checks that certificates issued by a custom
+# trusted CA are accepted but those from an unknown CA are rejected.
 
-import ./make-test-python.nix ({ pkgs, lib, ... }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
+
+with import ../lib/testing-python.nix { inherit system pkgs; };
 
 let
   makeCert = { caName, domain }: pkgs.runCommand "example-cert"
@@ -68,24 +73,8 @@ let
       domain = "bad.example.com";
     };
 
-in
-
-{
-  name = "custom-ca";
-  meta.maintainers = with lib.maintainers; [ rnhmjoj ];
-
-  enableOCR = true;
-
-  nodes.machine = { pkgs, ... }:
-    { imports = [ ./common/user-account.nix ./common/x11.nix ];
-
-      # chromium-based browsers refuse to run as root
-      test-support.displayManager.auto.user = "alice";
-
-      # browsers may hang with the default memory
-      virtualisation.memorySize = 600;
-
-      networking.hosts."127.0.0.1" = [ "good.example.com" "bad.example.com" ];
+  webserverConfig =
+    { networking.hosts."127.0.0.1" = [ "good.example.com" "bad.example.com" ];
       security.pki.certificateFiles = [ "${example-good-cert}/ca.crt" ];
 
       services.nginx.enable = true;
@@ -107,73 +96,96 @@ in
             return 200 'It does not work!';
           '';
         };
-
-      environment.systemPackages = with pkgs; [
-        xdotool
-        firefox
-        chromium
-        qutebrowser
-        midori
-      ];
     };
 
-  testScript = ''
-    from typing import Tuple
-    def execute_as(user: str, cmd: str) -> Tuple[int, str]:
-        """
-        Run a shell command as a specific user.
-        """
-        return machine.execute(f"sudo -u {user} {cmd}")
-
-
-    def wait_for_window_as(user: str, cls: str) -> None:
-        """
-        Wait until a X11 window of a given user appears.
-        """
-
-        def window_is_visible(last_try: bool) -> bool:
-            ret, stdout = execute_as(user, f"xdotool search --onlyvisible --class {cls}")
-            if last_try:
-                machine.log(f"Last chance to match {cls} on the window list")
-            return ret == 0
-
-        with machine.nested("Waiting for a window to appear"):
-            retry(window_is_visible)
-
-
-    machine.start()
-
-    with subtest("Good certificate is trusted in curl"):
-        machine.wait_for_unit("nginx")
-        machine.wait_for_open_port(443)
-        machine.succeed("curl -fv https://good.example.com")
-
-    with subtest("Unknown CA is untrusted in curl"):
-        machine.fail("curl -fv https://bad.example.com")
-
-    browsers = {
-      "firefox": "Security Risk",
-      "chromium": "not private",
-      "qutebrowser -T": "Certificate error",
-      "midori": "Security"
-    }
-
-    machine.wait_for_x()
-    for command, error in browsers.items():
-        browser = command.split()[0]
-        with subtest("Good certificate is trusted in " + browser):
-            execute_as(
-                "alice", f"{command} https://good.example.com >&2 &"
-            )
-            wait_for_window_as("alice", browser)
-            machine.wait_for_text("It works!")
-            machine.screenshot("good" + browser)
-            execute_as("alice", "xdotool key ctrl+w")  # close tab
-
-        with subtest("Unknown CA is untrusted in " + browser):
-            execute_as("alice", f"{command} https://bad.example.com >&2 &")
-            machine.wait_for_text(error)
-            machine.screenshot("bad" + browser)
-            machine.succeed("pkill -f " + browser)
-  '';
-})
+  curlTest = makeTest {
+    name = "custom-ca-curl";
+    meta.maintainers = with lib.maintainers; [ rnhmjoj ];
+    nodes.machine = { ... }: webserverConfig;
+    testScript = ''
+        with subtest("Good certificate is trusted in curl"):
+            machine.wait_for_unit("nginx")
+            machine.wait_for_open_port(443)
+            machine.succeed("curl -fv https://good.example.com")
+
+        with subtest("Unknown CA is untrusted in curl"):
+            machine.fail("curl -fv https://bad.example.com")
+    '';
+  };
+
+  mkBrowserTest = browser: testParams: makeTest {
+    name = "custom-ca-${browser}";
+    meta.maintainers = with lib.maintainers; [ rnhmjoj ];
+
+    enableOCR = true;
+
+    nodes.machine = { pkgs, ... }:
+      { imports =
+          [ ./common/user-account.nix
+            ./common/x11.nix
+            webserverConfig
+          ];
+
+        # chromium-based browsers refuse to run as root
+        test-support.displayManager.auto.user = "alice";
+
+        # browsers may hang with the default memory
+        virtualisation.memorySize = 600;
+
+        environment.systemPackages = [ pkgs.xdotool pkgs.${browser} ];
+      };
+
+    testScript = ''
+      from typing import Tuple
+      def execute_as(user: str, cmd: str) -> Tuple[int, str]:
+          """
+          Run a shell command as a specific user.
+          """
+          return machine.execute(f"sudo -u {user} {cmd}")
+
+
+      def wait_for_window_as(user: str, cls: str) -> None:
+          """
+          Wait until a X11 window of a given user appears.
+          """
+
+          def window_is_visible(last_try: bool) -> bool:
+              ret, stdout = execute_as(user, f"xdotool search --onlyvisible --class {cls}")
+              if last_try:
+                  machine.log(f"Last chance to match {cls} on the window list")
+              return ret == 0
+
+          with machine.nested("Waiting for a window to appear"):
+              retry(window_is_visible)
+
+
+      machine.start()
+      machine.wait_for_x()
+
+      command = "${browser} ${testParams.args or ""}"
+      with subtest("Good certificate is trusted in ${browser}"):
+          execute_as(
+              "alice", f"{command} https://good.example.com >&2 &"
+          )
+          wait_for_window_as("alice", "${browser}")
+          machine.wait_for_text("It works!")
+          machine.screenshot("good${browser}")
+          execute_as("alice", "xdotool key ctrl+w")  # close tab
+
+      with subtest("Unknown CA is untrusted in ${browser}"):
+          execute_as("alice", f"{command} https://bad.example.com >&2 &")
+          machine.wait_for_text("${testParams.error}")
+          machine.screenshot("bad${browser}")
+    '';
+  };
+
+in
+
+{
+  curl = curlTest;
+} // pkgs.lib.mapAttrs mkBrowserTest {
+  firefox = { error = "Security Risk"; };
+  chromium = { error = "not private"; };
+  qutebrowser = { args = "-T"; error = "Certificate error"; };
+  midori = { error = "Security"; };
+}