integration test driver: Auto-generate integration test driver's machine

method documentation of nixos docs from python doc strings

6 files changed, 105 insertions, 216 deletions

diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix
index 40af4c1fa0b06..f2fd6a682934a 100644
--- a/nixos/doc/manual/default.nix
+++ b/nixos/doc/manual/default.nix
@@ -63,6 +63,9 @@ let
       optionIdPrefix = "test-opt-";
     };
 
+  testDriverMachineDocstrings = pkgs.callPackage
+    ../../../nixos/lib/test-driver/nixos-test-driver-docstrings.nix {};
+
   prepareManualFromMD = ''
     cp -r --no-preserve=all $inputs/* .
 
@@ -80,6 +83,8 @@ let
       --replace \
         '@NIXOS_TEST_OPTIONS_JSON@' \
         ${testOptionsDoc.optionsJSON}/share/doc/nixos/options.json
+    sed -e '/@PYTHON_MACHINE_METHODS@/ {' -e 'r ${testDriverMachineDocstrings}/machine-methods.md' -e 'd' -e '}' \
+      -i ./development/writing-nixos-tests.section.md
   '';
 
 in rec {
diff --git a/nixos/doc/manual/development/writing-nixos-tests.section.md b/nixos/doc/manual/development/writing-nixos-tests.section.md
index 486a4b64a262f..84b247fd2042f 100644
--- a/nixos/doc/manual/development/writing-nixos-tests.section.md
+++ b/nixos/doc/manual/development/writing-nixos-tests.section.md
@@ -139,210 +139,7 @@ to Python as `machine_a`.
 
 The following methods are available on machine objects:
 
-`start`
-
-:   Start the virtual machine. This method is asynchronous --- it does
-    not wait for the machine to finish booting.
-
-`shutdown`
-
-:   Shut down the machine, waiting for the VM to exit.
-
-`crash`
-
-:   Simulate a sudden power failure, by telling the VM to exit
-    immediately.
-
-`block`
-
-:   Simulate unplugging the Ethernet cable that connects the machine to
-    the other machines.
-
-`unblock`
-
-:   Undo the effect of `block`.
-
-`screenshot`
-
-:   Take a picture of the display of the virtual machine, in PNG format.
-    The screenshot is linked from the HTML log.
-
-`get_screen_text_variants`
-
-:   Return a list of different interpretations of what is currently
-    visible on the machine's screen using optical character
-    recognition. The number and order of the interpretations is not
-    specified and is subject to change, but if no exception is raised at
-    least one will be returned.
-
-    ::: {.note}
-    This requires [`enableOCR`](#test-opt-enableOCR) to be set to `true`.
-    :::
-
-`get_screen_text`
-
-:   Return a textual representation of what is currently visible on the
-    machine's screen using optical character recognition.
-
-    ::: {.note}
-    This requires [`enableOCR`](#test-opt-enableOCR) to be set to `true`.
-    :::
-
-`send_monitor_command`
-
-:   Send a command to the QEMU monitor. This is rarely used, but allows
-    doing stuff such as attaching virtual USB disks to a running
-    machine.
-
-`send_key`
-
-:   Simulate pressing keys on the virtual keyboard, e.g.,
-    `send_key("ctrl-alt-delete")`.
-
-`send_chars`
-
-:   Simulate typing a sequence of characters on the virtual keyboard,
-    e.g., `send_chars("foobar\n")` will type the string `foobar`
-    followed by the Enter key.
-
-`send_console`
-
-:   Send keys to the kernel console. This allows interaction with the systemd
-    emergency mode, for example. Takes a string that is sent, e.g.,
-    `send_console("\n\nsystemctl default\n")`.
-
-`execute`
-
-:   Execute a shell command, returning a list `(status, stdout)`.
-
-    Commands are run with `set -euo pipefail` set:
-
-    -   If several commands are separated by `;` and one fails, the
-        command as a whole will fail.
-
-    -   For pipelines, the last non-zero exit status will be returned
-        (if there is one; otherwise zero will be returned).
-
-    -   Dereferencing unset variables fails the command.
-
-    -   It will wait for stdout to be closed.
-
-    If the command detaches, it must close stdout, as `execute` will wait
-    for this to consume all output reliably. This can be achieved by
-    redirecting stdout to stderr `>&2`, to `/dev/console`, `/dev/null` or
-    a file. Examples of detaching commands are `sleep 365d &`, where the
-    shell forks a new process that can write to stdout and `xclip -i`, where
-    the `xclip` command itself forks without closing stdout.
-
-    Takes an optional parameter `check_return` that defaults to `True`.
-    Setting this parameter to `False` will not check for the return code
-    and return -1 instead. This can be used for commands that shut down
-    the VM and would therefore break the pipe that would be used for
-    retrieving the return code.
-
-    A timeout for the command can be specified (in seconds) using the optional
-    `timeout` parameter, e.g., `execute(cmd, timeout=10)` or
-    `execute(cmd, timeout=None)`. The default is 900 seconds.
-
-`succeed`
-
-:   Execute a shell command, raising an exception if the exit status is
-    not zero, otherwise returning the standard output. Similar to `execute`,
-    except that the timeout is `None` by default. See `execute` for details on
-    command execution.
-
-`fail`
-
-:   Like `succeed`, but raising an exception if the command returns a zero
-    status.
-
-`wait_until_succeeds`
-
-:   Repeat a shell command with 1-second intervals until it succeeds.
-    Has a default timeout of 900 seconds which can be modified, e.g.
-    `wait_until_succeeds(cmd, timeout=10)`. See `execute` for details on
-    command execution.
-
-`wait_until_fails`
-
-:   Like `wait_until_succeeds`, but repeating the command until it fails.
-
-`wait_for_unit`
-
-:   Wait until the specified systemd unit has reached the "active"
-    state.
-
-`wait_for_file`
-
-:   Wait until the specified file exists.
-
-`wait_for_open_port`
-
-:   Wait until a process is listening on the given TCP port and IP address
-    (default `localhost`).
-
-`wait_for_closed_port`
-
-:   Wait until nobody is listening on the given TCP port and IP address
-    (default `localhost`).
-
-`wait_for_x`
-
-:   Wait until the X11 server is accepting connections.
-
-`wait_for_text`
-
-:   Wait until the supplied regular expressions matches the textual
-    contents of the screen by using optical character recognition (see
-    `get_screen_text` and `get_screen_text_variants`).
-
-    ::: {.note}
-    This requires [`enableOCR`](#test-opt-enableOCR) to be set to `true`.
-    :::
-
-`wait_for_console_text`
-
-:   Wait until the supplied regular expressions match a line of the
-    serial console output. This method is useful when OCR is not
-    possible or accurate enough.
-
-`wait_for_window`
-
-:   Wait until an X11 window has appeared whose name matches the given
-    regular expression, e.g., `wait_for_window("Terminal")`.
-
-`copy_from_host`
-
-:   Copies a file from host to machine, e.g.,
-    `copy_from_host("myfile", "/etc/my/important/file")`.
-
-    The first argument is the file on the host. The file needs to be
-    accessible while building the nix derivation. The second argument is
-    the location of the file on the machine.
-
-`systemctl`
-
-:   Runs `systemctl` commands with optional support for
-    `systemctl --user`
-
-    ```py
-    machine.systemctl("list-jobs --no-pager") # runs `systemctl list-jobs --no-pager`
-    machine.systemctl("list-jobs --no-pager", "any-user") # spawns a shell for `any-user` and runs `systemctl --user list-jobs --no-pager`
-    ```
-
-`shell_interact`
-
-:   Allows you to directly interact with the guest shell. This should
-    only be used during test development, not in production tests.
-    Killing the interactive session with `Ctrl-d` or `Ctrl-c` also ends
-    the guest session.
-
-`console_interact`
-
-:   Allows you to directly interact with QEMU's stdin. This should
-    only be used during test development, not in production tests.
-    Output from QEMU is only read line-wise. `Ctrl-c` kills QEMU and
-    `Ctrl-d` closes console and returns to the test runner.
+@PYTHON_MACHINE_METHODS@
 
 To test user units declared by `systemd.user.services` the optional
 `user` argument can be used:
diff --git a/nixos/lib/test-driver/extract-docstrings.py b/nixos/lib/test-driver/extract-docstrings.py
new file mode 100644
index 0000000000000..5aec4c89a9d74
--- /dev/null
+++ b/nixos/lib/test-driver/extract-docstrings.py
@@ -0,0 +1,66 @@
+import ast
+import sys
+
+"""
+This program takes all the Machine class methods and prints its methods in
+markdown-style. These can then be included in the NixOS test driver
+markdown style, assuming the docstrings themselves are also in markdown.
+
+These are included in the test driver documentation in the NixOS manual.
+See https://nixos.org/manual/nixos/stable/#ssec-machine-objects
+
+The python input looks like this:
+
+```py
+...
+
+class Machine(...):
+    ...
+
+    def some_function(self, param1, param2):
+        ""
+        documentation string of some_function.
+        foo bar baz.
+        ""
+        ...
+```
+
+Output will be:
+
+```markdown
+...
+
+some_function(param1, param2)
+
+:   documentation string of some_function.
+    foo bar baz.
+
+...
+```
+
+"""
+
+assert len(sys.argv) == 2
+
+with open(sys.argv[1], "r") as f:
+    module = ast.parse(f.read())
+
+class_definitions = (node for node in module.body if isinstance(node, ast.ClassDef))
+
+machine_class = next(filter(lambda x: x.name == "Machine", class_definitions))
+assert machine_class is not None
+
+function_definitions = [
+    node for node in machine_class.body if isinstance(node, ast.FunctionDef)
+]
+function_definitions.sort(key=lambda x: x.name)
+
+for f in function_definitions:
+    docstr = ast.get_docstring(f)
+    if docstr is not None:
+        args = ", ".join((a.arg for a in f.args.args[1:]))
+        args = f"({args})"
+
+        docstr = "\n".join((f"    {l}" for l in docstr.strip().splitlines()))
+
+        print(f"{f.name}{args}\n\n:{docstr[1:]}\n")
diff --git a/nixos/lib/test-driver/nixos-test-driver-docstrings.nix b/nixos/lib/test-driver/nixos-test-driver-docstrings.nix
new file mode 100644
index 0000000000000..a3ef50e4e8205
--- /dev/null
+++ b/nixos/lib/test-driver/nixos-test-driver-docstrings.nix
@@ -0,0 +1,13 @@
+{ runCommand
+, python3
+}:
+
+let
+  env = { nativeBuildInputs = [ python3 ]; };
+in
+
+runCommand "nixos-test-driver-docstrings" env ''
+  mkdir $out
+  python3 ${./extract-docstrings.py} ${./test_driver/machine.py} \
+    > $out/machine-methods.md
+''
diff --git a/nixos/lib/test-driver/test_driver/machine.py b/nixos/lib/test-driver/test_driver/machine.py
index 8a5eebdc820b6..dbfe256011ee4 100644
--- a/nixos/lib/test-driver/test_driver/machine.py
+++ b/nixos/lib/test-driver/test_driver/machine.py
@@ -417,9 +417,8 @@ class Machine:
 
     def send_monitor_command(self, command: str) -> str:
         """
-        Send a command to the QEMU monitor. This is rarely used, but allows
-        doing stuff such as attaching virtual USB disks to a running
-        machine.
+        Send a command to the QEMU monitor. This allows attaching
+        virtual USB disks to a running machine, among other things.
         """
         self.run_callbacks()
         message = f"{command}\n".encode()
@@ -630,9 +629,10 @@ class Machine:
 
     def console_interact(self) -> None:
         """
-        Allows you to directly interact with QEMU's stdin.
-        This should only be used during test development, not in production
-        tests.
+        Allows you to directly interact with QEMU's stdin, by forwarding
+        terminal input to the QEMU process.
+        This is for use with the interactive test driver, not for production
+        tests, which run unattended.
         Output from QEMU is only read line-wise. `Ctrl-c` kills QEMU and
         `Ctrl-d` closes console and returns to the test runner.
         """
@@ -885,12 +885,17 @@ class Machine:
         Copies a file from host to machine, e.g.,
         `copy_from_host("myfile", "/etc/my/important/file")`.
 
-        The first argument is the file on the host. The file needs to be
-        accessible while building the nix derivation. The second argument is
-        the location of the file on the machine.
+        The first argument is the file on the host. Note that the "host" refers
+        to the environment in which the test driver runs, which is typically the
+        Nix build sandbox.
+
+        The second argument is the location of the file on the machine that will
+        be written to.
 
         The file is copied via the `shared_dir` directory which is shared among
         all the VMs (using a temporary directory).
+        The access rights bits will mimic the ones from the host file and
+        user:group will be root:root.
         """
         host_src = Path(source)
         vm_target = Path(target)
@@ -995,7 +1000,7 @@ class Machine:
         """
         Wait until the supplied regular expressions match a line of the
         serial console output.
-        This method is useful when OCR is not possible or accurate enough.
+        This method is useful when OCR is not possible or inaccurate.
         """
         # Buffer the console output, this is needed
         # to match multiline regexes.
@@ -1026,6 +1031,9 @@ class Machine:
         """
         Simulate pressing keys on the virtual keyboard, e.g.,
         `send_key("ctrl-alt-delete")`.
+
+        Please also refer to the QEMU documentation for more information on the
+        input syntax: https://en.wikibooks.org/wiki/QEMU/Monitor#sendkey_keys
         """
         key = CHAR_TO_KEY.get(key, key)
         context = self.nested(f"sending key {repr(key)}") if log else nullcontext()
diff --git a/nixos/lib/testing/driver.nix b/nixos/lib/testing/driver.nix
index 5eb62d0b32c8b..23574698c0629 100644
--- a/nixos/lib/testing/driver.nix
+++ b/nixos/lib/testing/driver.nix
@@ -66,7 +66,7 @@ let
           echo -n "$testScript" >> testScriptWithTypes
 
           echo "Running type check (enable/disable: config.skipTypeCheck)"
-          echo "See https://nixos.org/manual/nixos/stable/#sec-test-options-reference"
+          echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipTypeCheck"
 
           mypy  --no-implicit-optional \
                 --pretty \
@@ -81,7 +81,7 @@ let
         ${testDriver}/bin/generate-driver-symbols
         ${lib.optionalString (!config.skipLint) ''
           echo "Linting test script (enable/disable: config.skipLint)"
-          echo "See https://nixos.org/manual/nixos/stable/#sec-test-options-reference"
+          echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipLint"
 
           PYFLAKES_BUILTINS="$(
             echo -n ${lib.escapeShellArg (lib.concatStringsSep "," pythonizedNames)},