diff options
| -rw-r--r-- | nixos/doc/manual/development/writing-nixos-tests.section.md | 73 | ||||
| -rw-r--r-- | nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml | 217 | ||||
| -rw-r--r-- | nixos/lib/default.nix | 8 | ||||
| -rw-r--r-- | nixos/lib/testing-python.nix | 33 | ||||
| -rw-r--r-- | nixos/lib/testing/default.nix | 24 |
5 files changed, 257 insertions, 98 deletions
diff --git a/nixos/doc/manual/development/writing-nixos-tests.section.md b/nixos/doc/manual/development/writing-nixos-tests.section.md index 6934bb0face76..8dd3e6fb7597f 100644 --- a/nixos/doc/manual/development/writing-nixos-tests.section.md +++ b/nixos/doc/manual/development/writing-nixos-tests.section.md @@ -1,9 +1,9 @@ # Writing Tests {#sec-writing-nixos-tests} -A NixOS test is a Nix expression that has the following structure: +A NixOS test is a module that has the following structure: ```nix -import ./make-test-python.nix { +{ # One or more machines: nodes = @@ -21,7 +21,10 @@ import ./make-test-python.nix { } ``` -The attribute `testScript` is a bit of Python code that executes the +We refer to the whole test above as a test module, whereas the values +in `nodes.<name>` are NixOS modules. (A NixOS configuration is a module.) + +The option `testScript` is a bit of Python code that executes the test (described below). During the test, it will start one or more virtual machines, the configuration of which is described by the attribute `nodes`. @@ -34,7 +37,64 @@ when switching between consoles, and so on. An interesting multi-node test is [`nfs/simple.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/nfs/simple.nix). It uses two client nodes to test correct locking across server crashes. -There are a few special NixOS configuration options for test VMs: +## Calling a test {#sec-calling-nixos-tests} + +Tests are invoked a bit differently depending on whether the test lives in NixOS or in another project. + +### Testing within NixOS {#sec-call-nixos-test-in-nixos} + +Test modules can be instantiated into derivations in multiple ways. + +Tests that are part of NixOS are added to [`nixos/tests/all-tests.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/all-tests.nix). + +```nix + hostname = runTest ./hostname.nix; +``` + +Overrides can be added by defining an anonymous module in `all-tests.nix`. +For the purpose of constructing a test matrix, use the `matrix` options instead. + +```nix + hostname = runTest { imports = [ ./hostname.nix ]; defaults.networking.firewall.enable = false; }; +``` + +You can run a test with attribute name `mytest` in `all-tests.nix` by invoking: + +```shell +nix-build -A nixosTests.mytest +``` + +### Testing outside the NixOS project {#sec-call-nixos-test-outside-nixos} + +Outside the `nixpkgs` repository, you can instantiate the test by first acquiring the NixOS library, + +```nix +# regular nix +let nixos-lib = import (nixpkgs + "/nixos/lib") { }; +in +``` + +```nix +# flake +let nixos-lib = nixpkgs.lib.nixos; +in +``` + +... and then invoking `runTest`, for example: + +```nix +nixos-lib.runTest { + imports = [ ./test.nix ]; + hostPkgs = pkgs; # the Nixpkgs package set used outside the VMs + defaults.services.foo.package = mypkg; +} +``` + +`runTest` returns a derivation that runs the test. + +## Configuring the nodes {#sec-nixos-test-nodes} + +There are a few special NixOS options for test VMs: `virtualisation.memorySize` @@ -304,7 +364,7 @@ For faster dev cycles it\'s also possible to disable the code-linters (this shouldn\'t be commited though): ```nix -import ./make-test-python.nix { +{ skipLint = true; nodes.machine = { config, pkgs, ... }: @@ -336,7 +396,7 @@ Similarly, the type checking of test scripts can be disabled in the following way: ```nix -import ./make-test-python.nix { +{ skipTypeCheck = true; nodes.machine = { config, pkgs, ... }: @@ -400,7 +460,6 @@ added using the parameter `extraPythonPackages`. For example, you could add `numpy` like this: ```nix -import ./make-test-python.nix { extraPythonPackages = p: [ p.numpy ]; diff --git a/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml b/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml index d6f4f61c0645b..6d0465e4230bd 100644 --- a/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml +++ b/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml @@ -1,10 +1,10 @@ <section xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xml:id="sec-writing-nixos-tests"> <title>Writing Tests</title> <para> - A NixOS test is a Nix expression that has the following structure: + A NixOS test is a module that has the following structure: </para> <programlisting language="bash"> -import ./make-test-python.nix { +{ # One or more machines: nodes = @@ -22,7 +22,12 @@ import ./make-test-python.nix { } </programlisting> <para> - The attribute <literal>testScript</literal> is a bit of Python code + We refer to the whole test above as a test module, whereas the + values in <literal>nodes.<name></literal> are NixOS modules. + (A NixOS configuration is a module.) + </para> + <para> + The option <literal>testScript</literal> is a bit of Python code that executes the test (described below). During the test, it will start one or more virtual machines, the configuration of which is described by the attribute <literal>nodes</literal>. @@ -38,78 +43,149 @@ import ./make-test-python.nix { It uses two client nodes to test correct locking across server crashes. </para> - <para> - There are a few special NixOS configuration options for test VMs: - </para> - <variablelist> - <varlistentry> - <term> - <literal>virtualisation.memorySize</literal> - </term> - <listitem> - <para> - The memory of the VM in megabytes. - </para> - </listitem> - </varlistentry> - <varlistentry> - <term> - <literal>virtualisation.vlans</literal> - </term> - <listitem> - <para> - The virtual networks to which the VM is connected. See - <link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/nat.nix"><literal>nat.nix</literal></link> - for an example. - </para> - </listitem> - </varlistentry> - <varlistentry> - <term> - <literal>virtualisation.writableStore</literal> - </term> - <listitem> - <para> - By default, the Nix store in the VM is not writable. If you - enable this option, a writable union file system is mounted on - top of the Nix store to make it appear writable. This is - necessary for tests that run Nix operations that modify the - store. - </para> - </listitem> - </varlistentry> - </variablelist> - <para> - For more options, see the module - <link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/virtualisation/qemu-vm.nix"><literal>qemu-vm.nix</literal></link>. - </para> - <para> - The test script is a sequence of Python statements that perform - various actions, such as starting VMs, executing commands in the - VMs, and so on. Each virtual machine is represented as an object - stored in the variable <literal>name</literal> if this is also the - identifier of the machine in the declarative config. If you - specified a node <literal>nodes.machine</literal>, the following - example starts the machine, waits until it has finished booting, - then executes a command and checks that the output is more-or-less - correct: - </para> - <programlisting language="python"> + <section xml:id="sec-calling-nixos-tests"> + <title>Calling a test</title> + <para> + Tests are invoked a bit differently depending on whether the test + lives in NixOS or in another project. + </para> + <section xml:id="sec-call-nixos-test-in-nixos"> + <title>Testing within NixOS</title> + <para> + Test modules can be instantiated into derivations in multiple + ways. + </para> + <para> + Tests that are part of NixOS are added to + <link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/all-tests.nix"><literal>nixos/tests/all-tests.nix</literal></link>. + </para> + <programlisting language="bash"> + hostname = runTest ./hostname.nix; +</programlisting> + <para> + Overrides can be added by defining an anonymous module in + <literal>all-tests.nix</literal>. For the purpose of + constructing a test matrix, use the <literal>matrix</literal> + options instead. + </para> + <programlisting language="bash"> + hostname = runTest { imports = [ ./hostname.nix ]; defaults.networking.firewall.enable = false; }; +</programlisting> + <para> + You can run a test with attribute name <literal>mytest</literal> + in <literal>all-tests.nix</literal> by invoking: + </para> + <programlisting> +nix-build -A nixosTests.mytest +</programlisting> + </section> + <section xml:id="sec-call-nixos-test-outside-nixos"> + <title>Testing outside the NixOS project</title> + <para> + Outside the <literal>nixpkgs</literal> repository, you can + instantiate the test by first acquiring the NixOS library, + </para> + <programlisting language="bash"> +# regular nix +let nixos-lib = import (nixpkgs + "/nixos/lib") { }; +in +</programlisting> + <programlisting language="bash"> +# flake +let nixos-lib = nixpkgs.lib.nixos; +in +</programlisting> + <para> + … and then invoking <literal>runTest</literal>, for example: + </para> + <programlisting language="bash"> +nixos-lib.runTest { + imports = [ ./test.nix ]; + hostPkgs = pkgs; # the Nixpkgs package set used outside the VMs + defaults.services.foo.package = mypkg; +} +</programlisting> + <para> + <literal>runTest</literal> returns a derivation that runs the + test. + </para> + </section> + </section> + <section xml:id="sec-nixos-test-nodes"> + <title>Configuring the nodes</title> + <para> + There are a few special NixOS options for test VMs: + </para> + <variablelist> + <varlistentry> + <term> + <literal>virtualisation.memorySize</literal> + </term> + <listitem> + <para> + The memory of the VM in megabytes. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term> + <literal>virtualisation.vlans</literal> + </term> + <listitem> + <para> + The virtual networks to which the VM is connected. See + <link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/nat.nix"><literal>nat.nix</literal></link> + for an example. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term> + <literal>virtualisation.writableStore</literal> + </term> + <listitem> + <para> + By default, the Nix store in the VM is not writable. If you + enable this option, a writable union file system is mounted + on top of the Nix store to make it appear writable. This is + necessary for tests that run Nix operations that modify the + store. + </para> + </listitem> + </varlistentry> + </variablelist> + <para> + For more options, see the module + <link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/virtualisation/qemu-vm.nix"><literal>qemu-vm.nix</literal></link>. + </para> + <para> + The test script is a sequence of Python statements that perform + various actions, such as starting VMs, executing commands in the + VMs, and so on. Each virtual machine is represented as an object + stored in the variable <literal>name</literal> if this is also the + identifier of the machine in the declarative config. If you + specified a node <literal>nodes.machine</literal>, the following + example starts the machine, waits until it has finished booting, + then executes a command and checks that the output is more-or-less + correct: + </para> + <programlisting language="python"> machine.start() machine.wait_for_unit("default.target") if not "Linux" in machine.succeed("uname"): raise Exception("Wrong OS") </programlisting> - <para> - The first line is technically unnecessary; machines are implicitly - started when you first execute an action on them (such as - <literal>wait_for_unit</literal> or <literal>succeed</literal>). If - you have multiple machines, you can speed up the test by starting - them in parallel: - </para> - <programlisting language="python"> + <para> + The first line is technically unnecessary; machines are implicitly + started when you first execute an action on them (such as + <literal>wait_for_unit</literal> or <literal>succeed</literal>). + If you have multiple machines, you can speed up the test by + starting them in parallel: + </para> + <programlisting language="python"> start_all() </programlisting> + </section> <section xml:id="ssec-machine-objects"> <title>Machine objects</title> <para> @@ -563,7 +639,7 @@ machine.wait_for_unit("xautolock.service", "x-session-user") code-linters (this shouldn't be commited though): </para> <programlisting language="bash"> -import ./make-test-python.nix { +{ skipLint = true; nodes.machine = { config, pkgs, ... }: @@ -595,7 +671,7 @@ import ./make-test-python.nix { the following way: </para> <programlisting language="bash"> -import ./make-test-python.nix { +{ skipTypeCheck = true; nodes.machine = { config, pkgs, ... }: @@ -669,7 +745,6 @@ def foo_running(): <literal>numpy</literal> like this: </para> <programlisting language="bash"> -import ./make-test-python.nix { extraPythonPackages = p: [ p.numpy ]; diff --git a/nixos/lib/default.nix b/nixos/lib/default.nix index 2b3056e01457f..65d91342d4d15 100644 --- a/nixos/lib/default.nix +++ b/nixos/lib/default.nix @@ -21,6 +21,8 @@ let seqAttrsIf = cond: a: lib.mapAttrs (_: v: seqIf cond a v); eval-config-minimal = import ./eval-config-minimal.nix { inherit lib; }; + + testing-lib = import ./testing/default.nix { inherit lib; }; in /* This attribute set appears as lib.nixos in the flake, or can be imported @@ -30,4 +32,10 @@ in inherit (seqAttrsIf (!featureFlags?minimalModules) minimalModulesWarning eval-config-minimal) evalModules ; + + inherit (testing-lib) + evalTest + runTest + ; + } diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix index 1c331a3c5162b..e72e5d476bfa6 100644 --- a/nixos/lib/testing-python.nix +++ b/nixos/lib/testing-python.nix @@ -12,6 +12,10 @@ with pkgs; +let + nixos-lib = import ./default.nix { inherit (pkgs) lib; }; +in + rec { inherit pkgs; @@ -166,26 +170,15 @@ rec { ${lib.optionalString (interactive) "--add-flags --interactive"} ''); - evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; }; - runTest = module: (evalTest module).config.run; - - testModules = [ - ./testing/driver.nix - ./testing/interactive.nix - ./testing/legacy.nix - ./testing/meta.nix - ./testing/name.nix - ./testing/network.nix - ./testing/nodes.nix - ./testing/pkgs.nix - ./testing/run.nix - ./testing/testScript.nix - { - config = { - hostPkgs = pkgs; - }; - } - ]; + evalTest = module: nixos-lib.evalTest { imports = [ extraTestModule module ]; }; + runTest = module: nixos-lib.runTest { imports = [ extraTestModule module ]; }; + + extraTestModule = { + config = { + hostPkgs = pkgs; + minimalResult = hydra; + }; + }; # Make a full-blown test makeTest = diff --git a/nixos/lib/testing/default.nix b/nixos/lib/testing/default.nix new file mode 100644 index 0000000000000..676d52f5c3fb7 --- /dev/null +++ b/nixos/lib/testing/default.nix @@ -0,0 +1,24 @@ +{ lib }: +let + + evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; }; + runTest = module: (evalTest module).config.result; + + testModules = [ + ./call-test.nix + ./driver.nix + ./interactive.nix + ./legacy.nix + ./meta.nix + ./name.nix + ./network.nix + ./nodes.nix + ./pkgs.nix + ./run.nix + ./testScript.nix + ]; + +in +{ + inherit evalTest runTest testModules; +} |