diff options
Diffstat (limited to 'nixos/lib')
-rw-r--r-- | nixos/lib/default.nix | 33 | ||||
-rw-r--r-- | nixos/lib/eval-cacheable-options.nix | 53 | ||||
-rw-r--r-- | nixos/lib/eval-config-minimal.nix | 49 | ||||
-rw-r--r-- | nixos/lib/eval-config.nix | 35 | ||||
-rw-r--r-- | nixos/lib/make-iso9660-image.sh | 1 | ||||
-rw-r--r-- | nixos/lib/make-options-doc/default.nix | 51 | ||||
-rw-r--r-- | nixos/lib/make-options-doc/mergeJSON.py | 86 | ||||
-rw-r--r-- | nixos/lib/qemu-common.nix | 4 | ||||
-rw-r--r-- | nixos/lib/systemd-lib.nix | 7 | ||||
-rw-r--r-- | nixos/lib/systemd-unit-options.nix | 42 | ||||
-rw-r--r-- | nixos/lib/test-driver/default.nix | 4 | ||||
-rw-r--r-- | nixos/lib/test-driver/setup.py | 2 | ||||
-rwxr-xr-x | nixos/lib/test-driver/test_driver/__init__.py | 34 | ||||
-rw-r--r-- | nixos/lib/test-driver/test_driver/driver.py | 74 | ||||
-rw-r--r-- | nixos/lib/test-driver/test_driver/machine.py | 31 | ||||
-rw-r--r-- | nixos/lib/test-driver/test_driver/polling_condition.py | 77 | ||||
-rw-r--r-- | nixos/lib/testing-python.nix | 13 | ||||
-rw-r--r-- | nixos/lib/utils.nix | 9 |
18 files changed, 537 insertions, 68 deletions
diff --git a/nixos/lib/default.nix b/nixos/lib/default.nix new file mode 100644 index 0000000000000..2b3056e01457f --- /dev/null +++ b/nixos/lib/default.nix @@ -0,0 +1,33 @@ +let + # The warning is in a top-level let binding so it is only printed once. + minimalModulesWarning = warn "lib.nixos.evalModules is experimental and subject to change. See nixos/lib/default.nix" null; + inherit (nonExtendedLib) warn; + nonExtendedLib = import ../../lib; +in +{ # Optional. Allows an extended `lib` to be used instead of the regular Nixpkgs lib. + lib ? nonExtendedLib, + + # Feature flags allow you to opt in to unfinished code. These may change some + # behavior or disable warnings. + featureFlags ? {}, + + # This file itself is rather new, so we accept unknown parameters to be forward + # compatible. This is generally not recommended, because typos go undetected. + ... +}: +let + seqIf = cond: if cond then builtins.seq else a: b: b; + # If cond, force `a` before returning any attr + seqAttrsIf = cond: a: lib.mapAttrs (_: v: seqIf cond a v); + + eval-config-minimal = import ./eval-config-minimal.nix { inherit lib; }; +in +/* + This attribute set appears as lib.nixos in the flake, or can be imported + using a binding like `nixosLib = import (nixpkgs + "/nixos/lib") { }`. +*/ +{ + inherit (seqAttrsIf (!featureFlags?minimalModules) minimalModulesWarning eval-config-minimal) + evalModules + ; +} diff --git a/nixos/lib/eval-cacheable-options.nix b/nixos/lib/eval-cacheable-options.nix new file mode 100644 index 0000000000000..c3ba2ce663758 --- /dev/null +++ b/nixos/lib/eval-cacheable-options.nix @@ -0,0 +1,53 @@ +{ libPath +, pkgsLibPath +, nixosPath +, modules +, stateVersion +, release +}: + +let + lib = import libPath; + modulesPath = "${nixosPath}/modules"; + # dummy pkgs set that contains no packages, only `pkgs.lib` from the full set. + # not having `pkgs.lib` causes all users of `pkgs.formats` to fail. + pkgs = import pkgsLibPath { + inherit lib; + pkgs = null; + }; + utils = import "${nixosPath}/lib/utils.nix" { + inherit config lib; + pkgs = null; + }; + # this is used both as a module and as specialArgs. + # as a module it sets the _module special values, as specialArgs it makes `config` + # unusable. this causes documentation attributes depending on `config` to fail. + config = { + _module.check = false; + _module.args = {}; + system.stateVersion = stateVersion; + }; + eval = lib.evalModules { + modules = (map (m: "${modulesPath}/${m}") modules) ++ [ + config + ]; + specialArgs = { + inherit config pkgs utils; + }; + }; + docs = import "${nixosPath}/doc/manual" { + pkgs = pkgs // { + inherit lib; + # duplicate of the declaration in all-packages.nix + buildPackages.nixosOptionsDoc = attrs: + (import "${nixosPath}/lib/make-options-doc") + ({ inherit pkgs lib; } // attrs); + }; + config = config.config; + options = eval.options; + version = release; + revision = "release-${release}"; + prefix = modulesPath; + }; +in + docs.optionsNix diff --git a/nixos/lib/eval-config-minimal.nix b/nixos/lib/eval-config-minimal.nix new file mode 100644 index 0000000000000..d45b9ffd42618 --- /dev/null +++ b/nixos/lib/eval-config-minimal.nix @@ -0,0 +1,49 @@ + +# DO NOT IMPORT. Use nixpkgsFlake.lib.nixos, or import (nixpkgs + "/nixos/lib") +{ lib }: # read -^ + +let + + /* + Invoke NixOS. Unlike traditional NixOS, this does not include all modules. + Any such modules have to be explicitly added via the `modules` parameter, + or imported using `imports` in a module. + + A minimal module list improves NixOS evaluation performance and allows + modules to be independently usable, supporting new use cases. + + Parameters: + + modules: A list of modules that constitute the configuration. + + specialArgs: An attribute set of module arguments. Unlike + `config._module.args`, these are available for use in + `imports`. + `config._module.args` should be preferred when possible. + + Return: + + An attribute set containing `config.system.build.toplevel` among other + attributes. See `lib.evalModules` in the Nixpkgs library. + + */ + evalModules = { + prefix ? [], + modules ? [], + specialArgs ? {}, + }: + # NOTE: Regular NixOS currently does use this function! Don't break it! + # Ideally we don't diverge, unless we learn that we should. + # In other words, only the public interface of nixos.evalModules + # is experimental. + lib.evalModules { + inherit prefix modules; + specialArgs = { + modulesPath = builtins.toString ../modules; + } // specialArgs; + }; + +in +{ + inherit evalModules; +} diff --git a/nixos/lib/eval-config.nix b/nixos/lib/eval-config.nix index 62d09b8173bda..2daaa8a118632 100644 --- a/nixos/lib/eval-config.nix +++ b/nixos/lib/eval-config.nix @@ -21,6 +21,7 @@ evalConfigArgs@ , # !!! See comment about args in lib/modules.nix specialArgs ? {} , modules +, modulesLocation ? (builtins.unsafeGetAttrPos "modules" evalConfigArgs).file or null , # !!! See comment about check in lib/modules.nix check ? true , prefix ? [] @@ -33,6 +34,12 @@ let pkgs_ = pkgs; in let + evalModulesMinimal = (import ./default.nix { + inherit lib; + # Implicit use of feature is noted in implementation. + featureFlags.minimalModules = { }; + }).evalModules; + pkgsModule = rec { _file = ./eval-config.nix; key = _file; @@ -68,13 +75,22 @@ let _module.check = lib.mkDefault check; }; }; - allUserModules = modules ++ legacyModules; - noUserModules = lib.evalModules ({ - inherit prefix; + allUserModules = + let + # Add the invoking file (or specified modulesLocation) as error message location + # for modules that don't have their own locations; presumably inline modules. + locatedModules = + if modulesLocation == null then + modules + else + map (lib.setDefaultModuleLocation modulesLocation) modules; + in + locatedModules ++ legacyModules; + + noUserModules = evalModulesMinimal ({ + inherit prefix specialArgs; modules = baseModules ++ extraModules ++ [ pkgsModule modulesModule ]; - specialArgs = - { modulesPath = builtins.toString ../modules; } // specialArgs; }); # Extra arguments that are useful for constructing a similar configuration. @@ -88,13 +104,8 @@ let nixosWithUserModules = noUserModules.extendModules { modules = allUserModules; }; -in withWarnings { - - # Merge the option definitions in all modules, forming the full - # system configuration. - inherit (nixosWithUserModules) config options _module type; - +in +withWarnings nixosWithUserModules // { inherit extraArgs; - inherit (nixosWithUserModules._module.args) pkgs; } diff --git a/nixos/lib/make-iso9660-image.sh b/nixos/lib/make-iso9660-image.sh index 4740b05f95571..9273b8d3db8dc 100644 --- a/nixos/lib/make-iso9660-image.sh +++ b/nixos/lib/make-iso9660-image.sh @@ -105,6 +105,7 @@ mkdir -p $out/iso # version-5 UUID's work) xorriso="xorriso -boot_image any gpt_disk_guid=$(uuid -v 5 daed2280-b91e-42c0-aed6-82c825ca41f3 $out | tr -d -) + -volume_date all_file_dates =$SOURCE_DATE_EPOCH -as mkisofs -iso-level 3 -volid ${volumeID} diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index 44bc25be92384..57652dd5db1e7 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -21,6 +21,13 @@ , options , transformOptions ? lib.id # function for additional tranformations of the options , revision ? "" # Specify revision for the options +# a set of options the docs we are generating will be merged into, as if by recursiveUpdate. +# used to split the options doc build into a static part (nixos/modules) and a dynamic part +# (non-nixos modules imported via configuration.nix, other module sources). +, baseOptionsJSON ? null +# instead of printing warnings for eg options with missing descriptions (which may be lost +# by nix build unless -L is given), emit errors instead and fail the build +, warningsAreErrors ? true }: let @@ -51,10 +58,15 @@ let # ../../../lib/options.nix influences. # # Each element of `relatedPackages` can be either - # - a string: that will be interpreted as an attribute name from `pkgs`, - # - a list: that will be interpreted as an attribute path from `pkgs`, - # - an attrset: that can specify `name`, `path`, `package`, `comment` + # - a string: that will be interpreted as an attribute name from `pkgs` and turned into a link + # to search.nixos.org, + # - a list: that will be interpreted as an attribute path from `pkgs` and turned into a link + # to search.nixos.org, + # - an attrset: that can specify `name`, `path`, `comment` # (either of `name`, `path` is required, the rest are optional). + # + # NOTE: No checks against `pkgs` are made to ensure that the referenced package actually exists. + # Such checks are not compatible with option docs caching. genRelatedPackages = packages: optName: let unpack = p: if lib.isString p then { name = p; } @@ -64,16 +76,16 @@ let let title = args.title or null; name = args.name or (lib.concatStringsSep "." args.path); - path = args.path or [ args.name ]; - package = args.package or (lib.attrByPath path (throw "Invalid package attribute path `${toString path}' found while evaluating `relatedPackages' of option `${optName}'") pkgs); - in "<listitem>" - + "<para><literal>${lib.optionalString (title != null) "${title} aka "}pkgs.${name} (${package.meta.name})</literal>" - + lib.optionalString (!package.meta.available) " <emphasis>[UNAVAILABLE]</emphasis>" - + ": ${package.meta.description or "???"}.</para>" - + lib.optionalString (args ? comment) "\n<para>${args.comment}</para>" - # Lots of `longDescription's break DocBook, so we just wrap them into <programlisting> - + lib.optionalString (package.meta ? longDescription) "\n<programlisting>${package.meta.longDescription}</programlisting>" - + "</listitem>"; + in '' + <listitem> + <para> + <link xlink:href="https://search.nixos.org/packages?show=${name}&sort=relevance&query=${name}"> + <literal>${lib.optionalString (title != null) "${title} aka "}pkgs.${name}</literal> + </link> + </para> + ${lib.optionalString (args ? comment) "<para>${args.comment}</para>"} + </listitem> + ''; in "<itemizedlist>${lib.concatStringsSep "\n" (map (p: describe (unpack p)) packages)}</itemizedlist>"; # Remove invisible and internal options. @@ -99,13 +111,24 @@ in rec { optionsJSON = pkgs.runCommand "options.json" { meta.description = "List of NixOS options in JSON format"; buildInputs = [ pkgs.brotli ]; + options = builtins.toFile "options.json" + (builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix)); } '' # Export list of options in different format. dst=$out/share/doc/nixos mkdir -p $dst - cp ${builtins.toFile "options.json" (builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix))} $dst/options.json + ${ + if baseOptionsJSON == null + then "cp $options $dst/options.json" + else '' + ${pkgs.python3Minimal}/bin/python ${./mergeJSON.py} \ + ${lib.optionalString warningsAreErrors "--warnings-are-errors"} \ + ${baseOptionsJSON} $options \ + > $dst/options.json + '' + } brotli -9 < $dst/options.json > $dst/options.json.br diff --git a/nixos/lib/make-options-doc/mergeJSON.py b/nixos/lib/make-options-doc/mergeJSON.py new file mode 100644 index 0000000000000..029787a31586c --- /dev/null +++ b/nixos/lib/make-options-doc/mergeJSON.py @@ -0,0 +1,86 @@ +import collections +import json +import sys +from typing import Any, Dict, List + +JSON = Dict[str, Any] + +class Key: + def __init__(self, path: List[str]): + self.path = path + def __hash__(self): + result = 0 + for id in self.path: + result ^= hash(id) + return result + def __eq__(self, other): + return type(self) is type(other) and self.path == other.path + +Option = collections.namedtuple('Option', ['name', 'value']) + +# pivot a dict of options keyed by their display name to a dict keyed by their path +def pivot(options: Dict[str, JSON]) -> Dict[Key, Option]: + result: Dict[Key, Option] = dict() + for (name, opt) in options.items(): + result[Key(opt['loc'])] = Option(name, opt) + return result + +# pivot back to indexed-by-full-name +# like the docbook build we'll just fail if multiple options with differing locs +# render to the same option name. +def unpivot(options: Dict[Key, Option]) -> Dict[str, JSON]: + result: Dict[str, Dict] = dict() + for (key, opt) in options.items(): + if opt.name in result: + raise RuntimeError( + 'multiple options with colliding ids found', + opt.name, + result[opt.name]['loc'], + opt.value['loc'], + ) + result[opt.name] = opt.value + return result + +warningsAreErrors = sys.argv[1] == "--warnings-are-errors" +optOffset = 1 if warningsAreErrors else 0 +options = pivot(json.load(open(sys.argv[1 + optOffset], 'r'))) +overrides = pivot(json.load(open(sys.argv[2 + optOffset], 'r'))) + +# fix up declaration paths in lazy options, since we don't eval them from a full nixpkgs dir +for (k, v) in options.items(): + v.value['declarations'] = list(map(lambda s: f'nixos/modules/{s}', v.value['declarations'])) + +# merge both descriptions +for (k, v) in overrides.items(): + cur = options.setdefault(k, v).value + for (ok, ov) in v.value.items(): + if ok == 'declarations': + decls = cur[ok] + for d in ov: + if d not in decls: + decls += [d] + elif ok == "type": + # ignore types of placeholder options + if ov != "_unspecified" or cur[ok] == "_unspecified": + cur[ok] = ov + elif ov is not None or cur.get(ok, None) is None: + cur[ok] = ov + +# check that every option has a description +hasWarnings = False +for (k, v) in options.items(): + if v.value.get('description', None) is None: + severity = "error" if warningsAreErrors else "warning" + hasWarnings = True + print(f"\x1b[1;31m{severity}: option {v.name} has no description\x1b[0m", file=sys.stderr) + v.value['description'] = "This option has no description." +if hasWarnings and warningsAreErrors: + print( + "\x1b[1;31m" + + "Treating warnings as errors. Set documentation.nixos.options.warningsAreErrors " + + "to false to ignore these warnings." + + "\x1b[0m", + file=sys.stderr) + sys.exit(1) + +json.dump(unpivot(options), fp=sys.stdout) diff --git a/nixos/lib/qemu-common.nix b/nixos/lib/qemu-common.nix index f3af85040bd62..20bbe9ff5d99f 100644 --- a/nixos/lib/qemu-common.nix +++ b/nixos/lib/qemu-common.nix @@ -17,12 +17,12 @@ rec { ''-netdev vde,id=vlan${toString nic},sock="$QEMU_VDE_SOCKET_${toString net}"'' ]; - qemuSerialDevice = if pkgs.stdenv.hostPlatform.isx86 then "ttyS0" + qemuSerialDevice = if pkgs.stdenv.hostPlatform.isx86 || pkgs.stdenv.hostPlatform.isRiscV then "ttyS0" else if (with pkgs.stdenv.hostPlatform; isAarch32 || isAarch64 || isPower) then "ttyAMA0" else throw "Unknown QEMU serial device for system '${pkgs.stdenv.hostPlatform.system}'"; qemuBinary = qemuPkg: { - x86_64-linux = "${qemuPkg}/bin/qemu-kvm -cpu qemu64"; + x86_64-linux = "${qemuPkg}/bin/qemu-kvm -cpu max"; armv7l-linux = "${qemuPkg}/bin/qemu-system-arm -enable-kvm -machine virt -cpu host"; aarch64-linux = "${qemuPkg}/bin/qemu-system-aarch64 -enable-kvm -machine virt,gic-version=host -cpu host"; powerpc64le-linux = "${qemuPkg}/bin/qemu-system-ppc64 -machine powernv"; diff --git a/nixos/lib/systemd-lib.nix b/nixos/lib/systemd-lib.nix index 6c4d27018eed8..ab166d7327cea 100644 --- a/nixos/lib/systemd-lib.nix +++ b/nixos/lib/systemd-lib.nix @@ -11,6 +11,9 @@ in rec { mkPathSafeName = lib.replaceChars ["@" ":" "\\" "[" "]"] ["-" "-" "-" "" ""]; + # a type for options that take a unit name + unitNameType = types.strMatching "[a-zA-Z0-9@%:_.\\-]+[.](service|socket|device|mount|automount|swap|target|path|timer|scope|slice)"; + makeUnit = name: unit: if unit.enable then pkgs.runCommand "unit-${mkPathSafeName name}" @@ -228,9 +231,7 @@ in rec { mkdir -p $out/getty.target.wants/ ln -s ../autovt@tty1.service $out/getty.target.wants/ - ln -s ../local-fs.target ../remote-fs.target \ - ../nss-lookup.target ../nss-user-lookup.target ../swap.target \ - $out/multi-user.target.wants/ + ln -s ../remote-fs.target $out/multi-user.target.wants/ ''} ''; # */ diff --git a/nixos/lib/systemd-unit-options.nix b/nixos/lib/systemd-unit-options.nix index 01f954a4d3e01..8029ba0e3f6cf 100644 --- a/nixos/lib/systemd-unit-options.nix +++ b/nixos/lib/systemd-unit-options.nix @@ -45,7 +45,7 @@ in rec { requiredBy = mkOption { default = []; - type = types.listOf types.str; + type = types.listOf unitNameType; description = '' Units that require (i.e. depend on and need to go down with) this unit. The discussion under <literal>wantedBy</literal> @@ -56,7 +56,7 @@ in rec { wantedBy = mkOption { default = []; - type = types.listOf types.str; + type = types.listOf unitNameType; description = '' Units that want (i.e. depend on) this unit. The standard way to make a unit start by default at boot is to set this option @@ -73,7 +73,7 @@ in rec { aliases = mkOption { default = []; - type = types.listOf types.str; + type = types.listOf unitNameType; description = "Aliases of that unit."; }; @@ -98,7 +98,7 @@ in rec { description = mkOption { default = ""; - type = types.str; + type = types.singleLineStr; description = "Description of this unit used in systemd messages and progress indicators."; }; @@ -110,7 +110,7 @@ in rec { requires = mkOption { default = []; - type = types.listOf types.str; + type = types.listOf unitNameType; description = '' Start the specified units when this unit is started, and stop this unit when the specified units are stopped or fail. @@ -119,7 +119,7 @@ in rec { wants = mkOption { default = []; - type = types.listOf types.str; + type = types.listOf unitNameType; description = '' Start the specified units when this unit is started. ''; @@ -127,7 +127,7 @@ in rec { after = mkOption { default = []; - type = types.listOf types.str; + type = types.listOf unitNameType; description = '' If the specified units are started at the same time as this unit, delay this unit until they have started. @@ -136,7 +136,7 @@ in rec { before = mkOption { default = []; - type = types.listOf types.str; + type = types.listOf unitNameType; description = '' If the specified units are started at the same time as this unit, delay them until this unit has started. @@ -145,7 +145,7 @@ in rec { bindsTo = mkOption { default = []; - type = types.listOf types.str; + type = types.listOf unitNameType; description = '' Like ‘requires’, but in addition, if the specified units unexpectedly disappear, this unit will be stopped as well. @@ -154,7 +154,7 @@ in rec { partOf = mkOption { default = []; - type = types.listOf types.str; + type = types.listOf unitNameType; description = '' If the specified units are stopped or restarted, then this unit is stopped or restarted as well. @@ -163,7 +163,7 @@ in rec { conflicts = mkOption { default = []; - type = types.listOf types.str; + type = types.listOf unitNameType; description = '' If the specified units are started, then this unit is stopped and vice versa. @@ -172,7 +172,7 @@ in rec { requisite = mkOption { default = []; - type = types.listOf types.str; + type = types.listOf unitNameType; description = '' Similar to requires. However if the units listed are not started, they will not be started and the transaction will fail. @@ -201,9 +201,20 @@ in rec { ''; }; + reloadTriggers = mkOption { + default = []; + type = types.listOf unitOption; + description = '' + An arbitrary list of items such as derivations. If any item + in the list changes between reconfigurations, the service will + be reloaded. If anything but a reload trigger changes in the + unit file, the unit will be restarted instead. + ''; + }; + onFailure = mkOption { default = []; - type = types.listOf types.str; + type = types.listOf unitNameType; description = '' A list of one or more units that are activated when this unit enters the "failed" state. @@ -338,6 +349,11 @@ in rec { configuration switch if its definition has changed. If enabled, the value of <option>restartIfChanged</option> is ignored. + + This option should not be used anymore in favor of + <option>reloadTriggers</option> which allows more granular + control of when a service is reloaded and when a service + is restarted. ''; }; diff --git a/nixos/lib/test-driver/default.nix b/nixos/lib/test-driver/default.nix index 3f63bc705b902..3aee913431890 100644 --- a/nixos/lib/test-driver/default.nix +++ b/nixos/lib/test-driver/default.nix @@ -14,7 +14,7 @@ python3Packages.buildPythonApplication rec { pname = "nixos-test-driver"; - version = "1.0"; + version = "1.1"; src = ./.; propagatedBuildInputs = [ coreutils netpbm python3Packages.colorama python3Packages.ptpython qemu_pkg socat vde2 ] @@ -26,7 +26,7 @@ python3Packages.buildPythonApplication rec { mypy --disallow-untyped-defs \ --no-implicit-optional \ --ignore-missing-imports ${src}/test_driver - pylint --errors-only ${src}/test_driver + pylint --errors-only --enable=unused-import ${src}/test_driver black --check --diff ${src}/test_driver ''; } diff --git a/nixos/lib/test-driver/setup.py b/nixos/lib/test-driver/setup.py index 156995472169e..476c7b2dab2a4 100644 --- a/nixos/lib/test-driver/setup.py +++ b/nixos/lib/test-driver/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name="nixos-test-driver", - version='1.0', + version='1.1', packages=find_packages(), entry_points={ "console_scripts": [ diff --git a/nixos/lib/test-driver/test_driver/__init__.py b/nixos/lib/test-driver/test_driver/__init__.py index 5477ab5cd038e..61d91c9ed6545 100755 --- a/nixos/lib/test-driver/test_driver/__init__.py +++ b/nixos/lib/test-driver/test_driver/__init__.py @@ -33,6 +33,22 @@ class EnvDefault(argparse.Action): setattr(namespace, self.dest, values) +def writeable_dir(arg: str) -> Path: + """Raises an ArgumentTypeError if the given argument isn't a writeable directory + Note: We want to fail as early as possible if a directory isn't writeable, + since an executed nixos-test could fail (very late) because of the test-driver + writing in a directory without proper permissions. + """ + path = Path(arg) + if not path.is_dir(): + raise argparse.ArgumentTypeError("{0} is not a directory".format(path)) + if not os.access(path, os.W_OK): + raise argparse.ArgumentTypeError( + "{0} is not a writeable directory".format(path) + ) + return path + + def main() -> None: arg_parser = argparse.ArgumentParser(prog="nixos-test-driver") arg_parser.add_argument( @@ -45,7 +61,7 @@ def main() -> None: "-I", "--interactive", help="drop into a python repl and run the tests interactively", - action="store_true", + action=argparse.BooleanOptionalAction, ) arg_parser.add_argument( "--start-scripts", @@ -64,6 +80,14 @@ def main() -> None: help="vlans to span by the driver", ) arg_parser.add_argument( + "-o", + "--output_directory", + help="""The path to the directory where outputs copied from the VM will be placed. + By e.g. Machine.copy_from_vm or Machine.screenshot""", + default=Path.cwd(), + type=writeable_dir, + ) + arg_parser.add_argument( "testscript", action=EnvDefault, envvar="testScript", @@ -77,7 +101,11 @@ def main() -> None: rootlog.info("Machine state will be reset. To keep it, pass --keep-vm-state") with Driver( - args.start_scripts, args.vlans, args.testscript.read_text(), args.keep_vm_state + args.start_scripts, + args.vlans, + args.testscript.read_text(), + args.output_directory.resolve(), + args.keep_vm_state, ) as driver: if args.interactive: ptpython.repl.embed(driver.test_symbols(), {}) @@ -94,7 +122,7 @@ def generate_driver_symbols() -> None: in user's test scripts. That list is then used by pyflakes to lint those scripts. """ - d = Driver([], [], "") + d = Driver([], [], "", Path()) test_symbols = d.test_symbols() with open("driver-symbols", "w") as fp: fp.write(",".join(test_symbols.keys())) diff --git a/nixos/lib/test-driver/test_driver/driver.py b/nixos/lib/test-driver/test_driver/driver.py index f3af98537ad67..880b1c5fdec0d 100644 --- a/nixos/lib/test-driver/test_driver/driver.py +++ b/nixos/lib/test-driver/test_driver/driver.py @@ -1,12 +1,35 @@ from contextlib import contextmanager from pathlib import Path -from typing import Any, Dict, Iterator, List +from typing import Any, Dict, Iterator, List, Union, Optional, Callable, ContextManager import os import tempfile from test_driver.logger import rootlog from test_driver.machine import Machine, NixStartScript, retry from test_driver.vlan import VLan +from test_driver.polling_condition import PollingCondition + + +def get_tmp_dir() -> Path: + """Returns a temporary directory that is defined by TMPDIR, TEMP, TMP or CWD + Raises an exception in case the retrieved temporary directory is not writeable + See https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir + """ + tmp_dir = Path(tempfile.gettempdir()) + tmp_dir.mkdir(mode=0o700, exist_ok=True) + if not tmp_dir.is_dir(): + raise NotADirectoryError( + "The directory defined by TMPDIR, TEMP, TMP or CWD: {0} is not a directory".format( + tmp_dir + ) + ) + if not os.access(tmp_dir, os.W_OK): + raise PermissionError( + "The directory defined by TMPDIR, TEMP, TMP, or CWD: {0} is not writeable".format( + tmp_dir + ) + ) + return tmp_dir class Driver: @@ -16,18 +39,20 @@ class Driver: tests: str vlans: List[VLan] machines: List[Machine] + polling_conditions: List[PollingCondition] def __init__( self, start_scripts: List[str], vlans: List[int], tests: str, + out_dir: Path, keep_vm_state: bool = False, ): self.tests = tests + self.out_dir = out_dir - tmp_dir = Path(os.environ.get("TMPDIR", tempfile.gettempdir())) - tmp_dir.mkdir(mode=0o700, exist_ok=True) + tmp_dir = get_tmp_dir() with rootlog.nested("start all VLans"): self.vlans = [VLan(nr, tmp_dir) for nr in vlans] @@ -36,12 +61,16 @@ class Driver: for s in scripts: yield NixStartScript(s) + self.polling_conditions = [] + self.machines = [ Machine( start_command=cmd, keep_vm_state=keep_vm_state, name=cmd.machine_name, tmp_dir=tmp_dir, + callbacks=[self.check_polling_conditions], + out_dir=self.out_dir, ) for cmd in cmd(start_scripts) ] @@ -84,6 +113,7 @@ class Driver: retry=retry, serial_stdout_off=self.serial_stdout_off, serial_stdout_on=self.serial_stdout_on, + polling_condition=self.polling_condition, Machine=Machine, # for typing ) machine_symbols = {m.name: m for m in self.machines} @@ -135,8 +165,8 @@ class Driver: "Using legacy create_machine(), please instantiate the" "Machine class directly, instead" ) - tmp_dir = Path(os.environ.get("TMPDIR", tempfile.gettempdir())) - tmp_dir.mkdir(mode=0o700, exist_ok=True) + + tmp_dir = get_tmp_dir() if args.get("startCommand"): start_command: str = args.get("startCommand", "") @@ -148,6 +178,7 @@ class Driver: return Machine( tmp_dir=tmp_dir, + out_dir=self.out_dir, start_command=cmd, name=name, keep_vm_state=args.get("keep_vm_state", False), @@ -159,3 +190,36 @@ class Driver: def serial_stdout_off(self) -> None: rootlog._print_serial_logs = False + + def check_polling_conditions(self) -> None: + for condition in self.polling_conditions: + condition.maybe_raise() + + def polling_condition( + self, + fun_: Optional[Callable] = None, + *, + seconds_interval: float = 2.0, + description: Optional[str] = None, + ) -> Union[Callable[[Callable], ContextManager], ContextManager]: + driver = self + + class Poll: + def __init__(self, fun: Callable): + self.condition = PollingCondition( + fun, + seconds_interval, + description, + ) + + def __enter__(self) -> None: + driver.polling_conditions.append(self.condition) + + def __exit__(self, a, b, c) -> None: # type: ignore + res = driver.polling_conditions.pop() + assert res is self.condition + + if fun_ is None: + return Poll + else: + return Poll(fun_) diff --git a/nixos/lib/test-driver/test_driver/machine.py b/nixos/lib/test-driver/test_driver/machine.py index b3dbe5126fcc6..569a0f3c61e48 100644 --- a/nixos/lib/test-driver/test_driver/machine.py +++ b/nixos/lib/test-driver/test_driver/machine.py @@ -241,9 +241,15 @@ class LegacyStartCommand(StartCommand): cdrom: Optional[str] = None, usb: Optional[str] = None, bios: Optional[str] = None, + qemuBinary: Optional[str] = None, qemuFlags: Optional[str] = None, ): - self._cmd = "qemu-kvm -m 384" + if qemuBinary is not None: + self._cmd = qemuBinary + else: + self._cmd = "qemu-kvm" + + self._cmd += " -m 384" # networking net_backend = "-netdev user,id=net0" @@ -297,6 +303,7 @@ class Machine: the machine lifecycle with the help of a start script / command.""" name: str + out_dir: Path tmp_dir: Path shared_dir: Path state_dir: Path @@ -318,23 +325,28 @@ class Machine: # Store last serial console lines for use # of wait_for_console_text last_lines: Queue = Queue() + callbacks: List[Callable] def __repr__(self) -> str: return f"<Machine '{self.name}'>" def __init__( self, + out_dir: Path, tmp_dir: Path, start_command: StartCommand, name: str = "machine", keep_vm_state: bool = False, allow_reboot: bool = False, + callbacks: Optional[List[Callable]] = None, ) -> None: + self.out_dir = out_dir self.tmp_dir = tmp_dir self.keep_vm_state = keep_vm_state self.allow_reboot = allow_reboot self.name = name self.start_command = start_command + self.callbacks = callbacks if callbacks is not None else [] # set up directories self.shared_dir = self.tmp_dir / "shared-xchg" @@ -375,6 +387,7 @@ class Machine: cdrom=args.get("cdrom"), usb=args.get("usb"), bios=args.get("bios"), + qemuBinary=args.get("qemuBinary"), qemuFlags=args.get("qemuFlags"), ) @@ -406,6 +419,7 @@ class Machine: return answer def send_monitor_command(self, command: str) -> str: + self.run_callbacks() with self.nested("sending monitor command: {}".format(command)): message = ("{}\n".format(command)).encode() assert self.monitor is not None @@ -509,6 +523,7 @@ class Machine: def execute( self, command: str, check_return: bool = True, timeout: Optional[int] = 900 ) -> Tuple[int, str]: + self.run_callbacks() self.connect() if timeout is not None: @@ -535,11 +550,11 @@ class Machine: Should only be used during test development, not in the production test.""" self.connect() - self.log("Terminal is ready (there is no prompt):") + self.log("Terminal is ready (there is no initial prompt):") assert self.shell subprocess.run( - ["socat", "READLINE", f"FD:{self.shell.fileno()}"], + ["socat", "READLINE,prompt=$ ", f"FD:{self.shell.fileno()}"], pass_fds=[self.shell.fileno()], ) @@ -697,10 +712,9 @@ class Machine: self.connected = True def screenshot(self, filename: str) -> None: - out_dir = os.environ.get("out", os.getcwd()) word_pattern = re.compile(r"^\w+$") if word_pattern.match(filename): - filename = os.path.join(out_dir, "{}.png".format(filename)) + filename = os.path.join(self.out_dir, "{}.png".format(filename)) tmp = "{}.ppm".format(filename) with self.nested( @@ -751,7 +765,6 @@ class Machine: all the VMs (using a temporary directory). """ # Compute the source, target, and intermediate shared file names - out_dir = Path(os.environ.get("out", os.getcwd())) vm_src = Path(source) with tempfile.TemporaryDirectory(dir=self.shared_dir) as shared_td: shared_temp = Path(shared_td) @@ -761,7 +774,7 @@ class Machine: # Copy the file to the shared directory inside VM self.succeed(make_command(["mkdir", "-p", vm_shared_temp])) self.succeed(make_command(["cp", "-r", vm_src, vm_intermediate])) - abs_target = out_dir / target_dir / vm_src.name + abs_target = self.out_dir / target_dir / vm_src.name abs_target.parent.mkdir(exist_ok=True, parents=True) # Copy the file from the shared directory outside VM if intermediate.is_dir(): @@ -969,3 +982,7 @@ class Machine: self.shell.close() self.monitor.close() self.serial_thread.join() + + def run_callbacks(self) -> None: + for callback in self.callbacks: + callback() diff --git a/nixos/lib/test-driver/test_driver/polling_condition.py b/nixos/lib/test-driver/test_driver/polling_condition.py new file mode 100644 index 0000000000000..459845452fa12 --- /dev/null +++ b/nixos/lib/test-driver/test_driver/polling_condition.py @@ -0,0 +1,77 @@ +from typing import Callable, Optional +import time + +from .logger import rootlog + + +class PollingConditionFailed(Exception): + pass + + +class PollingCondition: + condition: Callable[[], bool] + seconds_interval: float + description: Optional[str] + + last_called: float + entered: bool + + def __init__( + self, + condition: Callable[[], Optional[bool]], + seconds_interval: float = 2.0, + description: Optional[str] = None, + ): + self.condition = condition # type: ignore + self.seconds_interval = seconds_interval + + if description is None: + if condition.__doc__: + self.description = condition.__doc__ + else: + self.description = condition.__name__ + else: + self.description = str(description) + + self.last_called = float("-inf") + self.entered = False + + def check(self) -> bool: + if self.entered or not self.overdue: + return True + + with self, rootlog.nested(self.nested_message): + rootlog.info(f"Time since last: {time.monotonic() - self.last_called:.2f}s") + try: + res = self.condition() # type: ignore + except Exception: + res = False + res = res is None or res + rootlog.info(self.status_message(res)) + return res + + def maybe_raise(self) -> None: + if not self.check(): + raise PollingConditionFailed(self.status_message(False)) + + def status_message(self, status: bool) -> str: + return f"Polling condition {'succeeded' if status else 'failed'}: {self.description}" + + @property + def nested_message(self) -> str: + nested_message = ["Checking polling condition"] + if self.description is not None: + nested_message.append(repr(self.description)) + + return " ".join(nested_message) + + @property + def overdue(self) -> bool: + return self.last_called + self.seconds_interval < time.monotonic() + + def __enter__(self) -> None: + self.entered = True + + def __exit__(self, exc_type, exc_value, traceback) -> None: # type: ignore + self.entered = False + self.last_called = time.monotonic() diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix index 365e22714573e..0d3c3a89e7836 100644 --- a/nixos/lib/testing-python.nix +++ b/nixos/lib/testing-python.nix @@ -17,7 +17,7 @@ rec { inherit pkgs; # Run an automated test suite in the given virtual network. - runTests = { driver, pos }: + runTests = { driver, driverInteractive, pos }: stdenv.mkDerivation { name = "vm-test-run-${driver.testName}"; @@ -30,11 +30,11 @@ rec { # effectively mute the XMLLogger export LOGFILE=/dev/null - ${driver}/bin/nixos-test-driver + ${driver}/bin/nixos-test-driver -o $out ''; passthru = driver.passthru // { - inherit driver; + inherit driver driverInteractive; }; inherit pos; # for better debugging @@ -51,6 +51,7 @@ rec { , enableOCR ? false , skipLint ? false , passthru ? {} + , interactive ? false }: let # Reifies and correctly wraps the python test driver for @@ -139,7 +140,8 @@ rec { wrapProgram $out/bin/nixos-test-driver \ --set startScripts "''${vmStartScripts[*]}" \ --set testScript "$out/test-script" \ - --set vlans '${toString vlans}' + --set vlans '${toString vlans}' \ + ${lib.optionalString (interactive) "--add-flags --interactive"} ''); # Make a full-blown test @@ -217,6 +219,7 @@ rec { testName = name; qemu_pkg = pkgs.qemu; nodes = nodes pkgs.qemu; + interactive = true; }; test = @@ -224,7 +227,7 @@ rec { passMeta = drv: drv // lib.optionalAttrs (t ? meta) { meta = (drv.meta or { }) // t.meta; }; - in passMeta (runTests { inherit driver pos; }); + in passMeta (runTests { inherit driver pos driverInteractive; }); in test // { diff --git a/nixos/lib/utils.nix b/nixos/lib/utils.nix index bbebf8ba35a01..190c4db4d49d4 100644 --- a/nixos/lib/utils.nix +++ b/nixos/lib/utils.nix @@ -149,10 +149,16 @@ rec { if [[ -h '${output}' ]]; then rm '${output}' fi + + inherit_errexit_restore=$(shopt -p inherit_errexit) + shopt -s inherit_errexit '' + concatStringsSep "\n" - (imap1 (index: name: "export secret${toString index}=$(<'${secrets.${name}}')") + (imap1 (index: name: '' + secret${toString index}=$(<'${secrets.${name}}') + export secret${toString index} + '') (attrNames secrets)) + "\n" + "${pkgs.jq}/bin/jq >'${output}' '" @@ -164,6 +170,7 @@ rec { ' <<'EOF' ${builtins.toJSON set} EOF + $inherit_errexit_restore ''; systemdUtils = { |