diff options
author | github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> | 2023-10-22 00:02:31 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-10-22 00:02:31 +0000 |
commit | 1cf369a0dd162e2da454edd7880eb87bd4543eef (patch) | |
tree | a3d0ef6fd83f6657e7f064c2d8eb437e61b204b3 /nixos | |
parent | a01279cf2329f710bb84f3aa64d32ea87a8c4124 (diff) | |
parent | 76d4d2e76b2b8281aa297d164a9eb410bf81319a (diff) |
Merge master into staging-next
Diffstat (limited to 'nixos')
-rw-r--r-- | nixos/doc/manual/release-notes/rl-2311.section.md | 4 | ||||
-rw-r--r-- | nixos/modules/config/fanout.nix | 49 | ||||
-rw-r--r-- | nixos/modules/module-list.nix | 2 | ||||
-rw-r--r-- | nixos/modules/programs/firefox.nix | 10 | ||||
-rw-r--r-- | nixos/modules/services/cluster/hadoop/default.nix | 18 | ||||
-rw-r--r-- | nixos/modules/services/cluster/hadoop/yarn.nix | 2 | ||||
-rw-r--r-- | nixos/modules/services/misc/soft-serve.nix | 99 | ||||
-rw-r--r-- | nixos/modules/virtualisation/qemu-vm.nix | 1 | ||||
-rw-r--r-- | nixos/tests/all-tests.nix | 2 | ||||
-rw-r--r-- | nixos/tests/fanout.nix | 30 | ||||
-rw-r--r-- | nixos/tests/hadoop/hadoop.nix | 2 | ||||
-rw-r--r-- | nixos/tests/soft-serve.nix | 102 | ||||
-rw-r--r-- | nixos/tests/zfs.nix | 4 |
13 files changed, 305 insertions, 20 deletions
diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index 91e0eb2020215..688d7036d4582 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -117,6 +117,8 @@ - [virt-manager](https://virt-manager.org/), an UI for managing virtual machines in libvirt, is now available as `programs.virt-manager`. +- [Soft Serve](https://github.com/charmbracelet/soft-serve), a tasty, self-hostable Git server for the command line. Available as [services.soft-serve](#opt-services.soft-serve.enable). + ## Backward Incompatibilities {#sec-release-23.11-incompatibilities} - `network-online.target` has been fixed to no longer time out for systems with `networking.useDHCP = true` and `networking.useNetworkd = true`. @@ -345,6 +347,8 @@ - `jq` was updated to 1.7, its [first release in 5 years](https://github.com/jqlang/jq/releases/tag/jq-1.7). +- `zfs` was updated from 2.1.x to 2.2.0, [enabling newer kernel support and adding new features](https://github.com/openzfs/zfs/releases/tag/zfs-2.2.0). + - A new option was added to the virtualisation module that enables specifying explicitly named network interfaces in QEMU VMs. The existing `virtualisation.vlans` is still supported for cases where the name of the network interface is irrelevant. - DocBook option documentation is no longer supported, all module documentation now uses markdown. diff --git a/nixos/modules/config/fanout.nix b/nixos/modules/config/fanout.nix new file mode 100644 index 0000000000000..60ee145f19af4 --- /dev/null +++ b/nixos/modules/config/fanout.nix @@ -0,0 +1,49 @@ +{ config, lib, pkgs, ... }: +let + cfg = config.services.fanout; + mknodCmds = n: lib.lists.imap0 (i: s: + "mknod /dev/fanout${builtins.toString i} c $MAJOR ${builtins.toString i}" + ) (lib.lists.replicate n ""); +in +{ + options.services.fanout = { + enable = lib.mkEnableOption (lib.mdDoc "fanout"); + fanoutDevices = lib.mkOption { + type = lib.types.int; + default = 1; + description = "Number of /dev/fanout devices"; + }; + bufferSize = lib.mkOption { + type = lib.types.int; + default = 16384; + description = "Size of /dev/fanout buffer in bytes"; + }; + }; + + config = lib.mkIf cfg.enable { + boot.extraModulePackages = [ config.boot.kernelPackages.fanout.out ]; + + boot.kernelModules = [ "fanout" ]; + + boot.extraModprobeConfig = '' + options fanout buffersize=${builtins.toString cfg.bufferSize} + ''; + + systemd.services.fanout = { + description = "Bring up /dev/fanout devices"; + script = '' + MAJOR=$(${pkgs.gnugrep}/bin/grep fanout /proc/devices | ${pkgs.gawk}/bin/awk '{print $1}') + ${lib.strings.concatLines (mknodCmds cfg.fanoutDevices)} + ''; + + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Type = "oneshot"; + User = "root"; + RemainAfterExit = "yes"; + Restart = "no"; + }; + }; + }; +} diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 58b06b0a1e6d1..395a638033fe1 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -2,6 +2,7 @@ ./config/appstream.nix ./config/console.nix ./config/debug-info.nix + ./config/fanout.nix ./config/fonts/fontconfig.nix ./config/fonts/fontdir.nix ./config/fonts/ghostscript.nix @@ -730,6 +731,7 @@ ./services/misc/signald.nix ./services/misc/siproxd.nix ./services/misc/snapper.nix + ./services/misc/soft-serve.nix ./services/misc/sonarr.nix ./services/misc/sourcehut ./services/misc/spice-vdagentd.nix diff --git a/nixos/modules/programs/firefox.nix b/nixos/modules/programs/firefox.nix index 99236f01c5370..813e0e0105f69 100644 --- a/nixos/modules/programs/firefox.nix +++ b/nixos/modules/programs/firefox.nix @@ -224,11 +224,11 @@ in extraPrefs = cfg.autoConfig; extraNativeMessagingHosts = old.extraNativeMessagingHosts or [] - ++ optional nmh.ff2mpv ff2mpv - ++ optional nmh.euwebid web-eid-app - ++ optional nmh.gsconnect gnomeExtensions.gsconnect - ++ optional nmh.jabref jabref - ++ optional nmh.passff passff-host; + ++ optional nmh.ff2mpv pkgs.ff2mpv + ++ optional nmh.euwebid pkgs.web-eid-app + ++ optional nmh.gsconnect pkgs.gnomeExtensions.gsconnect + ++ optional nmh.jabref pkgs.jabref + ++ optional nmh.passff pkgs.passff-host; cfg = let # copy-pasted from the wrapper; TODO: figure out fix applicationName = cfg.package.binaryName or (lib.getName cfg.package); diff --git a/nixos/modules/services/cluster/hadoop/default.nix b/nixos/modules/services/cluster/hadoop/default.nix index 72bf25c211461..ff6b4d5588b13 100644 --- a/nixos/modules/services/cluster/hadoop/default.nix +++ b/nixos/modules/services/cluster/hadoop/default.nix @@ -67,16 +67,16 @@ with lib; mapredSiteDefault = mkOption { default = { "mapreduce.framework.name" = "yarn"; - "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}"; - "mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}"; - "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}"; + "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}"; + "mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}"; + "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}"; }; defaultText = literalExpression '' { "mapreduce.framework.name" = "yarn"; - "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}"; - "mapreduce.map.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}"; - "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}"; + "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}"; + "mapreduce.map.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}"; + "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}"; } ''; type = types.attrsOf types.anything; @@ -154,13 +154,13 @@ with lib; }; log4jProperties = mkOption { - default = "${cfg.package}/lib/${cfg.package.untarDir}/etc/hadoop/log4j.properties"; + default = "${cfg.package}/etc/hadoop/log4j.properties"; defaultText = literalExpression '' - "''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}/etc/hadoop/log4j.properties" + "''${config.${opt.package}}/etc/hadoop/log4j.properties" ''; type = types.path; example = literalExpression '' - "''${pkgs.hadoop}/lib/''${pkgs.hadoop.untarDir}/etc/hadoop/log4j.properties"; + "''${pkgs.hadoop}/etc/hadoop/log4j.properties"; ''; description = lib.mdDoc "log4j.properties file added to HADOOP_CONF_DIR"; }; diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix index 26077f35fdd0d..a49aafbd1dca8 100644 --- a/nixos/modules/services/cluster/hadoop/yarn.nix +++ b/nixos/modules/services/cluster/hadoop/yarn.nix @@ -160,7 +160,7 @@ in umount /run/wrappers/yarn-nodemanager/cgroup/cpu || true rm -rf /run/wrappers/yarn-nodemanager/ || true mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop,cgroup/cpu} - cp ${cfg.package}/lib/${cfg.package.untarDir}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/ + cp ${cfg.package}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/ chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor cp ${hadoopConf}/container-executor.cfg /run/wrappers/yarn-nodemanager/etc/hadoop/ diff --git a/nixos/modules/services/misc/soft-serve.nix b/nixos/modules/services/misc/soft-serve.nix new file mode 100644 index 0000000000000..0f246493880b9 --- /dev/null +++ b/nixos/modules/services/misc/soft-serve.nix @@ -0,0 +1,99 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.soft-serve; + configFile = format.generate "config.yaml" cfg.settings; + format = pkgs.formats.yaml { }; + docUrl = "https://charm.sh/blog/self-hosted-soft-serve/"; + stateDir = "/var/lib/soft-serve"; +in +{ + options = { + services.soft-serve = { + enable = mkEnableOption "Enable soft-serve service"; + + package = mkPackageOption pkgs "soft-serve" { }; + + settings = mkOption { + type = format.type; + default = { }; + description = mdDoc '' + The contents of the configuration file. + + See <${docUrl}>. + ''; + example = literalExpression '' + { + name = "dadada's repos"; + log_format = "text"; + ssh = { + listen_addr = ":23231"; + public_url = "ssh://localhost:23231"; + max_timeout = 30; + idle_timeout = 120; + }; + stats.listen_addr = ":23233"; + } + ''; + }; + }; + }; + + config = mkIf cfg.enable { + + systemd.tmpfiles.rules = [ + # The config file has to be inside the state dir + "L+ ${stateDir}/config.yaml - - - - ${configFile}" + ]; + + systemd.services.soft-serve = { + description = "Soft Serve git server"; + documentation = [ docUrl ]; + requires = [ "network-online.target" ]; + after = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment.SOFT_SERVE_DATA_PATH = stateDir; + + serviceConfig = { + Type = "simple"; + DynamicUser = true; + Restart = "always"; + ExecStart = "${getExe cfg.package} serve"; + StateDirectory = "soft-serve"; + WorkingDirectory = stateDir; + RuntimeDirectory = "soft-serve"; + RuntimeDirectoryMode = "0750"; + ProcSubset = "pid"; + ProtectProc = "invisible"; + UMask = "0027"; + CapabilityBoundingSet = ""; + ProtectHome = true; + PrivateDevices = true; + PrivateUsers = true; + ProtectHostname = true; + ProtectClock = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectControlGroups = true; + RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ]; + RestrictNamespaces = true; + LockPersonality = true; + MemoryDenyWriteExecute = true; + RestrictRealtime = true; + RemoveIPC = true; + PrivateMounts = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~@cpu-emulation @debug @keyring @module @mount @obsolete @privileged @raw-io @reboot @setuid @swap" + ]; + }; + }; + }; + + meta.maintainers = [ maintainers.dadada ]; +} diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index e0004df6f6b2f..737a935711ae6 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -267,6 +267,7 @@ let }; storeImage = import ../../lib/make-disk-image.nix { + name = "nix-store-image"; inherit pkgs config lib; additionalPaths = [ regInfo ]; format = "qcow2"; diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 88fcbd59a1e71..ef98efd7dbcaa 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -272,6 +272,7 @@ in { fail2ban = handleTest ./fail2ban.nix { }; fakeroute = handleTest ./fakeroute.nix {}; fancontrol = handleTest ./fancontrol.nix {}; + fanout = handleTest ./fanout.nix {}; fcitx5 = handleTest ./fcitx5 {}; fenics = handleTest ./fenics.nix {}; ferm = handleTest ./ferm.nix {}; @@ -733,6 +734,7 @@ in { snapper = handleTest ./snapper.nix {}; snipe-it = runTest ./web-apps/snipe-it.nix; soapui = handleTest ./soapui.nix {}; + soft-serve = handleTest ./soft-serve.nix {}; sogo = handleTest ./sogo.nix {}; solanum = handleTest ./solanum.nix {}; sonarr = handleTest ./sonarr.nix {}; diff --git a/nixos/tests/fanout.nix b/nixos/tests/fanout.nix new file mode 100644 index 0000000000000..c36d34dcce0be --- /dev/null +++ b/nixos/tests/fanout.nix @@ -0,0 +1,30 @@ +{ system ? builtins.currentSystem +, config ? {} +, pkgs ? import ../.. { inherit system config; } +}: +import ./make-test-python.nix ({lib, pkgs, ...}: { + name = "fanout"; + meta.maintainers = [ lib.maintainers.therishidesai ]; + + nodes = let + cfg = { ... }: { + services.fanout = { + enable = true; + fanoutDevices = 2; + bufferSize = 8192; + }; + }; + in { + machine = cfg; + }; + + testScript = '' + start_all() + + # mDNS. + machine.wait_for_unit("multi-user.target") + + machine.succeed("test -c /dev/fanout0") + machine.succeed("test -c /dev/fanout1") + ''; +}) diff --git a/nixos/tests/hadoop/hadoop.nix b/nixos/tests/hadoop/hadoop.nix index b132f4fa58b01..0de2366b1864c 100644 --- a/nixos/tests/hadoop/hadoop.nix +++ b/nixos/tests/hadoop/hadoop.nix @@ -249,7 +249,7 @@ import ../make-test-python.nix ({ package, ... }: { assert "standby" in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState") client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat") - assert "Estimated value of Pi is" in client.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~lib/hadoop-*/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10") + assert "Estimated value of Pi is" in client.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10") assert "SUCCEEDED" in client.succeed("yarn application -list -appStates FINISHED") ''; }) diff --git a/nixos/tests/soft-serve.nix b/nixos/tests/soft-serve.nix new file mode 100644 index 0000000000000..1c4cb4c95819e --- /dev/null +++ b/nixos/tests/soft-serve.nix @@ -0,0 +1,102 @@ +import ./make-test-python.nix ({ pkgs, lib, ... }: +let + inherit (import ./ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey; + sshPort = 8231; + httpPort = 8232; + statsPort = 8233; + gitPort = 8418; +in +{ + name = "soft-serve"; + meta.maintainers = with lib.maintainers; [ dadada ]; + nodes = { + client = { pkgs, ... }: { + environment.systemPackages = with pkgs; [ + curl + git + openssh + ]; + environment.etc.sshKey = { + source = snakeOilPrivateKey; + mode = "0600"; + }; + }; + + server = + { config, ... }: + { + services.soft-serve = { + enable = true; + settings = { + name = "TestServer"; + ssh.listen_addr = ":${toString sshPort}"; + git.listen_addr = ":${toString gitPort}"; + http.listen_addr = ":${toString httpPort}"; + stats.listen_addr = ":${toString statsPort}"; + initial_admin_keys = [ snakeOilPublicKey ]; + }; + }; + networking.firewall.allowedTCPPorts = [ sshPort httpPort statsPort ]; + }; + }; + + testScript = + { ... }: + '' + SSH_PORT = ${toString sshPort} + HTTP_PORT = ${toString httpPort} + STATS_PORT = ${toString statsPort} + KEY = "${snakeOilPublicKey}" + SSH_KEY = "/etc/sshKey" + SSH_COMMAND = f"ssh -p {SSH_PORT} -i {SSH_KEY} -o StrictHostKeyChecking=no" + TEST_DIR = "/tmp/test" + GIT = f"git -C {TEST_DIR}" + + for machine in client, server: + machine.wait_for_unit("network.target") + + server.wait_for_unit("soft-serve.service") + server.wait_for_open_port(SSH_PORT) + + with subtest("Get info"): + status, test = client.execute(f"{SSH_COMMAND} server info") + if status != 0: + raise Exception("Failed to get SSH info") + key = " ".join(KEY.split(" ")[0:2]) + if not key in test: + raise Exception("Admin key must be configured correctly") + + with subtest("Create user"): + client.succeed(f"{SSH_COMMAND} server user create beatrice") + client.succeed(f"{SSH_COMMAND} server user info beatrice") + + with subtest("Create repo"): + client.succeed(f"git init {TEST_DIR}") + client.succeed(f"{GIT} config --global user.email you@example.com") + client.succeed(f"touch {TEST_DIR}/foo") + client.succeed(f"{GIT} add foo") + client.succeed(f"{GIT} commit --allow-empty -m test") + client.succeed(f"{GIT} remote add origin git@server:test") + client.succeed(f"GIT_SSH_COMMAND='{SSH_COMMAND}' {GIT} push -u origin master") + client.execute("rm -r /tmp/test") + + server.wait_for_open_port(HTTP_PORT) + + with subtest("Clone over HTTP"): + client.succeed(f"curl --connect-timeout 10 http://server:{HTTP_PORT}/") + client.succeed(f"git clone http://server:{HTTP_PORT}/test /tmp/test") + client.execute("rm -r /tmp/test") + + with subtest("Clone over SSH"): + client.succeed(f"GIT_SSH_COMMAND='{SSH_COMMAND}' git clone git@server:test /tmp/test") + client.execute("rm -r /tmp/test") + + with subtest("Get stats over HTTP"): + server.wait_for_open_port(STATS_PORT) + status, test = client.execute(f"curl --connect-timeout 10 http://server:{STATS_PORT}/metrics") + if status != 0: + raise Exception("Failed to get metrics from status port") + if not "go_gc_duration_seconds_count" in test: + raise Exception("Metrics did not contain key 'go_gc_duration_seconds_count'") + ''; +}) diff --git a/nixos/tests/zfs.nix b/nixos/tests/zfs.nix index 800f5e43cd15c..3454fbaf78fe5 100644 --- a/nixos/tests/zfs.nix +++ b/nixos/tests/zfs.nix @@ -113,8 +113,6 @@ let }; testScript = '' - # TODO: Remove this when upgrading stable to zfs 2.2.0 - unstable = ${if enableUnstable then "True" else "False"}; machine.wait_for_unit("multi-user.target") machine.succeed( "zpool status", @@ -136,8 +134,6 @@ let machine.crash() machine.wait_for_unit("multi-user.target") machine.succeed("zfs set sharesmb=on rpool/shared_smb") - if not unstable: - machine.succeed("zfs share rpool/shared_smb") machine.succeed( "smbclient -gNL localhost | grep rpool_shared_smb", "umount /tmp/mnt", |