diff options
author | nikstur <nikstur@outlook.com> | 2023-10-13 00:02:48 +0200 |
---|---|---|
committer | nikstur <nikstur@outlook.com> | 2024-01-22 00:54:13 +0100 |
commit | 60f529fc8277f1e49082a8c240cdfba6d5a5c794 (patch) | |
tree | 6f5fa595539b37204a1dc136147b95bd79a3e347 /nixos | |
parent | 4bcec20fa17c47cb4a9587ca247d6989ce6daf2b (diff) |
nixos/etc: optionally mount etc as an overlay
Diffstat (limited to 'nixos')
-rw-r--r-- | nixos/modules/system/etc/build-composefs-dump.py | 209 | ||||
-rwxr-xr-x | nixos/modules/system/etc/check-build-composefs-dump.sh | 8 | ||||
-rw-r--r-- | nixos/modules/system/etc/etc-activation.nix | 98 | ||||
-rw-r--r-- | nixos/modules/system/etc/etc.nix | 116 | ||||
-rw-r--r-- | nixos/tests/activation/etc-overlay-immutable.nix | 30 | ||||
-rw-r--r-- | nixos/tests/activation/etc-overlay-mutable.nix | 30 | ||||
-rw-r--r-- | nixos/tests/all-tests.nix | 2 |
7 files changed, 481 insertions, 12 deletions
diff --git a/nixos/modules/system/etc/build-composefs-dump.py b/nixos/modules/system/etc/build-composefs-dump.py new file mode 100644 index 0000000000000..923d40008b63f --- /dev/null +++ b/nixos/modules/system/etc/build-composefs-dump.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 + +"""Build a composefs dump from a Json config + +See the man page of composefs-dump for details about the format: +https://github.com/containers/composefs/blob/main/man/composefs-dump.md + +Ensure to check the file with the check script when you make changes to it: + +./check-build-composefs-dump.sh ./build-composefs_dump.py +""" + +import glob +import json +import os +import sys +from enum import Enum +from pathlib import Path +from typing import Any + +Attrs = dict[str, Any] + + +class FileType(Enum): + """The filetype as defined by the `st_mode` stat field in octal + + You can check the st_mode stat field of a path in Python with + `oct(os.stat("/path/").st_mode)` + """ + + directory = "4" + file = "10" + symlink = "12" + + +class ComposefsPath: + path: str + size: int + filetype: FileType + mode: str + uid: str + gid: str + payload: str + rdev: str = "0" + nlink: int = 1 + mtime: str = "1.0" + content: str = "-" + digest: str = "-" + + def __init__( + self, + attrs: Attrs, + size: int, + filetype: FileType, + mode: str, + payload: str, + path: str | None = None, + ): + if path is None: + path = attrs["target"] + self.path = "/" + path + self.size = size + self.filetype = filetype + self.mode = mode + self.uid = attrs["uid"] + self.gid = attrs["gid"] + self.payload = payload + + def write_line(self) -> str: + line_list = [ + str(self.path), + str(self.size), + f"{self.filetype.value}{self.mode}", + str(self.nlink), + str(self.uid), + str(self.gid), + str(self.rdev), + str(self.mtime), + str(self.payload), + str(self.content), + str(self.digest), + ] + return " ".join(line_list) + + +def eprint(*args, **kwargs) -> None: + print(args, **kwargs, file=sys.stderr) + + +def leading_directories(path: str) -> list[str]: + """Return the leading directories of path + + Given the path "alsa/conf.d/50-pipewire.conf", for example, this function + returns `[ "alsa", "alsa/conf.d" ]`. + """ + parents = list(Path(path).parents) + parents.reverse() + # remove the implicit `.` from the start of a relative path or `/` from an + # absolute path + del parents[0] + return [str(i) for i in parents] + + +def add_leading_directories( + target: str, attrs: Attrs, paths: dict[str, ComposefsPath] +) -> None: + """Add the leading directories of a target path to the composefs paths + + mkcomposefs expects that all leading directories are explicitly listed in + the dump file. Given the path "alsa/conf.d/50-pipewire.conf", for example, + this function adds "alsa" and "alsa/conf.d" to the composefs paths. + """ + path_components = leading_directories(target) + for component in path_components: + composefs_path = ComposefsPath( + attrs, + path=component, + size=4096, + filetype=FileType.directory, + mode="0755", + payload="-", + ) + paths[component] = composefs_path + + +def main() -> None: + """Build a composefs dump from a Json config + + This config describes the files that the final composefs image is supposed + to contain. + """ + config_file = sys.argv[1] + if not config_file: + eprint("No config file was supplied.") + sys.exit(1) + + with open(config_file, "rb") as f: + config = json.load(f) + + if not config: + eprint("Config is empty.") + sys.exit(1) + + eprint("Building composefs dump...") + + paths: dict[str, ComposefsPath] = {} + for attrs in config: + target = attrs["target"] + source = attrs["source"] + mode = attrs["mode"] + + if "*" in source: # Path with globbing + glob_sources = glob.glob(source) + for glob_source in glob_sources: + basename = os.path.basename(glob_source) + glob_target = f"{target}/{basename}" + + composefs_path = ComposefsPath( + attrs, + path=glob_target, + size=100, + filetype=FileType.symlink, + mode="0777", + payload=glob_source, + ) + + paths[glob_target] = composefs_path + add_leading_directories(glob_target, attrs, paths) + else: # Without globbing + if mode == "symlink": + composefs_path = ComposefsPath( + attrs, + # A high approximation of the size of a symlink + size=100, + filetype=FileType.symlink, + mode="0777", + payload=source, + ) + else: + if os.path.isdir(source): + composefs_path = ComposefsPath( + attrs, + size=4096, + filetype=FileType.directory, + mode=mode, + payload=source, + ) + else: + composefs_path = ComposefsPath( + attrs, + size=os.stat(source).st_size, + filetype=FileType.file, + mode=mode, + payload=target, + ) + paths[target] = composefs_path + add_leading_directories(target, attrs, paths) + + composefs_dump = ["/ 4096 40755 1 0 0 0 0.0 - - -"] # Root directory + for key in sorted(paths): + composefs_path = paths[key] + eprint(composefs_path.path) + composefs_dump.append(composefs_path.write_line()) + + print("\n".join(composefs_dump)) + + +if __name__ == "__main__": + main() diff --git a/nixos/modules/system/etc/check-build-composefs-dump.sh b/nixos/modules/system/etc/check-build-composefs-dump.sh new file mode 100755 index 0000000000000..da61651d1a5d6 --- /dev/null +++ b/nixos/modules/system/etc/check-build-composefs-dump.sh @@ -0,0 +1,8 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p black ruff mypy + +file=$1 + +black --check --diff $file +ruff --line-length 88 $file +mypy --strict $file diff --git a/nixos/modules/system/etc/etc-activation.nix b/nixos/modules/system/etc/etc-activation.nix index 7801049501860..f47fd771c6592 100644 --- a/nixos/modules/system/etc/etc-activation.nix +++ b/nixos/modules/system/etc/etc-activation.nix @@ -1,12 +1,96 @@ { config, lib, ... }: -let - inherit (lib) stringAfter; -in { + +{ imports = [ ./etc.nix ]; - config = { - system.activationScripts.etc = - stringAfter [ "users" "groups" ] config.system.build.etcActivationCommands; - }; + config = lib.mkMerge [ + + { + system.activationScripts.etc = + lib.stringAfter [ "users" "groups" ] config.system.build.etcActivationCommands; + } + + (lib.mkIf config.system.etc.overlay.enable { + + assertions = [ + { + assertion = config.boot.initrd.systemd.enable; + message = "`system.etc.overlay.enable` requires `boot.initrd.systemd.enable`"; + } + { + assertion = (!config.system.etc.overlay.mutable) -> config.systemd.sysusers.enable; + message = "`system.etc.overlay.mutable = false` requires `systemd.sysusers.enable`"; + } + { + assertion = lib.versionAtLeast config.boot.kernelPackages.kernel.version "6.6"; + message = "`system.etc.overlay.enable requires a newer kernel, at least version 6.6"; + } + { + assertion = config.systemd.sysusers.enable -> (config.users.mutableUsers == config.system.etc.overlay.mutable); + message = '' + When using systemd-sysusers and mounting `/etc` via an overlay, users + can only be mutable when `/etc` is mutable and vice versa. + ''; + } + ]; + + boot.initrd.availableKernelModules = [ "loop" "erofs" "overlay" ]; + + boot.initrd.systemd = { + mounts = [ + { + where = "/run/etc-metadata"; + what = "/sysroot${config.system.build.etcMetadataImage}"; + type = "erofs"; + options = "loop"; + unitConfig.RequiresMountsFor = [ + "/sysroot/nix/store" + ]; + } + { + where = "/sysroot/etc"; + what = "overlay"; + type = "overlay"; + options = lib.concatStringsSep "," ([ + "relatime" + "redirect_dir=on" + "metacopy=on" + "lowerdir=/run/etc-metadata::/sysroot${config.system.build.etcBasedir}" + ] ++ lib.optionals config.system.etc.overlay.mutable [ + "rw" + "upperdir=/sysroot/.rw-etc/upper" + "workdir=/sysroot/.rw-etc/work" + ] ++ lib.optionals (!config.system.etc.overlay.mutable) [ + "ro" + ]); + wantedBy = [ "initrd-fs.target" ]; + before = [ "initrd-fs.target" ]; + requires = lib.mkIf config.system.etc.overlay.mutable [ "rw-etc.service" ]; + after = lib.mkIf config.system.etc.overlay.mutable [ "rw-etc.service" ]; + unitConfig.RequiresMountsFor = [ + "/sysroot/nix/store" + "/run/etc-metadata" + ]; + } + ]; + services = lib.mkIf config.system.etc.overlay.mutable { + rw-etc = { + unitConfig = { + DefaultDependencies = false; + RequiresMountsFor = "/sysroot"; + }; + serviceConfig = { + Type = "oneshot"; + ExecStart = '' + /bin/mkdir -p -m 0755 /sysroot/.rw-etc/upper /sysroot/.rw-etc/work + ''; + }; + }; + }; + }; + + }) + + ]; } diff --git a/nixos/modules/system/etc/etc.nix b/nixos/modules/system/etc/etc.nix index ea61e7384e60c..baf37ba6def34 100644 --- a/nixos/modules/system/etc/etc.nix +++ b/nixos/modules/system/etc/etc.nix @@ -62,6 +62,16 @@ let ]) etc'} ''; + etcHardlinks = filter (f: f.mode != "symlink") etc'; + + build-composefs-dump = pkgs.runCommand "build-composefs-dump.py" + { + buildInputs = [ pkgs.python3 ]; + } '' + install ${./build-composefs-dump.py} $out + patchShebangs --host $out + ''; + in { @@ -72,6 +82,30 @@ in options = { + system.etc.overlay = { + enable = mkOption { + type = types.bool; + default = false; + description = lib.mdDoc '' + Mount `/etc` as an overlayfs instead of generating it via a perl script. + + Note: This is currently experimental. Only enable this option if you're + confident that you can recover your system if it breaks. + ''; + }; + + mutable = mkOption { + type = types.bool; + default = true; + description = lib.mdDoc '' + Whether to mount `/etc` mutably (i.e. read-write) or immutably (i.e. read-only). + + If this is false, only the immutable lowerdir is mounted. If it is + true, a writable upperdir is mounted on top. + ''; + }; + }; + environment.etc = mkOption { default = {}; example = literalExpression '' @@ -190,12 +224,84 @@ in config = { system.build.etc = etc; - system.build.etcActivationCommands = - '' - # Set up the statically computed bits of /etc. - echo "setting up /etc..." - ${pkgs.perl.withPackages (p: [ p.FileSlurp ])}/bin/perl ${./setup-etc.pl} ${etc}/etc + system.build.etcActivationCommands = let + etcOverlayOptions = lib.concatStringsSep "," ([ + "relatime" + "redirect_dir=on" + "metacopy=on" + ] ++ lib.optionals config.system.etc.overlay.mutable [ + "upperdir=/.rw-etc/upper" + "workdir=/.rw-etc/work" + ]); + in if config.system.etc.overlay.enable then '' + # This script atomically remounts /etc when switching configuration. On a (re-)boot + # this should not run because /etc is mounted via a systemd mount unit + # instead. To a large extent this mimics what composefs does. Because + # it's relatively simple, however, we avoid the composefs dependency. + if [[ ! $IN_NIXOS_SYSTEMD_STAGE1 ]]; then + echo "remounting /etc..." + + tmpMetadataMount=$(mktemp --directory) + mount --type erofs ${config.system.build.etcMetadataImage} $tmpMetadataMount + + # Mount the new /etc overlay to a temporary private mount. + # This needs the indirection via a private bind mount because you + # cannot move shared mounts. + tmpEtcMount=$(mktemp --directory) + mount --bind --make-private $tmpEtcMount $tmpEtcMount + mount --type overlay overlay \ + --options lowerdir=$tmpMetadataMount::${config.system.build.etcBasedir},${etcOverlayOptions} \ + $tmpEtcMount + + # Move the new temporary /etc mount underneath the current /etc mount. + # + # This should eventually use util-linux to perform this move beneath, + # however, this functionality is not yet in util-linux. See this + # tracking issue: https://github.com/util-linux/util-linux/issues/2604 + ${pkgs.move-mount-beneath}/bin/move-mount --move --beneath $tmpEtcMount /etc + + # Unmount the top /etc mount to atomically reveal the new mount. + umount /etc + + fi + '' else '' + # Set up the statically computed bits of /etc. + echo "setting up /etc..." + ${pkgs.perl.withPackages (p: [ p.FileSlurp ])}/bin/perl ${./setup-etc.pl} ${etc}/etc + ''; + + system.build.etcBasedir = pkgs.runCommandLocal "etc-lowerdir" { } '' + set -euo pipefail + + makeEtcEntry() { + src="$1" + target="$2" + + mkdir -p "$out/$(dirname "$target")" + cp "$src" "$out/$target" + } + + mkdir -p "$out" + ${concatMapStringsSep "\n" (etcEntry: escapeShellArgs [ + "makeEtcEntry" + # Force local source paths to be added to the store + "${etcEntry.source}" + etcEntry.target + ]) etcHardlinks} + ''; + + system.build.etcMetadataImage = + let + etcJson = pkgs.writeText "etc-json" (builtins.toJSON etc'); + etcDump = pkgs.runCommand "etc-dump" { } "${build-composefs-dump} ${etcJson} > $out"; + in + pkgs.runCommand "etc-metadata.erofs" { + nativeBuildInputs = [ pkgs.composefs pkgs.erofs-utils ]; + } '' + mkcomposefs --from-file ${etcDump} $out + fsck.erofs $out ''; + }; } diff --git a/nixos/tests/activation/etc-overlay-immutable.nix b/nixos/tests/activation/etc-overlay-immutable.nix new file mode 100644 index 0000000000000..70c3623b929c5 --- /dev/null +++ b/nixos/tests/activation/etc-overlay-immutable.nix @@ -0,0 +1,30 @@ +{ lib, ... }: { + + name = "activation-etc-overlay-immutable"; + + meta.maintainers = with lib.maintainers; [ nikstur ]; + + nodes.machine = { pkgs, ... }: { + system.etc.overlay.enable = true; + system.etc.overlay.mutable = false; + + # Prerequisites + systemd.sysusers.enable = true; + users.mutableUsers = false; + boot.initrd.systemd.enable = true; + boot.kernelPackages = pkgs.linuxPackages_latest; + + specialisation.new-generation.configuration = { + environment.etc."newgen".text = "newgen"; + }; + }; + + testScript = '' + machine.succeed("findmnt --kernel --type overlay /etc") + machine.fail("stat /etc/newgen") + + machine.succeed("/run/current-system/specialisation/new-generation/bin/switch-to-configuration switch") + + assert machine.succeed("cat /etc/newgen") == "newgen" + ''; +} diff --git a/nixos/tests/activation/etc-overlay-mutable.nix b/nixos/tests/activation/etc-overlay-mutable.nix new file mode 100644 index 0000000000000..cfe7604fceb84 --- /dev/null +++ b/nixos/tests/activation/etc-overlay-mutable.nix @@ -0,0 +1,30 @@ +{ lib, ... }: { + + name = "activation-etc-overlay-mutable"; + + meta.maintainers = with lib.maintainers; [ nikstur ]; + + nodes.machine = { pkgs, ... }: { + system.etc.overlay.enable = true; + system.etc.overlay.mutable = true; + + # Prerequisites + boot.initrd.systemd.enable = true; + boot.kernelPackages = pkgs.linuxPackages_latest; + + specialisation.new-generation.configuration = { + environment.etc."newgen".text = "newgen"; + }; + }; + + testScript = '' + machine.succeed("findmnt --kernel --type overlay /etc") + machine.fail("stat /etc/newgen") + machine.succeed("echo -n 'mutable' > /etc/mutable") + + machine.succeed("/run/current-system/specialisation/new-generation/bin/switch-to-configuration switch") + + assert machine.succeed("cat /etc/newgen") == "newgen" + assert machine.succeed("cat /etc/mutable") == "mutable" + ''; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 5abb85c224bbc..ab2cf8c4e2f51 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -285,6 +285,8 @@ in { activation = pkgs.callPackage ../modules/system/activation/test.nix { }; activation-var = runTest ./activation/var.nix; activation-nix-channel = runTest ./activation/nix-channel.nix; + activation-etc-overlay-mutable = runTest ./activation/etc-overlay-mutable.nix; + activation-etc-overlay-immutable = runTest ./activation/etc-overlay-immutable.nix; etcd = handleTestOn ["x86_64-linux"] ./etcd.nix {}; etcd-cluster = handleTestOn ["x86_64-linux"] ./etcd-cluster.nix {}; etebase-server = handleTest ./etebase-server.nix {}; |