Merge branch 'master' into staging-next

8 files changed, 160 insertions, 192 deletions

diff --git a/nixos/doc/manual/release-notes/rl-2405.section.md b/nixos/doc/manual/release-notes/rl-2405.section.md
index d4fa7e4aea3f2..082db8301a5e9 100644
--- a/nixos/doc/manual/release-notes/rl-2405.section.md
+++ b/nixos/doc/manual/release-notes/rl-2405.section.md
@@ -48,9 +48,7 @@ Use `services.pipewire.extraConfig` or `services.pipewire.configPackages` for Pi
 
 - GNOME has been updated to v46. Refer to the [release notes](https://release.gnome.org/46/) for more details. Notably this release brings experimental VRR support, default GTK renderer changes and WebDAV support in Online Accounts. This release we have also stopped including the legacy and unsupported Adwaita-Dark theme by default.
 
-- A new option `virtualisation.containers.cdi` was added. It contains `static` and `dynamic` attributes (corresponding to `/etc/cdi` and `/run/cdi` respectively) to configure the Container Device Interface (CDI).
-
-- `virtualisation.docker.enableNvidia` and `virtualisation.podman.enableNvidia` options are deprecated. `virtualisation.containers.cdi.dynamic.nvidia.enable` should be used instead. This option will expose GPUs on containers with the `--device` CLI option. This is supported by Docker 25, Podman 3.2.0 and Singularity 4. Any container runtime that supports the CDI specification will take advantage of this feature.
+- `virtualisation.docker.enableNvidia` and `virtualisation.podman.enableNvidia` options are deprecated. `hardware.nvidia-container-toolkit.enable` should be used instead. This option will expose GPUs on containers with the `--device` CLI option. This is supported by Docker 25, Podman 3.2.0 and Singularity 4. Any container runtime that supports the CDI specification will take advantage of this feature.
 
 - `system.etc.overlay.enable` option was added. If enabled, `/etc` is
   mounted via an overlayfs instead of being created by a custom perl script.
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index 511d991e919cd..50157f62ed320 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -559,7 +559,7 @@
   ./services/hardware/kanata.nix
   ./services/hardware/lcd.nix
   ./services/hardware/lirc.nix
-  ./services/hardware/nvidia-container-toolkit-cdi-generator
+  ./services/hardware/nvidia-container-toolkit
   ./services/hardware/monado.nix
   ./services/hardware/nvidia-optimus.nix
   ./services/hardware/openrgb.nix
diff --git a/nixos/modules/services/editors/emacs.md b/nixos/modules/services/editors/emacs.md
index 885f927422bdc..da10286751554 100644
--- a/nixos/modules/services/editors/emacs.md
+++ b/nixos/modules/services/editors/emacs.md
@@ -366,44 +366,3 @@ convenient if you regularly edit Nix files.
 You can use `woman` to get completion of all available
 man pages. For example, type `M-x woman <RET> nixos-rebuild <RET>.`
 
-### Editing DocBook 5 XML Documents {#sec-emacs-docbook-xml}
-
-Emacs includes
-[nXML](https://www.gnu.org/software/emacs/manual/html_node/nxml-mode/Introduction.html),
-a major-mode for validating and editing XML documents. When editing DocBook
-5.0 documents, such as [this one](#book-nixos-manual),
-nXML needs to be configured with the relevant schema, which is not
-included.
-
-To install the DocBook 5.0 schemas, either add
-{var}`pkgs.docbook5` to [](#opt-environment.systemPackages)
-([NixOS](#sec-declarative-package-mgmt)), or run
-`nix-env -f '<nixpkgs>' -iA docbook5`
-([Nix](#sec-ad-hoc-packages)).
-
-Then customize the variable {var}`rng-schema-locating-files` to
-include {file}`~/.emacs.d/schemas.xml` and put the following
-text into that file:
-::: {.example #ex-emacs-docbook-xml}
-### nXML Schema Configuration (`~/.emacs.d/schemas.xml`)
-
-```xml
-<?xml version="1.0"?>
-<!--
-  To let emacs find this file, evaluate:
-  (add-to-list 'rng-schema-locating-files "~/.emacs.d/schemas.xml")
--->
-<locatingRules xmlns="http://thaiopensource.com/ns/locating-rules/1.0">
-  <!--
-    Use this variation if pkgs.docbook5 is added to environment.systemPackages
-  -->
-  <namespace ns="http://docbook.org/ns/docbook"
-             uri="/run/current-system/sw/share/xml/docbook-5.0/rng/docbookxi.rnc"/>
-  <!--
-    Use this variation if installing schema with "nix-env -iA pkgs.docbook5".
-  <namespace ns="http://docbook.org/ns/docbook"
-             uri="../.nix-profile/share/xml/docbook-5.0/rng/docbookxi.rnc"/>
-  -->
-</locatingRules>
-```
-:::
diff --git a/nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/cdi-generate.nix b/nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/cdi-generate.nix
deleted file mode 100644
index 1aaa2d07b9bde..0000000000000
--- a/nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/cdi-generate.nix
+++ /dev/null
@@ -1,60 +0,0 @@
-{
-  addDriverRunpath,
-  glibc,
-  jq,
-  lib,
-  nvidia-container-toolkit,
-  nvidia-driver,
-  runtimeShell,
-  writeScriptBin,
-}:
-let
-  mountOptions = { options = ["ro" "nosuid" "nodev" "bind"]; };
-  mounts = [
-    # FIXME: Making /usr mounts optional
-    { hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-control";
-      containerPath = "/usr/bin/nvidia-cuda-mps-control"; }
-    { hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-server";
-      containerPath = "/usr/bin/nvidia-cuda-mps-server"; }
-    { hostPath = lib.getExe' nvidia-driver "nvidia-debugdump";
-      containerPath = "/usr/bin/nvidia-debugdump"; }
-    { hostPath = lib.getExe' nvidia-driver "nvidia-powerd";
-      containerPath = "/usr/bin/nvidia-powerd"; }
-    { hostPath = lib.getExe' nvidia-driver "nvidia-smi";
-      containerPath = "/usr/bin/nvidia-smi"; }
-    { hostPath = lib.getExe' nvidia-container-toolkit "nvidia-ctk";
-      containerPath = "/usr/bin/nvidia-ctk"; }
-    { hostPath = "${lib.getLib glibc}/lib";
-      containerPath = "${lib.getLib glibc}/lib"; }
-
-    # FIXME: use closureinfo
-    {
-      hostPath = addDriverRunpath.driverLink;
-      containerPath = addDriverRunpath.driverLink;
-    }
-    { hostPath = "${lib.getLib glibc}/lib";
-      containerPath = "${lib.getLib glibc}/lib"; }
-    { hostPath = "${lib.getLib glibc}/lib64";
-      containerPath = "${lib.getLib glibc}/lib64"; }
-  ];
-  jqAddMountExpression = ".containerEdits.mounts[.containerEdits.mounts | length] |= . +";
-  mountsToJq = lib.concatMap
-    (mount:
-      ["${lib.getExe jq} '${jqAddMountExpression} ${builtins.toJSON (mount // mountOptions)}'"])
-    mounts;
-in
-writeScriptBin "nvidia-cdi-generator"
-''
-#! ${runtimeShell}
-
-function cdiGenerate {
-  ${lib.getExe' nvidia-container-toolkit "nvidia-ctk"} cdi generate \
-    --format json \
-    --ldconfig-path ${lib.getExe' glibc "ldconfig"} \
-    --library-search-path ${lib.getLib nvidia-driver}/lib \
-    --nvidia-ctk-path ${lib.getExe' nvidia-container-toolkit "nvidia-ctk"}
-}
-
-cdiGenerate | \
-  ${lib.concatStringsSep " | " mountsToJq} > $RUNTIME_DIRECTORY/nvidia-container-toolkit.json
-''
diff --git a/nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/default.nix b/nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/default.nix
deleted file mode 100644
index 5aa3c72ee0a06..0000000000000
--- a/nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/default.nix
+++ /dev/null
@@ -1,40 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-{
-
-  options = {
-
-    hardware.nvidia-container-toolkit-cdi-generator.enable = lib.mkOption {
-      default = false;
-      internal = true;
-      visible = false;
-      type = lib.types.bool;
-      description = ''
-        Enable dynamic CDI configuration for NVidia devices by running
-        nvidia-container-toolkit on boot.
-      '';
-    };
-
-  };
-
-  config = {
-
-    systemd.services.nvidia-container-toolkit-cdi-generator = lib.mkIf config.hardware.nvidia-container-toolkit-cdi-generator.enable {
-      description = "Container Device Interface (CDI) for Nvidia generator";
-      wantedBy = [ "multi-user.target" ];
-      after = [ "systemd-udev-settle.service" ];
-      serviceConfig = {
-        RuntimeDirectory = "cdi";
-        RemainAfterExit = true;
-        ExecStart =
-          let
-            script = pkgs.callPackage ./cdi-generate.nix { nvidia-driver = config.hardware.nvidia.package; };
-          in
-          lib.getExe script;
-        Type = "oneshot";
-      };
-    };
-
-  };
-
-}
diff --git a/nixos/modules/services/hardware/nvidia-container-toolkit/cdi-generate.nix b/nixos/modules/services/hardware/nvidia-container-toolkit/cdi-generate.nix
new file mode 100644
index 0000000000000..ca769cc44e5c9
--- /dev/null
+++ b/nixos/modules/services/hardware/nvidia-container-toolkit/cdi-generate.nix
@@ -0,0 +1,35 @@
+{
+  glibc,
+  jq,
+  lib,
+  mounts,
+  nvidia-container-toolkit,
+  nvidia-driver,
+  runtimeShell,
+  writeScriptBin,
+}: let
+  mkMount = {hostPath, containerPath, mountOptions}: {
+    inherit hostPath containerPath;
+    options = mountOptions;
+  };
+  jqAddMountExpression = ".containerEdits.mounts[.containerEdits.mounts | length] |= . +";
+  allJqMounts = lib.concatMap
+    (mount:
+      ["${lib.getExe jq} '${jqAddMountExpression} ${builtins.toJSON (mkMount mount)}'"])
+    mounts;
+in
+writeScriptBin "nvidia-cdi-generator"
+''
+#! ${runtimeShell}
+
+function cdiGenerate {
+  ${lib.getExe' nvidia-container-toolkit "nvidia-ctk"} cdi generate \
+    --format json \
+    --ldconfig-path ${lib.getExe' glibc "ldconfig"} \
+    --library-search-path ${lib.getLib nvidia-driver}/lib \
+    --nvidia-ctk-path ${lib.getExe' nvidia-container-toolkit "nvidia-ctk"}
+}
+
+cdiGenerate | \
+  ${lib.concatStringsSep " | " allJqMounts} > $RUNTIME_DIRECTORY/nvidia-container-toolkit.json
+''
diff --git a/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix b/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix
new file mode 100644
index 0000000000000..7b4973d3c6b0c
--- /dev/null
+++ b/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix
@@ -0,0 +1,121 @@
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [
+    (lib.mkRenamedOptionModule
+      [ "virtualisation" "containers" "cdi" "dynamic" "nvidia" "enable" ]
+      [ "hardware" "nvidia-container-toolkit" "enable" ])
+  ];
+
+  options = let
+    mountType = {
+      options = {
+        hostPath = lib.mkOption {
+          type = lib.types.str;
+          description = "Host path.";
+        };
+        containerPath = lib.mkOption {
+          type = lib.types.str;
+          description = "Container path.";
+        };
+        mountOptions = lib.mkOption {
+          default = [ "ro" "nosuid" "nodev" "bind" ];
+          type = lib.types.listOf lib.types.str;
+          description = "Mount options.";
+        };
+      };
+    };
+  in {
+
+    hardware.nvidia-container-toolkit = {
+      enable = lib.mkOption {
+        default = false;
+        type = lib.types.bool;
+        description = ''
+          Enable dynamic CDI configuration for NVidia devices by running
+          nvidia-container-toolkit on boot.
+        '';
+      };
+
+      mounts = lib.mkOption {
+        type = lib.types.listOf (lib.types.submodule mountType);
+        default = [];
+        description = "Mounts to be added to every container under the Nvidia CDI profile.";
+      };
+
+      mount-nvidia-executables = lib.mkOption {
+        default = true;
+        type = lib.types.bool;
+        description = ''
+          Mount executables nvidia-smi, nvidia-cuda-mps-control, nvidia-cuda-mps-server,
+          nvidia-debugdump, nvidia-powerd and nvidia-ctk on containers.
+        '';
+      };
+
+      mount-nvidia-docker-1-directories = lib.mkOption {
+        default = true;
+        type = lib.types.bool;
+        description = ''
+          Mount nvidia-docker-1 directories on containers: /usr/local/nvidia/lib and
+          /usr/local/nvidia/lib64.
+        '';
+      };
+    };
+
+  };
+
+  config = {
+
+    hardware.nvidia-container-toolkit.mounts = let
+      nvidia-driver = config.hardware.nvidia.package;
+    in (lib.mkMerge [
+      [{ hostPath = pkgs.addDriverRunpath.driverLink;
+         containerPath = pkgs.addDriverRunpath.driverLink; }
+       { hostPath = "${lib.getLib pkgs.glibc}/lib";
+         containerPath = "${lib.getLib pkgs.glibc}/lib"; }
+       { hostPath = "${lib.getLib pkgs.glibc}/lib64";
+         containerPath = "${lib.getLib pkgs.glibc}/lib64"; }]
+      (lib.mkIf config.hardware.nvidia-container-toolkit.mount-nvidia-executables
+        [{ hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-control";
+           containerPath = "/usr/bin/nvidia-cuda-mps-control"; }
+         { hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-server";
+           containerPath = "/usr/bin/nvidia-cuda-mps-server"; }
+         { hostPath = lib.getExe' nvidia-driver "nvidia-debugdump";
+           containerPath = "/usr/bin/nvidia-debugdump"; }
+         { hostPath = lib.getExe' nvidia-driver "nvidia-powerd";
+           containerPath = "/usr/bin/nvidia-powerd"; }
+         { hostPath = lib.getExe' nvidia-driver "nvidia-smi";
+           containerPath = "/usr/bin/nvidia-smi"; }])
+      # nvidia-docker 1.0 uses /usr/local/nvidia/lib{,64}
+      #   e.g.
+      #     - https://gitlab.com/nvidia/container-images/cuda/-/blob/e3ff10eab3a1424fe394899df0e0f8ca5a410f0f/dist/12.3.1/ubi9/base/Dockerfile#L44
+      #     - https://github.com/NVIDIA/nvidia-docker/blob/01d2c9436620d7dde4672e414698afe6da4a282f/src/nvidia/volumes.go#L104-L173
+      (lib.mkIf config.hardware.nvidia-container-toolkit.mount-nvidia-docker-1-directories
+        [{ hostPath = "${lib.getLib nvidia-driver}/lib";
+           containerPath = "/usr/local/nvidia/lib"; }
+         { hostPath = "${lib.getLib nvidia-driver}/lib";
+           containerPath = "/usr/local/nvidia/lib64"; }])
+    ]);
+
+    systemd.services.nvidia-container-toolkit-cdi-generator = lib.mkIf config.hardware.nvidia-container-toolkit.enable {
+      description = "Container Device Interface (CDI) for Nvidia generator";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "systemd-udev-settle.service" ];
+      serviceConfig = {
+        RuntimeDirectory = "cdi";
+        RemainAfterExit = true;
+        ExecStart =
+          let
+            script = pkgs.callPackage ./cdi-generate.nix {
+              inherit (config.hardware.nvidia-container-toolkit) mounts;
+              nvidia-driver = config.hardware.nvidia.package;
+            };
+          in
+          lib.getExe script;
+        Type = "oneshot";
+      };
+    };
+
+  };
+
+}
diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix
index d726955307861..65620dd3935b8 100644
--- a/nixos/modules/virtualisation/containers.nix
+++ b/nixos/modules/virtualisation/containers.nix
@@ -28,43 +28,6 @@ in
       description = "Enable the OCI seccomp BPF hook";
     };
 
-    cdi = {
-      dynamic.nvidia.enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Enable dynamic CDI configuration for NVidia devices by running nvidia-container-toolkit on boot.
-        '';
-      };
-
-      static = mkOption {
-        type = types.attrs;
-        default = { };
-        description = ''
-          Declarative CDI specification. Each key of the attribute set
-          will be mapped to a file in /etc/cdi. It is required for every
-          key to be provided in JSON format.
-        '';
-        example = {
-          some-vendor = builtins.fromJSON ''
-              {
-                "cdiVersion": "0.5.0",
-                "kind": "some-vendor.com/foo",
-                "devices": [],
-                "containerEdits": []
-              }
-            '';
-
-          some-other-vendor = {
-            cdiVersion = "0.5.0";
-            kind = "some-other-vendor.com/bar";
-            devices = [];
-            containerEdits = [];
-          };
-        };
-      };
-    };
-
     containersConf.settings = mkOption {
       type = toml.type;
       default = { };
@@ -150,8 +113,6 @@ in
 
   config = lib.mkIf cfg.enable {
 
-    hardware.nvidia-container-toolkit-cdi-generator.enable = lib.mkIf cfg.cdi.dynamic.nvidia.enable true;
-
     virtualisation.containers.containersConf.cniPlugins = [ pkgs.cni-plugins ];
 
     virtualisation.containers.containersConf.settings = {
@@ -163,13 +124,7 @@ in
       };
     };
 
-    environment.etc = let
-      cdiStaticConfigurationFiles = (lib.attrsets.mapAttrs'
-        (name: value:
-          lib.attrsets.nameValuePair "cdi/${name}.json"
-            { text = builtins.toJSON value; })
-        cfg.cdi.static);
-    in {
+    environment.etc = {
       "containers/containers.conf".source =
         toml.generate "containers.conf" cfg.containersConf.settings;
 
@@ -183,7 +138,7 @@ in
       "containers/policy.json".source =
         if cfg.policy != { } then pkgs.writeText "policy.json" (builtins.toJSON cfg.policy)
         else "${pkgs.skopeo.policy}/default-policy.json";
-    } // cdiStaticConfigurationFiles;
+    };
 
   };