nixos/nvidia-container-toolkit: rename from `virtualisation.containers.cdi.dynamic.nvidia.enable`

Add the NixOS option `hardware.nvidia-container-toolkit-cdi-generator.enable`.

This enables the ability to expose GPU's in containers for container
runtimes that support the Container Device Interface (CDI)

Remove `cdi.static` and `cdi.dynamic.nvidia.enable` attributes.

4 files changed, 118 insertions, 100 deletions

diff --git a/nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/cdi-generate.nix b/nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/cdi-generate.nix
deleted file mode 100644
index 1aaa2d07b9bde..0000000000000
--- a/nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/cdi-generate.nix
+++ /dev/null
@@ -1,60 +0,0 @@
-{
-  addDriverRunpath,
-  glibc,
-  jq,
-  lib,
-  nvidia-container-toolkit,
-  nvidia-driver,
-  runtimeShell,
-  writeScriptBin,
-}:
-let
-  mountOptions = { options = ["ro" "nosuid" "nodev" "bind"]; };
-  mounts = [
-    # FIXME: Making /usr mounts optional
-    { hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-control";
-      containerPath = "/usr/bin/nvidia-cuda-mps-control"; }
-    { hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-server";
-      containerPath = "/usr/bin/nvidia-cuda-mps-server"; }
-    { hostPath = lib.getExe' nvidia-driver "nvidia-debugdump";
-      containerPath = "/usr/bin/nvidia-debugdump"; }
-    { hostPath = lib.getExe' nvidia-driver "nvidia-powerd";
-      containerPath = "/usr/bin/nvidia-powerd"; }
-    { hostPath = lib.getExe' nvidia-driver "nvidia-smi";
-      containerPath = "/usr/bin/nvidia-smi"; }
-    { hostPath = lib.getExe' nvidia-container-toolkit "nvidia-ctk";
-      containerPath = "/usr/bin/nvidia-ctk"; }
-    { hostPath = "${lib.getLib glibc}/lib";
-      containerPath = "${lib.getLib glibc}/lib"; }
-
-    # FIXME: use closureinfo
-    {
-      hostPath = addDriverRunpath.driverLink;
-      containerPath = addDriverRunpath.driverLink;
-    }
-    { hostPath = "${lib.getLib glibc}/lib";
-      containerPath = "${lib.getLib glibc}/lib"; }
-    { hostPath = "${lib.getLib glibc}/lib64";
-      containerPath = "${lib.getLib glibc}/lib64"; }
-  ];
-  jqAddMountExpression = ".containerEdits.mounts[.containerEdits.mounts | length] |= . +";
-  mountsToJq = lib.concatMap
-    (mount:
-      ["${lib.getExe jq} '${jqAddMountExpression} ${builtins.toJSON (mount // mountOptions)}'"])
-    mounts;
-in
-writeScriptBin "nvidia-cdi-generator"
-''
-#! ${runtimeShell}
-
-function cdiGenerate {
-  ${lib.getExe' nvidia-container-toolkit "nvidia-ctk"} cdi generate \
-    --format json \
-    --ldconfig-path ${lib.getExe' glibc "ldconfig"} \
-    --library-search-path ${lib.getLib nvidia-driver}/lib \
-    --nvidia-ctk-path ${lib.getExe' nvidia-container-toolkit "nvidia-ctk"}
-}
-
-cdiGenerate | \
-  ${lib.concatStringsSep " | " mountsToJq} > $RUNTIME_DIRECTORY/nvidia-container-toolkit.json
-''
diff --git a/nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/default.nix b/nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/default.nix
deleted file mode 100644
index 5aa3c72ee0a06..0000000000000
--- a/nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/default.nix
+++ /dev/null
@@ -1,40 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-{
-
-  options = {
-
-    hardware.nvidia-container-toolkit-cdi-generator.enable = lib.mkOption {
-      default = false;
-      internal = true;
-      visible = false;
-      type = lib.types.bool;
-      description = ''
-        Enable dynamic CDI configuration for NVidia devices by running
-        nvidia-container-toolkit on boot.
-      '';
-    };
-
-  };
-
-  config = {
-
-    systemd.services.nvidia-container-toolkit-cdi-generator = lib.mkIf config.hardware.nvidia-container-toolkit-cdi-generator.enable {
-      description = "Container Device Interface (CDI) for Nvidia generator";
-      wantedBy = [ "multi-user.target" ];
-      after = [ "systemd-udev-settle.service" ];
-      serviceConfig = {
-        RuntimeDirectory = "cdi";
-        RemainAfterExit = true;
-        ExecStart =
-          let
-            script = pkgs.callPackage ./cdi-generate.nix { nvidia-driver = config.hardware.nvidia.package; };
-          in
-          lib.getExe script;
-        Type = "oneshot";
-      };
-    };
-
-  };
-
-}
diff --git a/nixos/modules/services/hardware/nvidia-container-toolkit/cdi-generate.nix b/nixos/modules/services/hardware/nvidia-container-toolkit/cdi-generate.nix
new file mode 100644
index 0000000000000..ca769cc44e5c9
--- /dev/null
+++ b/nixos/modules/services/hardware/nvidia-container-toolkit/cdi-generate.nix
@@ -0,0 +1,35 @@
+{
+  glibc,
+  jq,
+  lib,
+  mounts,
+  nvidia-container-toolkit,
+  nvidia-driver,
+  runtimeShell,
+  writeScriptBin,
+}: let
+  mkMount = {hostPath, containerPath, mountOptions}: {
+    inherit hostPath containerPath;
+    options = mountOptions;
+  };
+  jqAddMountExpression = ".containerEdits.mounts[.containerEdits.mounts | length] |= . +";
+  allJqMounts = lib.concatMap
+    (mount:
+      ["${lib.getExe jq} '${jqAddMountExpression} ${builtins.toJSON (mkMount mount)}'"])
+    mounts;
+in
+writeScriptBin "nvidia-cdi-generator"
+''
+#! ${runtimeShell}
+
+function cdiGenerate {
+  ${lib.getExe' nvidia-container-toolkit "nvidia-ctk"} cdi generate \
+    --format json \
+    --ldconfig-path ${lib.getExe' glibc "ldconfig"} \
+    --library-search-path ${lib.getLib nvidia-driver}/lib \
+    --nvidia-ctk-path ${lib.getExe' nvidia-container-toolkit "nvidia-ctk"}
+}
+
+cdiGenerate | \
+  ${lib.concatStringsSep " | " allJqMounts} > $RUNTIME_DIRECTORY/nvidia-container-toolkit.json
+''
diff --git a/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix b/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix
new file mode 100644
index 0000000000000..f4f3b69e1dc6a
--- /dev/null
+++ b/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix
@@ -0,0 +1,83 @@
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [
+    (lib.mkRenamedOptionModule
+      [ "virtualisation" "containers" "cdi" "dynamic" "nvidia" "enable" ]
+      [ "hardware" "nvidia-container-toolkit" "enable" ])
+  ];
+
+  options = let
+    mountType = {
+      options = {
+        hostPath = lib.mkOption {
+          type = lib.types.str;
+          description = "Host path.";
+        };
+        containerPath = lib.mkOption {
+          type = lib.types.str;
+          description = "Container path.";
+        };
+        mountOptions = lib.mkOption {
+          default = [ "ro" "nosuid" "nodev" "bind" ];
+          type = lib.types.listOf lib.types.str;
+          description = "Mount options.";
+        };
+      };
+    };
+  in {
+
+    hardware.nvidia-container-toolkit = {
+      enable = lib.mkOption {
+        default = false;
+        type = lib.types.bool;
+        description = ''
+          Enable dynamic CDI configuration for NVidia devices by running
+          nvidia-container-toolkit on boot.
+        '';
+      };
+
+      mounts = lib.mkOption {
+        type = lib.types.listOf (lib.types.submodule mountType);
+        default = [];
+        description = "Mounts to be added to every container under the Nvidia CDI profile.";
+      };
+    };
+
+  };
+
+  config = {
+
+    hardware.nvidia-container-toolkit.mounts = let
+      nvidia-driver = config.hardware.nvidia.package;
+    in (lib.mkMerge [
+      [{ hostPath = pkgs.addDriverRunpath.driverLink;
+         containerPath = pkgs.addDriverRunpath.driverLink; }
+       { hostPath = "${lib.getLib pkgs.glibc}/lib";
+         containerPath = "${lib.getLib pkgs.glibc}/lib"; }
+       { hostPath = "${lib.getLib pkgs.glibc}/lib64";
+         containerPath = "${lib.getLib pkgs.glibc}/lib64"; }]
+    ]);
+
+    systemd.services.nvidia-container-toolkit-cdi-generator = lib.mkIf config.hardware.nvidia-container-toolkit.enable {
+      description = "Container Device Interface (CDI) for Nvidia generator";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "systemd-udev-settle.service" ];
+      serviceConfig = {
+        RuntimeDirectory = "cdi";
+        RemainAfterExit = true;
+        ExecStart =
+          let
+            script = pkgs.callPackage ./cdi-generate.nix {
+              inherit (config.hardware.nvidia-container-toolkit) mounts;
+              nvidia-driver = config.hardware.nvidia.package;
+            };
+          in
+          lib.getExe script;
+        Type = "oneshot";
+      };
+    };
+
+  };
+
+}