Merge pull request #199150 from Ma27/grafana-fixup

nixos/grafana: documentation/warning improvements after #191768

4 files changed, 404 insertions, 193 deletions

diff --git a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml
index c56cfb10538bb..8e97e58f81a61 100644
--- a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml
+++ b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml
@@ -1206,22 +1206,146 @@ services.github-runner.serviceOverrides.SupplementaryGroups = [
       </listitem>
       <listitem>
         <para>
-          The <literal>services.grafana</literal> options were converted
-          to a
+          The module <literal>services.grafana</literal> was refactored
+          to be compliant with
           <link xlink:href="https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md">RFC
-          0042</link> configuration.
-        </para>
-      </listitem>
-      <listitem>
-        <para>
-          The <literal>services.grafana.provision.datasources</literal>
-          and <literal>services.grafana.provision.dashboards</literal>
-          options were converted to a
-          <link xlink:href="https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md">RFC
-          0042</link> configuration. They also now support specifying
-          the provisioning YAML file with <literal>path</literal>
-          option.
+          0042</link>. To be precise, this means that the following
+          things have changed:
         </para>
+        <itemizedlist>
+          <listitem>
+            <para>
+              The newly introduced option
+              <xref linkend="opt-services.grafana.settings" /> is an
+              attribute-set that will be converted into Grafana’s INI
+              format. This means that the configuration from
+              <link xlink:href="https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/">Grafana’s
+              configuration reference</link> can be directly written as
+              attribute-set in Nix within this option.
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              The option
+              <literal>services.grafana.extraOptions</literal> has been
+              removed. This option was an association of environment
+              variables for Grafana. If you had an expression like
+            </para>
+            <programlisting language="bash">
+{
+  services.grafana.extraOptions.SECURITY_ADMIN_USER = &quot;foobar&quot;;
+}
+</programlisting>
+            <para>
+              your Grafana instance was running with
+              <literal>GF_SECURITY_ADMIN_USER=foobar</literal> in its
+              environment.
+            </para>
+            <para>
+              For the migration, it is recommended to turn it into the
+              INI format, i.e. to declare
+            </para>
+            <programlisting language="bash">
+{
+  services.grafana.settings.security.admin_user = &quot;foobar&quot;;
+}
+</programlisting>
+            <para>
+              instead.
+            </para>
+            <para>
+              The keys in
+              <literal>services.grafana.extraOptions</literal> have the
+              format
+              <literal>&lt;INI section name&gt;_&lt;Key Name&gt;</literal>.
+              Further details are outlined in the
+              <link xlink:href="https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#override-configuration-with-environment-variables">configuration
+              reference</link>.
+            </para>
+            <para>
+              Alternatively you can also set all your values from
+              <literal>extraOptions</literal> to
+              <literal>systemd.services.grafana.environment</literal>,
+              make sure you don’t forget to add the
+              <literal>GF_</literal> prefix though!
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              Previously, the options
+              <xref linkend="opt-services.grafana.provision.datasources" />
+              and
+              <xref linkend="opt-services.grafana.provision.dashboards" />
+              expected lists of datasources or dashboards for the
+              <link xlink:href="https://grafana.com/docs/grafana/latest/administration/provisioning/">declarative
+              provisioning</link>.
+            </para>
+            <para>
+              To declare lists of
+            </para>
+            <itemizedlist spacing="compact">
+              <listitem>
+                <para>
+                  <emphasis role="strong">datasources</emphasis>, please
+                  rename your declarations to
+                  <xref linkend="opt-services.grafana.provision.datasources.settings.datasources" />.
+                </para>
+              </listitem>
+              <listitem>
+                <para>
+                  <emphasis role="strong">dashboards</emphasis>, please
+                  rename your declarations to
+                  <xref linkend="opt-services.grafana.provision.dashboards.settings.providers" />.
+                </para>
+              </listitem>
+            </itemizedlist>
+            <para>
+              This change was made to support more features for that:
+            </para>
+            <itemizedlist>
+              <listitem>
+                <para>
+                  It’s possible to declare the
+                  <literal>apiVersion</literal> of your dashboards and
+                  datasources by
+                  <xref linkend="opt-services.grafana.provision.datasources.settings.apiVersion" />
+                  (or
+                  <xref linkend="opt-services.grafana.provision.dashboards.settings.apiVersion" />).
+                </para>
+              </listitem>
+              <listitem>
+                <para>
+                  Instead of declaring datasources and dashboards in
+                  pure Nix, it’s also possible to specify configuration
+                  files (or directories) with YAML instead using
+                  <xref linkend="opt-services.grafana.provision.datasources.path" />
+                  (or
+                  <xref linkend="opt-services.grafana.provision.dashboards.path" />.
+                  This is useful when having provisioning files from
+                  non-NixOS Grafana instances that you also want to
+                  deploy to NixOS.
+                </para>
+                <para>
+                  <emphasis role="strong">Note:</emphasis> secrets from
+                  these files will be leaked into the store unless you
+                  use a
+                  <link xlink:href="https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider"><emphasis role="strong">file</emphasis>-provider
+                  or env-var</link> for secrets!
+                </para>
+              </listitem>
+              <listitem>
+                <para>
+                  <xref linkend="opt-services.grafana.provision.notifiers" />
+                  is not affected by this change because this feature is
+                  deprecated by Grafana and will probably removed in
+                  Grafana 10. It’s recommended to use
+                  <literal>services.grafana.provision.alerting.contactPoints</literal>
+                  instead.
+                </para>
+              </listitem>
+            </itemizedlist>
+          </listitem>
+        </itemizedlist>
       </listitem>
       <listitem>
         <para>
diff --git a/nixos/doc/manual/release-notes/rl-2211.section.md b/nixos/doc/manual/release-notes/rl-2211.section.md
index 5daafa8e986b4..ecfba7215c8b2 100644
--- a/nixos/doc/manual/release-notes/rl-2211.section.md
+++ b/nixos/doc/manual/release-notes/rl-2211.section.md
@@ -379,9 +379,66 @@ Available as [services.patroni](options.html#opt-services.patroni.enable).
 
 - The `services.matrix-synapse` systemd unit has been hardened.
 
-- The `services.grafana` options were converted to a [RFC 0042](https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md) configuration.
-
-- The `services.grafana.provision.datasources` and `services.grafana.provision.dashboards` options were converted to a [RFC 0042](https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md) configuration. They also now support specifying the provisioning YAML file with `path` option.
+- The module `services.grafana` was refactored to be compliant with [RFC 0042](https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md). To be precise, this means that the following things have changed:
+  - The newly introduced option [](#opt-services.grafana.settings) is an attribute-set that
+    will be converted into Grafana's INI format. This means that the configuration from
+    [Grafana's configuration reference](https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/)
+    can be directly written as attribute-set in Nix within this option.
+  - The option `services.grafana.extraOptions` has been removed. This option was an association
+    of environment variables for Grafana. If you had an expression like
+
+    ```nix
+    {
+      services.grafana.extraOptions.SECURITY_ADMIN_USER = "foobar";
+    }
+    ```
+
+    your Grafana instance was running with `GF_SECURITY_ADMIN_USER=foobar` in its environment.
+
+    For the migration, it is recommended to turn it into the INI format, i.e.
+    to declare
+
+    ```nix
+    {
+      services.grafana.settings.security.admin_user = "foobar";
+    }
+    ```
+
+    instead.
+
+    The keys in `services.grafana.extraOptions` have the format `<INI section name>_<Key Name>`.
+    Further details are outlined in the [configuration reference](https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#override-configuration-with-environment-variables).
+
+    Alternatively you can also set all your values from `extraOptions` to
+    `systemd.services.grafana.environment`, make sure you don't forget to add
+    the `GF_` prefix though!
+  - Previously, the options [](#opt-services.grafana.provision.datasources) and
+    [](#opt-services.grafana.provision.dashboards) expected lists of datasources
+    or dashboards for the [declarative provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/).
+
+    To declare lists of
+    - **datasources**, please rename your declarations to [](#opt-services.grafana.provision.datasources.settings.datasources).
+    - **dashboards**, please rename your declarations to [](#opt-services.grafana.provision.dashboards.settings.providers).
+
+    This change was made to support more features for that:
+
+    - It's possible to declare the `apiVersion` of your dashboards and datasources
+      by [](#opt-services.grafana.provision.datasources.settings.apiVersion) (or
+      [](#opt-services.grafana.provision.dashboards.settings.apiVersion)).
+
+    - Instead of declaring datasources and dashboards in pure Nix, it's also possible
+      to specify configuration files (or directories) with YAML instead using
+      [](#opt-services.grafana.provision.datasources.path) (or
+      [](#opt-services.grafana.provision.dashboards.path). This is useful when having
+      provisioning files from non-NixOS Grafana instances that you also want to
+      deploy to NixOS.
+
+      __Note:__ secrets from these files will be leaked into the store unless you use a
+      [**file**-provider or env-var](https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider) for secrets!
+
+    - [](#opt-services.grafana.provision.notifiers) is not affected by this change because
+      this feature is deprecated by Grafana and will probably removed in Grafana 10.
+      It's recommended to use `services.grafana.provision.alerting.contactPoints` instead.
 
 - The `services.grafana.provision.alerting` option was added. It includes suboptions for every alerting-related objects (with the exception of `notifiers`), which means it's now possible to configure modern Grafana alerting declaratively.
 
diff --git a/nixos/modules/services/monitoring/grafana.nix b/nixos/modules/services/monitoring/grafana.nix
index 52d5cab9f5159..9b3068796d8e3 100644
--- a/nixos/modules/services/monitoring/grafana.nix
+++ b/nixos/modules/services/monitoring/grafana.nix
@@ -13,57 +13,96 @@ let
   settingsFormatIni = pkgs.formats.ini {};
   configFile = settingsFormatIni.generate "config.ini" cfg.settings;
 
-  datasourceConfiguration = {
-    apiVersion = 1;
-    datasources = cfg.provision.datasources;
-  };
-
-  datasourceFileNew = if (cfg.provision.datasources.path == null) then provisioningSettingsFormat.generate "datasource.yaml" cfg.provision.datasources.settings else cfg.provision.datasources.path;
-  datasourceFile = if (builtins.isList cfg.provision.datasources) then provisioningSettingsFormat.generate "datasource.yaml" datasourceConfiguration else datasourceFileNew;
-
-  dashboardConfiguration = {
-    apiVersion = 1;
-    providers = cfg.provision.dashboards;
-  };
-
-  dashboardFileNew = if (cfg.provision.dashboards.path == null) then provisioningSettingsFormat.generate "dashboard.yaml" cfg.provision.dashboards.settings else cfg.provision.dashboards.path;
-  dashboardFile = if (builtins.isList cfg.provision.dashboards) then provisioningSettingsFormat.generate "dashboard.yaml" dashboardConfiguration else dashboardFileNew;
+  mkProvisionCfg = name: attr: provisionCfg:
+    if provisionCfg.path != null
+      then provisionCfg.path
+    else
+      provisioningSettingsFormat.generate "${name}.yaml"
+        (if provisionCfg.settings != null
+          then provisionCfg.settings
+          else {
+            apiVersion = 1;
+            ${attr} = [];
+          });
+
+  datasourceFileOrDir = mkProvisionCfg "datasource" "datasources" cfg.provision.datasources;
+  dashboardFileOrDir = mkProvisionCfg "dashboard" "providers" cfg.provision.dashboards;
 
   notifierConfiguration = {
     apiVersion = 1;
     notifiers = cfg.provision.notifiers;
   };
 
-  notifierFile = pkgs.writeText "notifier.yaml" (builtins.toJSON notifierConfiguration);
+  notifierFileOrDir = pkgs.writeText "notifier.yaml" (builtins.toJSON notifierConfiguration);
 
   generateAlertingProvisioningYaml = x: if (cfg.provision.alerting."${x}".path == null)
                                         then provisioningSettingsFormat.generate "${x}.yaml" cfg.provision.alerting."${x}".settings
                                         else cfg.provision.alerting."${x}".path;
-  rulesFile = generateAlertingProvisioningYaml "rules";
-  contactPointsFile = generateAlertingProvisioningYaml "contactPoints";
-  policiesFile = generateAlertingProvisioningYaml "policies";
-  templatesFile = generateAlertingProvisioningYaml "templates";
-  muteTimingsFile = generateAlertingProvisioningYaml "muteTimings";
-
-  provisionConfDir =  pkgs.runCommand "grafana-provisioning" { } ''
+  rulesFileOrDir = generateAlertingProvisioningYaml "rules";
+  contactPointsFileOrDir = generateAlertingProvisioningYaml "contactPoints";
+  policiesFileOrDir = generateAlertingProvisioningYaml "policies";
+  templatesFileOrDir = generateAlertingProvisioningYaml "templates";
+  muteTimingsFileOrDir = generateAlertingProvisioningYaml "muteTimings";
+
+  ln = { src, dir, filename }: ''
+    if [[ -d "${src}" ]]; then
+      pushd $out/${dir} &>/dev/null
+        lndir "${src}"
+      popd &>/dev/null
+    else
+      ln -sf ${src} $out/${dir}/${filename}.yaml
+    fi
+  '';
+  provisionConfDir = pkgs.runCommand "grafana-provisioning" { nativeBuildInputs = [ pkgs.xorg.lndir ]; } ''
     mkdir -p $out/{datasources,dashboards,notifiers,alerting}
-    ln -sf ${datasourceFile} $out/datasources/datasource.yaml
-    ln -sf ${dashboardFile} $out/dashboards/dashboard.yaml
-    ln -sf ${notifierFile} $out/notifiers/notifier.yaml
-    ln -sf ${rulesFile} $out/alerting/rules.yaml
-    ln -sf ${contactPointsFile} $out/alerting/contactPoints.yaml
-    ln -sf ${policiesFile} $out/alerting/policies.yaml
-    ln -sf ${templatesFile} $out/alerting/templates.yaml
-    ln -sf ${muteTimingsFile} $out/alerting/muteTimings.yaml
+    ${ln { src = datasourceFileOrDir;    dir = "datasources"; filename = "datasource"; }}
+    ${ln { src = dashboardFileOrDir;     dir = "dashboards";  filename = "dashbaord"; }}
+    ${ln { src = notifierFileOrDir;      dir = "notifiers";   filename = "notifier"; }}
+    ${ln { src = rulesFileOrDir;         dir = "alerting";    filename = "rules"; }}
+    ${ln { src = contactPointsFileOrDir; dir = "alerting";    filename = "contactPoints"; }}
+    ${ln { src = policiesFileOrDir;      dir = "alerting";    filename = "policies"; }}
+    ${ln { src = templatesFileOrDir;     dir = "alerting";    filename = "templates"; }}
+    ${ln { src = muteTimingsFileOrDir;   dir = "alerting";    filename = "muteTimings"; }}
   '';
 
   # Get a submodule without any embedded metadata:
   _filter = x: filterAttrs (k: v: k != "_module") x;
 
+  # FIXME(@Ma27) remove before 23.05. This is just a helper-type
+  # because `mkRenamedOptionModule` doesn't work if `foo.bar` is renamed
+  # to `foo.bar.baz`.
+  submodule' = module: types.coercedTo
+    (mkOptionType {
+      name = "grafana-provision-submodule";
+      description = "Wrapper-type for backwards compat of Grafana's declarative provisioning";
+      check = x:
+        if builtins.isList x then
+          throw ''
+            Provisioning dashboards and datasources declaratively by
+            setting `dashboards` or `datasources` to a list is not supported
+            anymore. Use `services.grafana.provision.datasources.settings.datasources`
+            (or `services.grafana.provision.dashboards.settings.providers`) instead.
+          ''
+        else isAttrs x || isFunction x;
+    })
+    id
+    (types.submodule module);
+
   # http://docs.grafana.org/administration/provisioning/#datasources
   grafanaTypes.datasourceConfig = types.submodule {
     freeformType = provisioningSettingsFormat.type;
 
+    imports = [
+      (mkRemovedOptionModule [ "password" ] ''
+        `services.grafana.provision.datasources.settings.datasources.<name>.password` has been removed
+        in Grafana 9. Use `secureJsonData` instead.
+      '')
+      (mkRemovedOptionModule [ "basicAuthPassword" ] ''
+        `services.grafana.provision.datasources.settings.datasources.<name>.basicAuthPassword` has been removed
+        in Grafana 9. Use `secureJsonData` instead.
+      '')
+    ];
+
     options = {
       name = mkOption {
         type = types.str;
@@ -93,28 +132,6 @@ let
         default = false;
         description = lib.mdDoc "Allow users to edit datasources from the UI.";
       };
-      password = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = lib.mdDoc ''
-          Database password, if used. Please note that the contents of this option
-          will end up in a world-readable Nix store. Use the file provider
-          pointing at a reasonably secured file in the local filesystem
-          to work around that. Look at the documentation for details:
-          <https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider>
-        '';
-      };
-      basicAuthPassword = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = lib.mdDoc ''
-          Basic auth password. Please note that the contents of this option
-          will end up in a world-readable Nix store. Use the file provider
-          pointing at a reasonably secured file in the local filesystem
-          to work around that. Look at the documentation for details:
-          <https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider>
-        '';
-      };
       secureJsonData = mkOption {
         type = types.nullOr types.attrs;
         default = null;
@@ -276,6 +293,10 @@ in {
     (mkRemovedOptionModule [ "services" "grafana" "auth" "google" "clientSecretFile" ] ''
       This option has been removed. Use 'services.grafana.settings.google.client_secret' with file provider instead.
     '')
+    (mkRemovedOptionModule [ "services" "grafana" "extraOptions" ] ''
+      This option has been removed. Use 'services.grafana.settings' instead. For a detailed migration guide, please
+      review the release notes of NixOS 22.11.
+    '')
 
     (mkRemovedOptionModule [ "services" "grafana" "auth" "azuread" "tenantId" ] "This option has been deprecated upstream.")
   ];
@@ -330,19 +351,7 @@ in {
                 Don't change the value of this option if you are planning to use `services.grafana.provision` options.
               '';
               default = provisionConfDir;
-              defaultText = literalExpression ''
-                pkgs.runCommand "grafana-provisioning" { } \'\'
-                  mkdir -p $out/{datasources,dashboards,notifiers,alerting}
-                  ln -sf ''${datasourceFile} $out/datasources/datasource.yaml
-                  ln -sf ''${dashboardFile} $out/dashboards/dashboard.yaml
-                  ln -sf ''${notifierFile} $out/notifiers/notifier.yaml
-                  ln -sf ''${rulesFile} $out/alerting/rules.yaml
-                  ln -sf ''${contactPointsFile} $out/alerting/contactPoints.yaml
-                  ln -sf ''${policiesFile} $out/alerting/policies.yaml
-                  ln -sf ''${templatesFile} $out/alerting/templates.yaml
-                  ln -sf ''${muteTimingsFile} $out/alerting/muteTimings.yaml
-                  \'\'
-              '';
+              defaultText = "directory with links to files generated from services.grafana.provision";
               type = types.path;
             };
           };
@@ -564,17 +573,14 @@ in {
 
       datasources = mkOption {
         description = lib.mdDoc ''
-          Deprecated option for Grafana datasource configuration. Use either
-          `services.grafana.provision.datasources.settings` or
-          `services.grafana.provision.datasources.path` instead.
+          Declaratively provision Grafana's datasources.
         '';
-        default = [];
-        apply = x: if (builtins.isList x) then map _filter x else x;
-        type = with types; either (listOf grafanaTypes.datasourceConfig) (submodule {
+        default = {};
+        type = submodule' {
           options.settings = mkOption {
             description = lib.mdDoc ''
               Grafana datasource configuration in Nix. Can't be used with
-              `services.grafana.provision.datasources.path` simultaneously. See
+              [](#opt-services.grafana.provision.datasources.path) simultaneously. See
               <https://grafana.com/docs/grafana/latest/administration/provisioning/#data-sources>
               for supported options.
             '';
@@ -591,6 +597,7 @@ in {
                   description = lib.mdDoc "List of datasources to insert/update.";
                   default = [];
                   type = types.listOf grafanaTypes.datasourceConfig;
+                  apply = map (flip builtins.removeAttrs [ "password" "basicAuthPassword" ]);
                 };
 
                 deleteDatasources = mkOption {
@@ -630,28 +637,26 @@ in {
           options.path = mkOption {
             description = lib.mdDoc ''
               Path to YAML datasource configuration. Can't be used with
-              `services.grafana.provision.datasources.settings` simultaneously.
+              [](#opt-services.grafana.provision.datasources.settings) simultaneously.
+              Can be either a directory or a single YAML file. Will end up in the store.
             '';
             default = null;
             type = types.nullOr types.path;
           };
-        });
+        };
       };
 
 
       dashboards = mkOption {
         description = lib.mdDoc ''
-          Deprecated option for Grafana dashboard configuration. Use either
-          `services.grafana.provision.dashboards.settings` or
-          `services.grafana.provision.dashboards.path` instead.
+          Declaratively provision Grafana's dashboards.
         '';
-        default = [];
-        apply = x: if (builtins.isList x) then map _filter x else x;
-        type = with types; either (listOf grafanaTypes.dashboardConfig) (submodule {
+        default = {};
+        type = submodule' {
           options.settings = mkOption {
             description = lib.mdDoc ''
               Grafana dashboard configuration in Nix. Can't be used with
-              `services.grafana.provision.dashboards.path` simultaneously. See
+              [](#opt-services.grafana.provision.dashboards.path) simultaneously. See
               <https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards>
               for supported options.
             '';
@@ -684,12 +689,13 @@ in {
           options.path = mkOption {
             description = lib.mdDoc ''
               Path to YAML dashboard configuration. Can't be used with
-              `services.grafana.provision.dashboards.settings` simultaneously.
+              [](#opt-services.grafana.provision.dashboards.settings) simultaneously.
+              Can be either a directory or a single YAML file. Will end up in the store.
             '';
             default = null;
             type = types.nullOr types.path;
           };
-        });
+        };
       };
 
 
@@ -706,7 +712,8 @@ in {
           path = mkOption {
             description = lib.mdDoc ''
               Path to YAML rules configuration. Can't be used with
-              `services.grafana.provision.alerting.rules.settings` simultaneously.
+              [](#opt-services.grafana.provision.alerting.rules.settings) simultaneously.
+              Can be either a directory or a single YAML file. Will end up in the store.
             '';
             default = null;
             type = types.nullOr types.path;
@@ -715,7 +722,7 @@ in {
           settings = mkOption {
             description = lib.mdDoc ''
               Grafana rules configuration in Nix. Can't be used with
-              `services.grafana.provision.alerting.rules.path` simultaneously. See
+              [](#opt-services.grafana.provision.alerting.rules.path) simultaneously. See
               <https://grafana.com/docs/grafana/latest/administration/provisioning/#rules>
               for supported options.
             '';
@@ -829,7 +836,8 @@ in {
           path = mkOption {
             description = lib.mdDoc ''
               Path to YAML contact points configuration. Can't be used with
-              `services.grafana.provision.alerting.contactPoints.settings` simultaneously.
+              [](#opt-services.grafana.provision.alerting.contactPoints.settings) simultaneously.
+              Can be either a directory or a single YAML file. Will end up in the store.
             '';
             default = null;
             type = types.nullOr types.path;
@@ -838,7 +846,7 @@ in {
           settings = mkOption {
             description = lib.mdDoc ''
               Grafana contact points configuration in Nix. Can't be used with
-              `services.grafana.provision.alerting.contactPoints.path` simultaneously. See
+              [](#opt-services.grafana.provision.alerting.contactPoints.path) simultaneously. See
               <https://grafana.com/docs/grafana/latest/administration/provisioning/#contact-points>
               for supported options.
             '';
@@ -852,7 +860,7 @@ in {
                 };
 
                 contactPoints = mkOption {
-                  description = lib.mdDoc "List of contact points to import or update. Please note that sensitive data will end up in world-readable Nix store.";
+                  description = lib.mdDoc "List of contact points to import or update.";
                   default = [];
                   type = types.listOf (types.submodule {
                     freeformType = provisioningSettingsFormat.type;
@@ -909,7 +917,8 @@ in {
           path = mkOption {
             description = lib.mdDoc ''
               Path to YAML notification policies configuration. Can't be used with
-              `services.grafana.provision.alerting.policies.settings` simultaneously.
+              [](#opt-services.grafana.provision.alerting.policies.settings) simultaneously.
+              Can be either a directory or a single YAML file. Will end up in the store.
             '';
             default = null;
             type = types.nullOr types.path;
@@ -918,7 +927,7 @@ in {
           settings = mkOption {
             description = lib.mdDoc ''
               Grafana notification policies configuration in Nix. Can't be used with
-              `services.grafana.provision.alerting.policies.path` simultaneously. See
+              [](#opt-services.grafana.provision.alerting.policies.path) simultaneously. See
               <https://grafana.com/docs/grafana/latest/administration/provisioning/#notification-policies>
               for supported options.
             '';
@@ -978,7 +987,8 @@ in {
           path = mkOption {
             description = lib.mdDoc ''
               Path to YAML templates configuration. Can't be used with
-              `services.grafana.provision.alerting.templates.settings` simultaneously.
+              [](#opt-services.grafana.provision.alerting.templates.settings) simultaneously.
+              Can be either a directory or a single YAML file. Will end up in the store.
             '';
             default = null;
             type = types.nullOr types.path;
@@ -987,7 +997,7 @@ in {
           settings = mkOption {
             description = lib.mdDoc ''
               Grafana templates configuration in Nix. Can't be used with
-              `services.grafana.provision.alerting.templates.path` simultaneously. See
+              [](#opt-services.grafana.provision.alerting.templates.path) simultaneously. See
               <https://grafana.com/docs/grafana/latest/administration/provisioning/#templates>
               for supported options.
             '';
@@ -1059,7 +1069,8 @@ in {
           path = mkOption {
             description = lib.mdDoc ''
               Path to YAML mute timings configuration. Can't be used with
-              `services.grafana.provision.alerting.muteTimings.settings` simultaneously.
+              [](#opt-services.grafana.provision.alerting.muteTimings.settings) simultaneously.
+              Can be either a directory or a single YAML file. Will end up in the store.
             '';
             default = null;
             type = types.nullOr types.path;
@@ -1068,7 +1079,7 @@ in {
           settings = mkOption {
             description = lib.mdDoc ''
               Grafana mute timings configuration in Nix. Can't be used with
-              `services.grafana.provision.alerting.muteTimings.path` simultaneously. See
+              [](#opt-services.grafana.provision.alerting.muteTimings.path) simultaneously. See
               <https://grafana.com/docs/grafana/latest/administration/provisioning/#mute-timings>
               for supported options.
             '';
@@ -1159,52 +1170,50 @@ in {
 
   config = mkIf cfg.enable {
     warnings = let
-      usesFileProvider = opt: defaultValue: builtins.match "^${defaultValue}$|^\\$__file\\{.*}$" opt != null;
-    in flatten [
-      (optional (
-        ! usesFileProvider cfg.settings.database.password "" ||
-        ! usesFileProvider cfg.settings.security.admin_password "admin"
-      ) "Grafana passwords will be stored as plaintext in the Nix store! Use file provider instead.")
-      (optional (
+      doesntUseFileProvider = opt: defaultValue:
         let
-          checkOpts = opt: any (x: x.password != null || x.basicAuthPassword != null || x.secureJsonData != null) opt;
-          datasourcesUsed = if (cfg.provision.datasources.settings == null) then [] else cfg.provision.datasources.settings.datasources;
-        in if (builtins.isList cfg.provision.datasources) then checkOpts cfg.provision.datasources else checkOpts datasourcesUsed
-        ) ''
-          Datasource passwords will be stored as plaintext in the Nix store!
-          It is not possible to use file provider in provisioning; please provision
-          datasources via `services.grafana.provision.datasources.path` instead.
-        '')
-      (optional (
-        any (x: x.secure_settings != null) cfg.provision.notifiers
-      ) "Notifier secure settings will be stored as plaintext in the Nix store! Use file provider instead.")
+          regex = "${optionalString (defaultValue != null) "^${defaultValue}$|"}^\\$__(file|env)\\{.*}$|^\\$[^_\\$][^ ]+$";
+        in builtins.match regex opt == null;
+    in
+      # Ensure that no custom credentials are leaked into the Nix store. Unless the default value
+      # is specified, this can be achieved by using the file/env provider:
+      # https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion
       (optional (
-        builtins.isList cfg.provision.datasources && cfg.provision.datasources != []
+        doesntUseFileProvider cfg.settings.database.password "" ||
+        doesntUseFileProvider cfg.settings.security.admin_password "admin"
       ) ''
-          Provisioning Grafana datasources with options has been deprecated.
-          Use `services.grafana.provision.datasources.settings` or
-          `services.grafana.provision.datasources.path` instead.
-        '')
-      (optional (
-        builtins.isList cfg.provision.datasources && cfg.provision.dashboards != []
+        Grafana passwords will be stored as plaintext in the Nix store!
+        Use file provider or an env-var instead.
+      '')
+      # Warn about deprecated notifiers.
+      ++ (optional (cfg.provision.notifiers != []) ''
+        Notifiers are deprecated upstream and will be removed in Grafana 10.
+        Use `services.grafana.provision.alerting.contactPoints` instead.
+      '')
+      # Ensure that `secureJsonData` of datasources provisioned via `datasources.settings`
+      # only uses file/env providers.
+      ++ (optional (
+        let
+          datasourcesToCheck = optionals
+            (cfg.provision.datasources.settings != null)
+            cfg.provision.datasources.settings.datasources;
+          declarationUnsafe = { secureJsonData, ... }:
+            secureJsonData != null
+            && any (flip doesntUseFileProvider null) (attrValues secureJsonData);
+        in any declarationUnsafe datasourcesToCheck
       ) ''
-          Provisioning Grafana dashboards with options has been deprecated.
-          Use `services.grafana.provision.dashboards.settings` or
-          `services.grafana.provision.dashboards.path` instead.
-        '')
-      (optional (
-        cfg.provision.notifiers != []
-        ) ''
-            Notifiers are deprecated upstream and will be removed in Grafana 10.
-            Use `services.grafana.provision.alerting.contactPoints` instead.
-        '')
-    ];
+        Declarations in the `secureJsonData`-block of a datasource will be leaked to the
+        Nix store unless a file-provider or an env-var is used!
+      '')
+      ++ (optional (
+        any (x: x.secure_settings != null) cfg.provision.notifiers
+      ) "Notifier secure settings will be stored as plaintext in the Nix store! Use file provider instead.");
 
     environment.systemPackages = [ cfg.package ];
 
     assertions = [
       {
-        assertion = if (builtins.isList cfg.provision.datasources) then true else cfg.provision.datasources.settings == null || cfg.provision.datasources.path == null;
+        assertion = cfg.provision.datasources.settings == null || cfg.provision.datasources.path == null;
         message = "Cannot set both datasources settings and datasources path";
       }
       {
@@ -1213,12 +1222,11 @@ in {
           ({ type, access, ... }: type == "prometheus" -> access != "direct")
           opt;
         in
-          if (builtins.isList cfg.provision.datasources) then prometheusIsNotDirect cfg.provision.datasources
-          else cfg.provision.datasources.settings == null || prometheusIsNotDirect cfg.provision.datasources.settings.datasources;
+          cfg.provision.datasources.settings == null || prometheusIsNotDirect cfg.provision.datasources.settings.datasources;
         message = "For datasources of type `prometheus`, the `direct` access mode is not supported anymore (since Grafana 9.2.0)";
       }
       {
-        assertion = if (builtins.isList cfg.provision.dashboards) then true else cfg.provision.dashboards.settings == null || cfg.provision.dashboards.path == null;
+        assertion = cfg.provision.dashboards.settings == null || cfg.provision.dashboards.path == null;
         message = "Cannot set both dashboards settings and dashboards path";
       }
       {
diff --git a/nixos/tests/grafana/provision/default.nix b/nixos/tests/grafana/provision/default.nix
index 7a707ab9fed1f..1eb927632eb7a 100644
--- a/nixos/tests/grafana/provision/default.nix
+++ b/nixos/tests/grafana/provision/default.nix
@@ -17,7 +17,7 @@ let
 
         security = {
           admin_user = "testadmin";
-          admin_password = "snakeoilpwd";
+          admin_password = "$__file{${pkgs.writeText "pwd" "snakeoilpwd"}}";
         };
       };
     };
@@ -28,17 +28,24 @@ let
   };
 
   extraNodeConfs = {
-    provisionOld = {
+    provisionLegacyNotifiers = {
       services.grafana.provision = {
-        datasources = [{
-          name = "Test Datasource";
-          type = "testdata";
-          access = "proxy";
-          uid = "test_datasource";
-        }];
-
-        dashboards = [{ options.path = "/var/lib/grafana/dashboards"; }];
-
+        datasources.settings = {
+          apiVersion = 1;
+          datasources = [{
+            name = "Test Datasource";
+            type = "testdata";
+            access = "proxy";
+            uid = "test_datasource";
+          }];
+        };
+        dashboards.settings = {
+          apiVersion = 1;
+          providers = [{
+            name = "default";
+            options.path = "/var/lib/grafana/dashboards";
+          }];
+        };
         notifiers = [{
           uid = "test_notifiers";
           name = "Test Notifiers";
@@ -50,7 +57,6 @@ let
         }];
       };
     };
-
     provisionNix = {
       services.grafana.provision = {
         datasources.settings = {
@@ -157,6 +163,22 @@ let
         };
       };
     };
+
+    provisionYamlDirs = let
+      mkdir = p: pkgs.writeTextDir (baseNameOf p) (builtins.readFile p);
+    in {
+      services.grafana.provision = {
+        datasources.path = mkdir ./datasources.yaml;
+        dashboards.path = mkdir ./dashboards.yaml;
+        alerting = {
+          rules.path = mkdir ./rules.yaml;
+          contactPoints.path = mkdir ./contact-points.yaml;
+          policies.path = mkdir ./policies.yaml;
+          templates.path = mkdir ./templates.yaml;
+          muteTimings.path = mkdir ./mute-timings.yaml;
+        };
+      };
+    };
   };
 
   nodes = builtins.mapAttrs (_: val: mkMerge [ val baseGrafanaConf ]) extraNodeConfs;
@@ -172,58 +194,58 @@ in {
   testScript = ''
     start_all()
 
-    nodeOld = ("Nix (old format)", provisionOld)
     nodeNix = ("Nix (new format)", provisionNix)
     nodeYaml = ("Nix (YAML)", provisionYaml)
+    nodeYamlDir = ("Nix (YAML in dirs)", provisionYamlDirs)
 
-    for nodeInfo in [nodeOld, nodeNix, nodeYaml]:
-        with subtest(f"Should start provision node: {nodeInfo[0]}"):
-            nodeInfo[1].wait_for_unit("grafana.service")
-            nodeInfo[1].wait_for_open_port(3000)
+    for description, machine in [nodeNix, nodeYaml, nodeYamlDir]:
+        with subtest(f"Should start provision node: {description}"):
+            machine.wait_for_unit("grafana.service")
+            machine.wait_for_open_port(3000)
 
-        with subtest(f"Successful datasource provision with {nodeInfo[0]}"):
-            nodeInfo[1].succeed(
+        with subtest(f"Successful datasource provision with {description}"):
+            machine.succeed(
                 "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/datasources/uid/test_datasource | grep Test\ Datasource"
             )
 
-        with subtest(f"Successful dashboard provision with {nodeInfo[0]}"):
-            nodeInfo[1].succeed(
+        with subtest(f"Successful dashboard provision with {description}"):
+            machine.succeed(
                 "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/dashboards/uid/test_dashboard | grep Test\ Dashboard"
             )
 
-
-
-    with subtest(f"Successful notifiers provision with {nodeOld[0]}"):
-        nodeOld[1].succeed(
-            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/alert-notifications/uid/test_notifiers | grep Test\ Notifiers"
-        )
-
-
-
-    for nodeInfo in [nodeNix, nodeYaml]:
-        with subtest(f"Successful rule provision with {nodeInfo[0]}"):
-            nodeInfo[1].succeed(
+        with subtest(f"Successful rule provision with {description}"):
+            machine.succeed(
                 "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/alert-rules/test_rule | grep Test\ Rule"
             )
 
-        with subtest(f"Successful contact point provision with {nodeInfo[0]}"):
-            nodeInfo[1].succeed(
+        with subtest(f"Successful contact point provision with {description}"):
+            machine.succeed(
                 "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/contact-points | grep Test\ Contact\ Point"
             )
 
-        with subtest(f"Successful policy provision with {nodeInfo[0]}"):
-            nodeInfo[1].succeed(
+        with subtest(f"Successful policy provision with {description}"):
+            machine.succeed(
                 "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/policies | grep Test\ Contact\ Point"
             )
 
-        with subtest(f"Successful template provision with {nodeInfo[0]}"):
-            nodeInfo[1].succeed(
+        with subtest(f"Successful template provision with {description}"):
+            machine.succeed(
                 "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/templates | grep Test\ Template"
             )
 
-        with subtest("Successful mute timings provision with {nodeInfo[0]}"):
-            nodeInfo[1].succeed(
+        with subtest("Successful mute timings provision with {description}"):
+            machine.succeed(
                 "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/v1/provisioning/mute-timings | grep Test\ Mute\ Timing"
             )
+
+    with subtest("Successful notifiers provision"):
+        provisionLegacyNotifiers.wait_for_unit("grafana.service")
+        provisionLegacyNotifiers.wait_for_open_port(3000)
+        print(provisionLegacyNotifiers.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/alert-notifications/uid/test_notifiers"
+        ))
+        provisionLegacyNotifiers.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/alert-notifications/uid/test_notifiers | grep Test\ Notifiers"
+        )
   '';
 })