module/prometheus: optionally support reloading on config changes

The new option `services.prometheus.enableReload` has been introduced
which, when enabled, causes the prometheus systemd service to reload
when its config file changes.

More specifically the following property holds: switching to a
configuration (`switch-to-configuration`) that changes the prometheus
configuration only finishes successully when prometheus has finished
loading the new configuration.

`enableReload` is `false` by default in which case the old semantics
of restarting the prometheus systemd service are in effect.

1 files changed, 100 insertions, 0 deletions

diff --git a/nixos/tests/prometheus.nix b/nixos/tests/prometheus.nix
index 70ac78a4a4689..d102b4c075186 100644
--- a/nixos/tests/prometheus.nix
+++ b/nixos/tests/prometheus.nix
@@ -41,6 +41,7 @@ in import ./make-test-python.nix {
       networking.firewall.allowedTCPPorts = [ grpcPort ];
       services.prometheus = {
         enable = true;
+        enableReload = true;
         scrapeConfigs = [
           {
             job_name = "prometheus";
@@ -118,6 +119,36 @@ in import ./make-test-python.nix {
         #  };
         #};
       };
+      # Adds a "specialisation" of the above config which allows us to
+      # "switch" to it and see if the services.prometheus.enableReload
+      # functionality actually reloads the prometheus service instead of
+      # restarting it.
+      specialisation = {
+        "prometheus-config-change" = {
+          configuration = {
+            environment.systemPackages = [ pkgs.yq ];
+
+            # This configuration just adds a new prometheus job
+            # to scrape the node_exporter metrics of the s3 machine.
+            # We also use an environmentFile to test if that works correctly.
+            services.prometheus = {
+              environmentFile = pkgs.writeText "prometheus-config-env-file" ''
+                JOB_NAME=s3-node_exporter
+              '';
+              scrapeConfigs = [
+                {
+                  job_name = "$JOB_NAME";
+                  static_configs = [
+                    {
+                      targets = [ "s3:9100" ];
+                    }
+                  ];
+                }
+              ];
+            };
+          };
+        };
+      };
     };
 
     query = { pkgs, ... }: {
@@ -171,10 +202,17 @@ in import ./make-test-python.nix {
       };
 
       environment.systemPackages = [ pkgs.minio-client ];
+
+      services.prometheus.exporters.node = {
+        enable = true;
+        openFirewall = true;
+      };
     };
   };
 
   testScript = { nodes, ... } : ''
+    import json
+
     # Before starting the other machines we first make sure that our S3 service is online
     # and has a bucket added for thanos:
     s3.start()
@@ -193,6 +231,12 @@ in import ./make-test-python.nix {
 
     # Check if prometheus responds to requests:
     prometheus.wait_for_unit("prometheus.service")
+
+    # Check if prometheus' config file is correctly locked down because it could contain secrets.
+    prometheus.succeed(
+        "stat -c '%a %U' /var/lib/prometheus2/prometheus-substituted.yaml | grep '600 prometheus'"
+    )
+
     prometheus.wait_for_open_port(${toString queryPort})
     prometheus.succeed("curl -sf http://127.0.0.1:${toString queryPort}/metrics")
 
@@ -245,5 +289,61 @@ in import ./make-test-python.nix {
         + "jq .thanos.labels.some_label | "
         + "grep 'required by thanos'"
     )
+
+    # Check if switching to a NixOS configuration that changes the prometheus
+    # configuration reloads (instead of restarts) prometheus before the switch
+    # finishes successfully:
+    with subtest("config change reloads prometheus"):
+        # We check if prometheus has finished reloading by looking for the message
+        # "Completed loading of configuration file" in the journal between the start
+        # and finish of switching to the new NixOS configuration.
+        #
+        # To mark the start we record the journal cursor before starting the switch:
+        cursor_before_switching = json.loads(
+            prometheus.succeed("journalctl -n1 -o json --output-fields=__CURSOR")
+        )["__CURSOR"]
+
+        # Now we switch:
+        prometheus_config_change = prometheus.succeed(
+            "readlink /run/current-system/specialisation/prometheus-config-change"
+        ).strip()
+        prometheus.succeed(prometheus_config_change + "/bin/switch-to-configuration test")
+
+        # Next we retrieve all logs since the start of switching:
+        logs_after_starting_switching = prometheus.succeed(
+            """
+              journalctl --after-cursor='{cursor_before_switching}' -o json --output-fields=MESSAGE
+            """.format(
+                cursor_before_switching=cursor_before_switching
+            )
+        )
+
+        # Finally we check if the message "Completed loading of configuration file"
+        # occurs before the "finished switching to system configuration" message:
+        finished_switching_msg = (
+            "finished switching to system configuration " + prometheus_config_change
+        )
+        reloaded_before_switching_finished = False
+        finished_switching = False
+        for log_line in logs_after_starting_switching.split("\n"):
+            msg = json.loads(log_line)["MESSAGE"]
+            if "Completed loading of configuration file" in msg:
+                reloaded_before_switching_finished = True
+            if msg == finished_switching_msg:
+                finished_switching = True
+                break
+
+        assert reloaded_before_switching_finished
+        assert finished_switching
+
+        # Check if the reloaded config includes the new s3-node_exporter job:
+        prometheus.succeed(
+          """
+            curl -sf http://127.0.0.1:${toString queryPort}/api/v1/status/config \
+              | jq -r .data.yaml \
+              | yq '.scrape_configs | any(.job_name == "s3-node_exporter")' \
+              | grep true
+          """
+        )
   '';
 }