Merge master into staging-next

9 files changed, 391 insertions, 36 deletions

diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index 4341c8c238a86..5041f2ace2271 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -540,6 +540,7 @@
   ./services/monitoring/do-agent.nix
   ./services/monitoring/fusion-inventory.nix
   ./services/monitoring/grafana.nix
+  ./services/monitoring/grafana-image-renderer.nix
   ./services/monitoring/grafana-reporter.nix
   ./services/monitoring/graphite.nix
   ./services/monitoring/hdaps.nix
diff --git a/nixos/modules/services/monitoring/grafana-image-renderer.nix b/nixos/modules/services/monitoring/grafana-image-renderer.nix
new file mode 100644
index 0000000000000..a010a5316baeb
--- /dev/null
+++ b/nixos/modules/services/monitoring/grafana-image-renderer.nix
@@ -0,0 +1,150 @@
+{ lib, pkgs, config, ... }:
+
+with lib;
+
+let
+  cfg = config.services.grafana-image-renderer;
+
+  format = pkgs.formats.json { };
+
+  configFile = format.generate "grafana-image-renderer-config.json" cfg.settings;
+in {
+  options.services.grafana-image-renderer = {
+    enable = mkEnableOption "grafana-image-renderer";
+
+    chromium = mkOption {
+      type = types.package;
+      description = ''
+        The chromium to use for image rendering.
+      '';
+    };
+
+    verbose = mkEnableOption "verbosity for the service";
+
+    provisionGrafana = mkEnableOption "Grafana configuration for grafana-image-renderer";
+
+    settings = mkOption {
+      type = types.submodule {
+        freeformType = format.type;
+
+        options = {
+          service = {
+            port = mkOption {
+              type = types.port;
+              default = 8081;
+              description = ''
+                The TCP port to use for the rendering server.
+              '';
+            };
+            logging.level = mkOption {
+              type = types.enum [ "error" "warning" "info" "debug" ];
+              default = "info";
+              description = ''
+                The log-level of the <filename>grafana-image-renderer.service</filename>-unit.
+              '';
+            };
+          };
+          rendering = {
+            width = mkOption {
+              default = 1000;
+              type = types.ints.positive;
+              description = ''
+                Width of the PNG used to display the alerting graph.
+              '';
+            };
+            height = mkOption {
+              default = 500;
+              type = types.ints.positive;
+              description = ''
+                Height of the PNG used to display the alerting graph.
+              '';
+            };
+            mode = mkOption {
+              default = "default";
+              type = types.enum [ "default" "reusable" "clustered" ];
+              description = ''
+                Rendering mode of <package>grafana-image-renderer</package>:
+                <itemizedlist>
+                <listitem><para><literal>default:</literal> Creates on browser-instance
+                  per rendering request.</para></listitem>
+                <listitem><para><literal>reusable:</literal> One browser instance
+                  will be started and reused for each rendering request.</para></listitem>
+                <listitem><para><literal>clustered:</literal> allows to precisely
+                  configure how many browser-instances are supposed to be used. The values
+                  for that mode can be declared in <literal>rendering.clustering</literal>.
+                  </para></listitem>
+                </itemizedlist>
+              '';
+            };
+            args = mkOption {
+              type = types.listOf types.str;
+              default = [ "--no-sandbox" ];
+              description = ''
+                List of CLI flags passed to <package>chromium</package>.
+              '';
+            };
+          };
+        };
+      };
+
+      default = {};
+
+      description = ''
+        Configuration attributes for <package>grafana-image-renderer</package>.
+
+        See <link xlink:href="https://github.com/grafana/grafana-image-renderer/blob/ce1f81438e5f69c7fd7c73ce08bab624c4c92e25/default.json" />
+        for supported values.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    assertions = [
+      { assertion = cfg.provisionGrafana -> config.services.grafana.enable;
+        message = ''
+          To provision a Grafana instance to use grafana-image-renderer,
+          `services.grafana.enable` must be set to `true`!
+        '';
+      }
+    ];
+
+    services.grafana.extraOptions = mkIf cfg.provisionGrafana {
+      RENDERING_SERVER_URL = "http://localhost:${toString cfg.settings.service.port}/render";
+      RENDERING_CALLBACK_URL = "http://localhost:${toString config.services.grafana.port}";
+    };
+
+    services.grafana-image-renderer.chromium = mkDefault pkgs.chromium;
+
+    services.grafana-image-renderer.settings = {
+      rendering = mapAttrs (const mkDefault) {
+        chromeBin = "${cfg.chromium}/bin/chromium";
+        verboseLogging = cfg.verbose;
+        timezone = config.time.timeZone;
+      };
+
+      services = {
+        logging.level = mkIf cfg.verbose (mkDefault "debug");
+        metrics.enabled = mkDefault false;
+      };
+    };
+
+    systemd.services.grafana-image-renderer = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      description = " A Grafana backend plugin that handles rendering of panels & dashboards to PNGs using headless browser (Chromium/Chrome)";
+
+      environment = {
+        PUPPETEER_SKIP_CHROMIUM_DOWNLOAD = "true";
+      };
+
+      serviceConfig = {
+        DynamicUser = true;
+        PrivateTmp = true;
+        ExecStart = "${pkgs.grafana-image-renderer}/bin/grafana-image-renderer server --config=${configFile}";
+        Restart = "always";
+      };
+    };
+  };
+
+  meta.maintainers = with maintainers; [ ma27 ];
+}
diff --git a/nixos/modules/services/monitoring/grafana.nix b/nixos/modules/services/monitoring/grafana.nix
index b0c81a46d4d81..c8515c4b8988c 100644
--- a/nixos/modules/services/monitoring/grafana.nix
+++ b/nixos/modules/services/monitoring/grafana.nix
@@ -5,10 +5,11 @@ with lib;
 let
   cfg = config.services.grafana;
   opt = options.services.grafana;
+  declarativePlugins = pkgs.linkFarm "grafana-plugins" (builtins.map (pkg: { name = pkg.pname; path = pkg; }) cfg.declarativePlugins);
 
   envOptions = {
     PATHS_DATA = cfg.dataDir;
-    PATHS_PLUGINS = "${cfg.dataDir}/plugins";
+    PATHS_PLUGINS = if builtins.isNull cfg.declarativePlugins then "${cfg.dataDir}/plugins" else declarativePlugins;
     PATHS_LOGS = "${cfg.dataDir}/log";
 
     SERVER_PROTOCOL = cfg.protocol;
@@ -260,6 +261,12 @@ in {
       defaultText = "pkgs.grafana";
       type = types.package;
     };
+    declarativePlugins = mkOption {
+      type = with types; nullOr (listOf path);
+      default = null;
+      description = "If non-null, then a list of packages containing Grafana plugins to install. If set, plugins cannot be manually installed.";
+      example = literalExample "with pkgs.grafanaPlugins; [ grafana-piechart-panel ]";
+    };
 
     dataDir = mkOption {
       description = "Data directory.";
diff --git a/nixos/modules/services/monitoring/prometheus/default.nix b/nixos/modules/services/monitoring/prometheus/default.nix
index ace62342c9c63..19741b46f24e6 100644
--- a/nixos/modules/services/monitoring/prometheus/default.nix
+++ b/nixos/modules/services/monitoring/prometheus/default.nix
@@ -112,17 +112,21 @@ let
           http://tools.ietf.org/html/rfc4366#section-3.1
         '';
       };
-      remote_timeout = mkDefOpt types.str "30s" ''
-        Timeout for requests to the remote write endpoint.
-      '';
-      relabel_configs = mkOpt (types.listOf promTypes.relabel_config) ''
-         List of remote write relabel configurations.
-         List of relabel configurations.
-       '';
       name = mkOpt types.string ''
-        Name of the remote write config, which if specified must be unique among remote write configs.
+        Name of the remote read config, which if specified must be unique among remote read configs.
         The name will be used in metrics and logging in place of a generated value to help users distinguish between
-        remote write configs.
+        remote read configs.
+      '';
+      required_matchers = mkOpt (types.attrsOf types.str) ''
+        An optional list of equality matchers which have to be
+        present in a selector to query the remote read endpoint.
+      '';
+      remote_timeout = mkOpt types.str ''
+        Timeout for requests to the remote read endpoint.
+      '';
+      read_recent = mkOpt types.bool ''
+        Whether reads should be made for queries for time ranges that
+        the local storage should have complete data for.
       '';
       basic_auth = mkOpt (types.submodule {
         options = {
@@ -136,30 +140,22 @@ let
           password_file = mkOpt types.str "HTTP password file";
         };
       }) ''
-        Sets the `Authorization` header on every remote write request with the
+        Sets the `Authorization` header on every remote read request with the
         configured username and password.
         password and password_file are mutually exclusive.
       '';
       bearer_token = mkOpt types.str ''
-        Sets the `Authorization` header on every remote write request with
+        Sets the `Authorization` header on every remote read request with
         the configured bearer token. It is mutually exclusive with `bearer_token_file`.
       '';
       bearer_token_file = mkOpt types.str ''
-        Sets the `Authorization` header on every remote write request with the bearer token
+        Sets the `Authorization` header on every remote read request with the bearer token
         read from the configured file. It is mutually exclusive with `bearer_token`.
       '';
       tls_config = mkOpt promTypes.tls_config ''
-        Configures the remote write request's TLS settings.
+        Configures the remote read request's TLS settings.
       '';
       proxy_url = mkOpt types.str "Optional Proxy URL.";
-      metadata_config = {
-        send = mkDefOpt types.bool "true" ''
-          Whether metric metadata is sent to remote storage or not.
-        '';
-        send_interval = mkDefOpt types.str "1m" ''
-          How frequently metric metadata is sent to remote storage.
-        '';
-      };
     };
   };
 
@@ -172,13 +168,12 @@ let
           http://tools.ietf.org/html/rfc4366#section-3.1
         '';
       };
-      remote_timeout = mkDefOpt types.str "30s" ''
+      remote_timeout = mkOpt types.str ''
         Timeout for requests to the remote write endpoint.
       '';
-      relabel_configs = mkOpt (types.listOf promTypes.relabel_config) ''
-         List of remote write relabel configurations.
-         List of relabel configurations.
-       '';
+      write_relabel_configs = mkOpt (types.listOf promTypes.relabel_config) ''
+        List of remote write relabel configurations.
+      '';
       name = mkOpt types.string ''
         Name of the remote write config, which if specified must be unique among remote write configs.
         The name will be used in metrics and logging in place of a generated value to help users distinguish between
@@ -212,14 +207,50 @@ let
         Configures the remote write request's TLS settings.
       '';
       proxy_url = mkOpt types.str "Optional Proxy URL.";
-      metadata_config = {
-        send = mkDefOpt types.bool "true" ''
-          Whether metric metadata is sent to remote storage or not.
-        '';
-        send_interval = mkDefOpt types.str "1m" ''
-          How frequently metric metadata is sent to remote storage.
-        '';
-      };
+      queue_config = mkOpt (types.submodule {
+        options = {
+          capacity = mkOpt types.int ''
+            Number of samples to buffer per shard before we block reading of more
+            samples from the WAL. It is recommended to have enough capacity in each
+            shard to buffer several requests to keep throughput up while processing
+            occasional slow remote requests.
+          '';
+          max_shards = mkOpt types.int ''
+            Maximum number of shards, i.e. amount of concurrency.
+          '';
+          min_shards = mkOpt types.int ''
+            Minimum number of shards, i.e. amount of concurrency.
+          '';
+          max_samples_per_send = mkOpt types.int ''
+            Maximum number of samples per send.
+          '';
+          batch_send_deadline = mkOpt types.str ''
+            Maximum time a sample will wait in buffer.
+          '';
+          min_backoff = mkOpt types.str ''
+            Initial retry delay. Gets doubled for every retry.
+          '';
+          max_backoff = mkOpt types.str ''
+            Maximum retry delay.
+          '';
+        };
+      }) ''
+        Configures the queue used to write to remote storage.
+      '';
+      metadata_config = mkOpt (types.submodule {
+        options = {
+          send = mkOpt types.bool ''
+            Whether metric metadata is sent to remote storage or not.
+          '';
+          send_interval = mkOpt types.str ''
+            How frequently metric metadata is sent to remote storage.
+          '';
+        };
+      }) ''
+        Configures the sending of series metadata to remote storage.
+        Metadata configuration is subject to change at any point
+        or be removed in future releases.
+      '';
     };
   };
 
diff --git a/nixos/modules/services/monitoring/prometheus/exporters.nix b/nixos/modules/services/monitoring/prometheus/exporters.nix
index 683dc0390c59b..fe9d1a6590784 100644
--- a/nixos/modules/services/monitoring/prometheus/exporters.nix
+++ b/nixos/modules/services/monitoring/prometheus/exporters.nix
@@ -23,6 +23,7 @@ let
   exporterOpts = genAttrs [
     "apcupsd"
     "bind"
+    "bird"
     "blackbox"
     "collectd"
     "dnsmasq"
@@ -47,6 +48,7 @@ let
     "rspamd"
     "rtl_433"
     "snmp"
+    "smokeping"
     "sql"
     "surfboard"
     "tor"
diff --git a/nixos/modules/services/monitoring/prometheus/exporters/bird.nix b/nixos/modules/services/monitoring/prometheus/exporters/bird.nix
new file mode 100644
index 0000000000000..d8a526eafcea9
--- /dev/null
+++ b/nixos/modules/services/monitoring/prometheus/exporters/bird.nix
@@ -0,0 +1,46 @@
+{ config, lib, pkgs, options }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.exporters.bird;
+in
+{
+  port = 9324;
+  extraOpts = {
+    birdVersion = mkOption {
+      type = types.enum [ 1 2 ];
+      default = 2;
+      description = ''
+        Specifies whether BIRD1 or BIRD2 is in use.
+      '';
+    };
+    birdSocket = mkOption {
+      type = types.path;
+      default = "/var/run/bird.ctl";
+      description = ''
+        Path to BIRD2 (or BIRD1 v4) socket.
+      '';
+    };
+    newMetricFormat = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Enable the new more-generic metric format.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      SupplementaryGroups = singleton (if cfg.birdVersion == 1 then "bird" else "bird2");
+      ExecStart = ''
+        ${pkgs.prometheus-bird-exporter}/bin/bird_exporter \
+          -web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+          -bird.socket ${cfg.birdSocket} \
+          -bird.v2=${if cfg.birdVersion == 2 then "true" else "false"} \
+          -format.new=${if cfg.newMetricFormat then "true" else "false"} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}
diff --git a/nixos/modules/services/monitoring/prometheus/exporters/smokeping.nix b/nixos/modules/services/monitoring/prometheus/exporters/smokeping.nix
new file mode 100644
index 0000000000000..0a7bb9c27be28
--- /dev/null
+++ b/nixos/modules/services/monitoring/prometheus/exporters/smokeping.nix
@@ -0,0 +1,60 @@
+{ config, lib, pkgs, options }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.exporters.smokeping;
+  goDuration = types.mkOptionType {
+    name = "goDuration";
+    description = "Go duration (https://golang.org/pkg/time/#ParseDuration)";
+    check = x: types.str.check x && builtins.match "(-?[0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+" x != null;
+    inherit (types.str) merge;
+  };
+in
+{
+  port = 9374;
+  extraOpts = {
+    telemetryPath = mkOption {
+      type = types.str;
+      default = "/metrics";
+      description = ''
+        Path under which to expose metrics.
+      '';
+    };
+    pingInterval = mkOption {
+      type = goDuration;
+      default = "1s";
+      description = ''
+        Interval between pings.
+      '';
+    };
+    buckets = mkOption {
+      type = types.commas;
+      default = "5e-05,0.0001,0.0002,0.0004,0.0008,0.0016,0.0032,0.0064,0.0128,0.0256,0.0512,0.1024,0.2048,0.4096,0.8192,1.6384,3.2768,6.5536,13.1072,26.2144";
+      description = ''
+        List of buckets to use for the response duration histogram.
+      '';
+    };
+    hosts = mkOption {
+      type = with types; listOf str;
+      description = ''
+        List of endpoints to probe.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      AmbientCapabilities = [ "CAP_NET_RAW" ];
+      ExecStart = ''
+        ${pkgs.prometheus-smokeping-prober}/bin/smokeping_prober \
+          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+          --web.telemetry-path ${cfg.telemetryPath} \
+          --buckets ${cfg.buckets} \
+          --ping.interval ${cfg.pingInterval} \
+          --privileged \
+          ${concatStringsSep " \\\n  " cfg.extraFlags} \
+          ${concatStringsSep " " cfg.hosts}
+      '';
+    };
+  };
+}
diff --git a/nixos/tests/grafana.nix b/nixos/tests/grafana.nix
index 4b453ece7f1ef..4ba091b893f42 100644
--- a/nixos/tests/grafana.nix
+++ b/nixos/tests/grafana.nix
@@ -17,6 +17,10 @@ let
   };
 
   extraNodeConfs = {
+    declarativePlugins = {
+      services.grafana.declarativePlugins = [ pkgs.grafanaPlugins.grafana-clock-panel ];
+    };
+
     postgresql = {
       services.grafana.database = {
         host = "127.0.0.1:5432";
@@ -52,7 +56,7 @@ let
     nameValuePair dbName (mkMerge [
     baseGrafanaConf
     (extraNodeConfs.${dbName} or {})
-  ])) [ "sqlite" "postgresql" "mysql" ]);
+  ])) [ "sqlite" "declarativePlugins" "postgresql" "mysql" ]);
 
 in {
   name = "grafana";
@@ -66,6 +70,14 @@ in {
   testScript = ''
     start_all()
 
+    with subtest("Declarative plugins installed"):
+        declarativePlugins.wait_for_unit("grafana.service")
+        declarativePlugins.wait_for_open_port(3000)
+        declarativePlugins.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/plugins | grep -q grafana-clock-panel"
+        )
+        declarativePlugins.shutdown()
+
     with subtest("Successful API query as admin user with sqlite db"):
         sqlite.wait_for_unit("grafana.service")
         sqlite.wait_for_open_port(3000)
diff --git a/nixos/tests/prometheus-exporters.nix b/nixos/tests/prometheus-exporters.nix
index ffa7f420c093d..f611d961769ab 100644
--- a/nixos/tests/prometheus-exporters.nix
+++ b/nixos/tests/prometheus-exporters.nix
@@ -96,6 +96,31 @@ let
       '';
     };
 
+    bird = {
+      exporterConfig = {
+        enable = true;
+      };
+      metricProvider = {
+        services.bird2.enable = true;
+        services.bird2.config = ''
+          protocol kernel MyObviousTestString {
+            ipv4 {
+              import all;
+              export none;
+            };
+          }
+
+          protocol device {
+          }
+        '';
+      };
+      exporterTest = ''
+        wait_for_unit("prometheus-bird-exporter.service")
+        wait_for_open_port(9324)
+        succeed("curl -sSf http://localhost:9324/metrics | grep -q 'MyObviousTestString'")
+      '';
+    };
+
     blackbox = {
       exporterConfig = {
         enable = true;
@@ -670,6 +695,27 @@ let
       '';
     };
 
+    smokeping = {
+      exporterConfig = {
+        enable = true;
+        hosts = ["127.0.0.1"];
+      };
+      exporterTest = ''
+        wait_for_unit("prometheus-smokeping-exporter.service")
+        wait_for_open_port(9374)
+        wait_until_succeeds(
+            "curl -sSf localhost:9374/metrics | grep '{}' | grep -qv ' 0$'".format(
+                'smokeping_requests_total{host="127.0.0.1",ip="127.0.0.1"} '
+            )
+        )
+        wait_until_succeeds(
+            "curl -sSf localhost:9374/metrics | grep -q '{}'".format(
+                'smokeping_response_ttl{host="127.0.0.1",ip="127.0.0.1"}'
+            )
+        )
+      '';
+    };
+
     snmp = {
       exporterConfig = {
         enable = true;