Merge staging-next into staging

7 files changed, 198 insertions, 26 deletions

diff --git a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml
index 3f06e42ae98cd..a09a567b51aec 100644
--- a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml
+++ b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml
@@ -520,7 +520,9 @@
       <listitem>
         <para>
           <literal>pkgs.cosign</literal> does not provide the
-          <literal>cosigned</literal> binary anymore.
+          <literal>cosigned</literal> binary anymore. The
+          <literal>sget</literal> binary has been moved into its own
+          package.
         </para>
       </listitem>
       <listitem>
diff --git a/nixos/doc/manual/release-notes/rl-2211.section.md b/nixos/doc/manual/release-notes/rl-2211.section.md
index 99acf6e64b62c..94840012e58b1 100644
--- a/nixos/doc/manual/release-notes/rl-2211.section.md
+++ b/nixos/doc/manual/release-notes/rl-2211.section.md
@@ -173,7 +173,7 @@ Available as [services.patroni](options.html#opt-services.patroni.enable).
 - PHP 7.4 is no longer supported due to upstream not supporting this
   version for the entire lifecycle of the 22.11 release.
 
-- `pkgs.cosign` does not provide the `cosigned` binary anymore.
+- `pkgs.cosign` does not provide the `cosigned` binary anymore. The `sget` binary has been moved into its own package.
 
 - Emacs now uses the Lucid toolkit by default instead of GTK because of stability and compatibility issues.
   Users who still wish to remain using GTK can do so by using `emacs-gtk`.
diff --git a/nixos/lib/utils.nix b/nixos/lib/utils.nix
index d7671a374999a..f646f70323e35 100644
--- a/nixos/lib/utils.nix
+++ b/nixos/lib/utils.nix
@@ -102,7 +102,11 @@ rec {
         if item ? ${attr} then
           nameValuePair prefix item.${attr}
         else if isAttrs item then
-          map (name: recurse (prefix + "." + name) item.${name}) (attrNames item)
+          map (name:
+            let
+              escapedName = ''"${replaceChars [''"'' "\\"] [''\"'' "\\\\"] name}"'';
+            in
+              recurse (prefix + "." + escapedName) item.${name}) (attrNames item)
         else if isList item then
           imap0 (index: item: recurse (prefix + "[${toString index}]") item) item
         else
@@ -182,13 +186,13 @@ rec {
                 '')
                (attrNames secrets))
     + "\n"
-    + "${pkgs.jq}/bin/jq >'${output}' '"
-    + concatStringsSep
+    + "${pkgs.jq}/bin/jq >'${output}' "
+    + lib.escapeShellArg (concatStringsSep
       " | "
       (imap1 (index: name: ''${name} = $ENV.secret${toString index}'')
-             (attrNames secrets))
+             (attrNames secrets)))
     + ''
-      ' <<'EOF'
+       <<'EOF'
       ${builtins.toJSON set}
       EOF
       (( ! $inherit_errexit_enabled )) && shopt -u inherit_errexit
diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix
index c41f2efeec474..4988517a9b66a 100644
--- a/nixos/modules/services/misc/gitlab.nix
+++ b/nixos/modules/services/misc/gitlab.nix
@@ -6,6 +6,9 @@ let
   cfg = config.services.gitlab;
   opt = options.services.gitlab;
 
+  toml = pkgs.formats.toml {};
+  yaml = pkgs.formats.yaml {};
+
   ruby = cfg.packages.gitlab.ruby;
 
   postgresqlPackage = if config.services.postgresql.enable then
@@ -89,17 +92,18 @@ let
     repos_path = "${cfg.statePath}/repositories";
     secret_file = "${cfg.statePath}/gitlab_shell_secret";
     log_file = "${cfg.statePath}/log/gitlab-shell.log";
-    redis = {
-      bin = "${pkgs.redis}/bin/redis-cli";
-      host = "127.0.0.1";
-      port = config.services.redis.servers.gitlab.port;
-      database = 0;
-      namespace = "resque:gitlab";
-    };
   };
 
   redisConfig.production.url = cfg.redisUrl;
 
+  cableYml = yaml.generate "cable.yml" {
+    production = {
+      adapter = "redis";
+      url = cfg.redisUrl;
+      channel_prefix = "gitlab_production";
+    };
+  };
+
   pagesArgs = [
     "-pages-domain" gitlabConfig.production.pages.host
     "-pages-root" "${gitlabConfig.production.shared.path}/pages"
@@ -188,6 +192,17 @@ let
     MALLOC_ARENA_MAX = "2";
   } // cfg.extraEnv;
 
+  runtimeDeps = with pkgs; [
+    nodejs
+    gzip
+    git
+    gnutar
+    postgresqlPackage
+    coreutils
+    procps
+    findutils # Needed for gitlab:cleanup:orphan_job_artifact_files
+  ];
+
   gitlab-rake = pkgs.stdenv.mkDerivation {
     name = "gitlab-rake";
     nativeBuildInputs = [ pkgs.makeWrapper ];
@@ -197,7 +212,7 @@ let
       mkdir -p $out/bin
       makeWrapper ${cfg.packages.gitlab.rubyEnv}/bin/rake $out/bin/gitlab-rake \
           ${concatStrings (mapAttrsToList (name: value: "--set ${name} '${value}' ") gitlabEnv)} \
-          --set PATH '${lib.makeBinPath [ pkgs.nodejs pkgs.gzip pkgs.git pkgs.gnutar postgresqlPackage pkgs.coreutils pkgs.procps ]}:$PATH' \
+          --set PATH '${lib.makeBinPath runtimeDeps}:$PATH' \
           --set RAKEOPT '-f ${cfg.packages.gitlab}/share/gitlab/Rakefile' \
           --chdir '${cfg.packages.gitlab}/share/gitlab'
      '';
@@ -212,7 +227,7 @@ let
       mkdir -p $out/bin
       makeWrapper ${cfg.packages.gitlab.rubyEnv}/bin/rails $out/bin/gitlab-rails \
           ${concatStrings (mapAttrsToList (name: value: "--set ${name} '${value}' ") gitlabEnv)} \
-          --set PATH '${lib.makeBinPath [ pkgs.nodejs pkgs.gzip pkgs.git pkgs.gnutar postgresqlPackage pkgs.coreutils pkgs.procps ]}:$PATH' \
+          --set PATH '${lib.makeBinPath runtimeDeps}:$PATH' \
           --chdir '${cfg.packages.gitlab}/share/gitlab'
      '';
   };
@@ -468,9 +483,9 @@ in {
 
       redisUrl = mkOption {
         type = types.str;
-        default = "redis://localhost:${toString config.services.redis.servers.gitlab.port}/";
-        defaultText = literalExpression ''redis://localhost:''${toString config.services.redis.servers.gitlab.port}/'';
-        description = lib.mdDoc "Redis URL for all GitLab services except gitlab-shell";
+        default = "unix:/run/gitlab/redis.sock";
+        example = "redis://localhost:6379/";
+        description = lib.mdDoc "Redis URL for all GitLab services.";
       };
 
       extraGitlabRb = mkOption {
@@ -867,8 +882,41 @@ in {
         };
       };
 
+      workhorse.config = mkOption {
+        type = toml.type;
+        default = {};
+        example = literalExpression ''
+          {
+            object_storage.provider = "AWS";
+            object_storage.s3 = {
+              aws_access_key_id = "AKIAXXXXXXXXXXXXXXXX";
+              aws_secret_access_key = { _secret = "/var/keys/aws_secret_access_key"; };
+            };
+          };
+        '';
+        description = lib.mdDoc ''
+          Configuration options to add to Workhorse's configuration
+          file.
+
+          See
+          <https://gitlab.com/gitlab-org/gitlab/-/blob/master/workhorse/config.toml.example>
+          and
+          <https://docs.gitlab.com/ee/development/workhorse/configuration.html>
+          for examples and option documentation.
+
+          Options containing secret data should be set to an attribute
+          set containing the attribute `_secret` - a string pointing
+          to a file containing the value the option should be set
+          to. See the example to get a better picture of this: in the
+          resulting configuration file, the
+          `object_storage.s3.aws_secret_access_key` key will be set to
+          the contents of the {file}`/var/keys/aws_secret_access_key`
+          file.
+        '';
+      };
+
       extraConfig = mkOption {
-        type = types.attrs;
+        type = yaml.type;
         default = {};
         example = literalExpression ''
           {
@@ -972,8 +1020,9 @@ in {
     # Redis is required for the sidekiq queue runner.
     services.redis.servers.gitlab = {
       enable = mkDefault true;
-      port = mkDefault 31636;
-      bind = mkDefault "127.0.0.1";
+      user = mkDefault cfg.user;
+      unixSocket = mkDefault "/run/gitlab/redis.sock";
+      unixSocketPerm = mkDefault 770;
     };
 
     # We use postgres as the main data store.
@@ -1062,6 +1111,7 @@ in {
     # Ensure Docker Registry launches after the certificate generation job
     systemd.services.docker-registry = optionalAttrs cfg.registry.enable {
       wants = [ "gitlab-registry-cert.service" ];
+      after = [ "gitlab-registry-cert.service" ];
     };
 
     # Enable Docker Registry, if GitLab-Container Registry is enabled
@@ -1115,6 +1165,7 @@ in {
       "d ${gitlabConfig.production.shared.path}/lfs-objects 0750 ${cfg.user} ${cfg.group} -"
       "d ${gitlabConfig.production.shared.path}/packages 0750 ${cfg.user} ${cfg.group} -"
       "d ${gitlabConfig.production.shared.path}/pages 0750 ${cfg.user} ${cfg.group} -"
+      "d ${gitlabConfig.production.shared.path}/registry 0750 ${cfg.user} ${cfg.group} -"
       "d ${gitlabConfig.production.shared.path}/terraform_state 0750 ${cfg.user} ${cfg.group} -"
       "L+ /run/gitlab/config - - - - ${cfg.statePath}/config"
       "L+ /run/gitlab/log - - - - ${cfg.statePath}/log"
@@ -1168,6 +1219,7 @@ in {
           cp -rf --no-preserve=mode ${cfg.packages.gitlab}/share/gitlab/config.dist/* ${cfg.statePath}/config
           cp -rf --no-preserve=mode ${cfg.packages.gitlab}/share/gitlab/db/* ${cfg.statePath}/db
           ln -sf ${extraGitlabRb} ${cfg.statePath}/config/initializers/extra-gitlab.rb
+          ln -sf ${cableYml} ${cfg.statePath}/config/cable.yml
 
           ${cfg.packages.gitlab-shell}/bin/install
 
@@ -1357,6 +1409,7 @@ in {
       wantedBy = [ "gitlab.target" ];
       partOf = [ "gitlab.target" ];
       path = with pkgs; [
+        remarshal
         exiftool
         gitPackage
         gnutar
@@ -1371,6 +1424,17 @@ in {
         TimeoutSec = "infinity";
         Restart = "on-failure";
         WorkingDirectory = gitlabEnv.HOME;
+        ExecStartPre = pkgs.writeShellScript "gitlab-workhorse-pre-start" ''
+          set -o errexit -o pipefail -o nounset
+          shopt -s dotglob nullglob inherit_errexit
+
+          ${utils.genJqSecretsReplacementSnippet
+              cfg.workhorse.config
+              "${cfg.statePath}/config/gitlab-workhorse.json"}
+
+          json2toml "${cfg.statePath}/config/gitlab-workhorse.json" "${cfg.statePath}/config/gitlab-workhorse.toml"
+          rm "${cfg.statePath}/config/gitlab-workhorse.json"
+        '';
         ExecStart =
           "${cfg.packages.gitlab-workhorse}/bin/workhorse "
           + "-listenUmask 0 "
@@ -1378,6 +1442,7 @@ in {
           + "-listenAddr /run/gitlab/gitlab-workhorse.socket "
           + "-authSocket ${gitlabSocket} "
           + "-documentRoot ${cfg.packages.gitlab}/share/gitlab/public "
+          + "-config ${cfg.statePath}/config/gitlab-workhorse.toml "
           + "-secretPath ${cfg.statePath}/.gitlab_workhorse_secret";
       };
     };
diff --git a/nixos/modules/services/networking/coturn.nix b/nixos/modules/services/networking/coturn.nix
index 4d83d2d48e377..2f34a72377ce2 100644
--- a/nixos/modules/services/networking/coturn.nix
+++ b/nixos/modules/services/networking/coturn.nix
@@ -335,9 +335,10 @@ in {
         preStart = ''
           cat ${configFile} > ${runConfig}
           ${optionalString (cfg.static-auth-secret-file != null) ''
-            STATIC_AUTH_SECRET="$(head -n1 ${cfg.static-auth-secret-file} || :)"
-            sed -e "s,#static-auth-secret#,$STATIC_AUTH_SECRET,g" \
-              -i ${runConfig}
+            ${pkgs.replace-secret}/bin/replace-secret \
+              "#static-auth-secret#" \
+              ${cfg.static-auth-secret-file} \
+              ${runConfig}
           '' }
           chmod 640 ${runConfig}
         '';
diff --git a/nixos/modules/services/security/privacyidea.nix b/nixos/modules/services/security/privacyidea.nix
index 5cd338ebf7fe0..e446e606cad8b 100644
--- a/nixos/modules/services/security/privacyidea.nix
+++ b/nixos/modules/services/security/privacyidea.nix
@@ -61,6 +61,12 @@ let
       (flip mapAttrs cfg.ldap-proxy.settings
         (const (mapAttrs (const renderValue)))));
 
+  privacyidea-token-janitor = pkgs.writeShellScriptBin "privacyidea-token-janitor" ''
+    exec -a privacyidea-token-janitor \
+      /run/wrappers/bin/sudo -u ${cfg.user} \
+      env PRIVACYIDEA_CONFIGFILE=${cfg.stateDir}/privacyidea.cfg \
+      ${penv}/bin/privacyidea-token-janitor $@
+  '';
 in
 
 {
@@ -178,6 +184,42 @@ in
         description = lib.mdDoc "Group account under which PrivacyIDEA runs.";
       };
 
+      tokenjanitor = {
+        enable = mkEnableOption (lib.mdDoc "automatic runs of the token janitor");
+        interval = mkOption {
+          default = "quarterly";
+          type = types.str;
+          description = lib.mdDoc ''
+            Interval in which the cleanup program is supposed to run.
+            See {manpage}`systemd.time(7)` for further information.
+          '';
+        };
+        action = mkOption {
+          type = types.enum [ "delete" "mark" "disable" "unassign" ];
+          description = lib.mdDoc ''
+            Which action to take for matching tokens.
+          '';
+        };
+        unassigned = mkOption {
+          default = false;
+          type = types.bool;
+          description = lib.mdDoc ''
+            Whether to search for **unassigned** tokens
+            and apply [](#opt-services.privacyidea.tokenjanitor.action)
+            onto them.
+          '';
+        };
+        orphaned = mkOption {
+          default = true;
+          type = types.bool;
+          description = lib.mdDoc ''
+            Whether to search for **orphaned** tokens
+            and apply [](#opt-services.privacyidea.tokenjanitor.action)
+            onto them.
+          '';
+        };
+      };
+
       ldap-proxy = {
         enable = mkEnableOption (lib.mdDoc "PrivacyIDEA LDAP Proxy");
 
@@ -228,10 +270,60 @@ in
 
     (mkIf cfg.enable {
 
-      environment.systemPackages = [ pkgs.privacyidea ];
+      assertions = [
+        {
+          assertion = cfg.tokenjanitor.enable -> (cfg.tokenjanitor.orphaned || cfg.tokenjanitor.unassigned);
+          message = ''
+            privacyidea-token-janitor has no effect if neither orphaned nor unassigned tokens
+            are to be searched.
+          '';
+        }
+      ];
+
+      environment.systemPackages = [ pkgs.privacyidea (hiPrio privacyidea-token-janitor) ];
 
       services.postgresql.enable = mkDefault true;
 
+      systemd.services.privacyidea-tokenjanitor = mkIf cfg.tokenjanitor.enable {
+        environment.PRIVACYIDEA_CONFIGFILE = "${cfg.stateDir}/privacyidea.cfg";
+        path = [ penv ];
+        serviceConfig = {
+          CapabilityBoundingSet = [ "" ];
+          ExecStart = "${pkgs.writeShellScript "pi-token-janitor" ''
+            ${optionalString cfg.tokenjanitor.orphaned ''
+              echo >&2 "Removing orphaned tokens..."
+              privacyidea-token-janitor find \
+                --orphaned true \
+                --action ${cfg.tokenjanitor.action}
+            ''}
+            ${optionalString cfg.tokenjanitor.unassigned ''
+              echo >&2 "Removing unassigned tokens..."
+              privacyidea-token-janitor find \
+                --assigned false \
+                --action ${cfg.tokenjanitor.action}
+            ''}
+          ''}";
+          Group = cfg.group;
+          LockPersonality = true;
+          MemoryDenyWriteExecute = true;
+          ProtectHome = true;
+          ProtectHostname = true;
+          ProtectKernelLogs = true;
+          ProtectKernelModules = true;
+          ProtectKernelTunables = true;
+          ProtectSystem = "strict";
+          ReadWritePaths = cfg.stateDir;
+          Type = "oneshot";
+          User = cfg.user;
+          WorkingDirectory = cfg.stateDir;
+        };
+      };
+      systemd.timers.privacyidea-tokenjanitor = mkIf cfg.tokenjanitor.enable {
+        wantedBy = [ "timers.target" ];
+        timerConfig.OnCalendar = cfg.tokenjanitor.interval;
+        timerConfig.Persistent = true;
+      };
+
       systemd.services.privacyidea = let
         piuwsgi = pkgs.writeText "uwsgi.json" (builtins.toJSON {
           uwsgi = {
diff --git a/nixos/tests/paperless.nix b/nixos/tests/paperless.nix
index 12883cd62c60c..b97834835c2c9 100644
--- a/nixos/tests/paperless.nix
+++ b/nixos/tests/paperless.nix
@@ -40,5 +40,13 @@ import ./make-test-python.nix ({ lib, ... }: {
         docs = json.loads(machine.succeed("curl -u admin:admin -fs localhost:28981/api/documents/"))['results']
         assert "2005-10-16" in docs[0]['created']
         assert "2005-10-16" in docs[1]['created']
+
+    # Detects gunicorn issues, see PR #190888
+    with subtest("Document metadata can be accessed"):
+        metadata = json.loads(machine.succeed("curl -u admin:admin -fs localhost:28981/api/documents/1/metadata/"))
+        assert "original_checksum" in metadata
+
+        metadata = json.loads(machine.succeed("curl -u admin:admin -fs localhost:28981/api/documents/2/metadata/"))
+        assert "original_checksum" in metadata
   '';
 })