From 31fe90d6effeb480057b400a9d7bf976021626a0 Mon Sep 17 00:00:00 2001 From: talyz Date: Tue, 13 Oct 2020 11:44:02 +0200 Subject: nixos/keycloak: Add test --- nixos/tests/all-tests.nix | 1 + nixos/tests/keycloak.nix | 139 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 nixos/tests/keycloak.nix (limited to 'nixos/tests') diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 9ffeba27a7fe4..5a10f60fc9a38 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -175,6 +175,7 @@ in kernel-latest = handleTest ./kernel-latest.nix {}; kernel-lts = handleTest ./kernel-lts.nix {}; kernel-testing = handleTest ./kernel-testing.nix {}; + keycloak = handleTest ./keycloak.nix {}; keymap = handleTest ./keymap.nix {}; knot = handleTest ./knot.nix {}; krb5 = discoverTests (import ./krb5 {}); diff --git a/nixos/tests/keycloak.nix b/nixos/tests/keycloak.nix new file mode 100644 index 0000000000000..e5e31b038e9cf --- /dev/null +++ b/nixos/tests/keycloak.nix @@ -0,0 +1,139 @@ +# This tests Keycloak: it starts the service, creates a realm with an +# OIDC client and a user, and simulates the user logging in to the +# client using their Keycloak login. + +import ./make-test-python.nix ( + { pkgs, ... }: + let + frontendUrl = "http://keycloak/auth"; + initialAdminPassword = "h4IhoJFnt2iQIR9"; + in + { + name = "keycloak"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ talyz ]; + }; + + nodes = { + keycloak = { ... }: { + virtualisation.memorySize = 1024; + services.keycloak = { + enable = true; + inherit frontendUrl initialAdminPassword; + databasePasswordFile = pkgs.writeText "dbPassword" "wzf6vOCbPp6cqTH"; + }; + environment.systemPackages = with pkgs; [ + xmlstarlet + libtidy + jq + ]; + }; + }; + + testScript = + let + client = { + clientId = "test-client"; + name = "test-client"; + redirectUris = [ "urn:ietf:wg:oauth:2.0:oob" ]; + }; + + user = { + firstName = "Chuck"; + lastName = "Testa"; + username = "chuck.testa"; + email = "chuck.testa@example.com"; + }; + + password = "password1234"; + + realm = { + enabled = true; + realm = "test-realm"; + clients = [ client ]; + users = [( + user // { + enabled = true; + credentials = [{ + type = "password"; + temporary = false; + value = password; + }]; + } + )]; + }; + + realmDataJson = pkgs.writeText "realm-data.json" (builtins.toJSON realm); + + jqCheckUserinfo = pkgs.writeText "check-userinfo.jq" '' + if { + "firstName": .given_name, + "lastName": .family_name, + "username": .preferred_username, + "email": .email + } != ${builtins.toJSON user} then + error("Wrong user info!") + else + empty + end + ''; + in '' + keycloak.start() + keycloak.wait_for_unit("keycloak.service") + keycloak.wait_until_succeeds("curl -sSf ${frontendUrl}") + + + ### Realm Setup ### + + # Get an admin interface access token + keycloak.succeed( + "curl -sSf -d 'client_id=admin-cli' -d 'username=admin' -d 'password=${initialAdminPassword}' -d 'grant_type=password' '${frontendUrl}/realms/master/protocol/openid-connect/token' | jq -r '\"Authorization: bearer \" + .access_token' >admin_auth_header" + ) + + # Publish the realm, including a test OIDC client and user + keycloak.succeed( + "curl -sSf -H @admin_auth_header -X POST -H 'Content-Type: application/json' -d @${realmDataJson} '${frontendUrl}/admin/realms/'" + ) + + # Generate and save the client secret. To do this we need + # Keycloak's internal id for the client. + keycloak.succeed( + "curl -sSf -H @admin_auth_header '${frontendUrl}/admin/realms/${realm.realm}/clients?clientId=${client.name}' | jq -r '.[].id' >client_id", + "curl -sSf -H @admin_auth_header -X POST '${frontendUrl}/admin/realms/${realm.realm}/clients/'$(client_secret", + ) + + + ### Authentication Testing ### + + # Start the login process by sending an initial request to the + # OIDC authentication endpoint, saving the returned page. Tidy + # up the HTML (XmlStarlet is picky) and extract the login form + # post url. + keycloak.succeed( + "curl -sSf -c cookie '${frontendUrl}/realms/${realm.realm}/protocol/openid-connect/auth?client_id=${client.name}&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=openid+email&response_type=code&response_mode=query&nonce=qw4o89g3qqm' >login_form", + "tidy -q -m login_form || true", + "xml sel -T -t -m \"_:html/_:body/_:div/_:div/_:div/_:div/_:div/_:div/_:form[@id='kc-form-login']\" -v @action login_form >form_post_url", + ) + + # Post the login form and save the response. Once again tidy up + # the HTML, then extract the authorization code. + keycloak.succeed( + "curl -sSf -L -b cookie -d 'username=${user.username}' -d 'password=${password}' -d 'credentialId=' \"$(auth_code_html", + "tidy -q -m auth_code_html || true", + "xml sel -T -t -m \"_:html/_:body/_:div/_:div/_:div/_:div/_:div/_:input[@id='code']\" -v @value auth_code_html >auth_code", + ) + + # Exchange the authorization code for an access token. + keycloak.succeed( + "curl -sSf -d grant_type=authorization_code -d code=$(auth_header" + ) + + # Use the access token on the OIDC userinfo endpoint and check + # that the returned user info matches what we initialized the + # realm with. + keycloak.succeed( + "curl -sSf -H @auth_header '${frontendUrl}/realms/${realm.realm}/protocol/openid-connect/userinfo' | jq -f ${jqCheckUserinfo}" + ) + ''; + } +) -- cgit 1.4.1 From 89e83833af35bd0ec3fdc65c435358a676a41d89 Mon Sep 17 00:00:00 2001 From: talyz Date: Mon, 26 Oct 2020 15:33:57 +0100 Subject: nixos/keycloak: Add support for MySQL and external DBs with SSL - Add support for using MySQL as an option to PostgreSQL. - Enable connecting to external DBs with SSL - Add a database port config option --- nixos/modules/services/web-apps/keycloak.nix | 265 ++++++++++++++++++++------- nixos/modules/services/web-apps/keycloak.xml | 27 ++- nixos/tests/all-tests.nix | 2 +- nixos/tests/keycloak.nix | 21 ++- pkgs/servers/keycloak/default.nix | 23 ++- 5 files changed, 243 insertions(+), 95 deletions(-) (limited to 'nixos/tests') diff --git a/nixos/modules/services/web-apps/keycloak.nix b/nixos/modules/services/web-apps/keycloak.nix index 9c6a5ca305cbb..bbb0c8d048313 100644 --- a/nixos/modules/services/web-apps/keycloak.nix +++ b/nixos/modules/services/web-apps/keycloak.nix @@ -97,11 +97,59 @@ in ''; }; + databaseType = lib.mkOption { + type = lib.types.enum [ "mysql" "postgresql" ]; + default = "postgresql"; + example = "mysql"; + description = '' + The type of database Keycloak should connect to. + ''; + }; + databaseHost = lib.mkOption { type = lib.types.str; default = "localhost"; description = '' - Hostname of the PostgreSQL database to connect to. + Hostname of the database to connect to. + ''; + }; + + databasePort = + let + dbPorts = { + postgresql = 5432; + mysql = 3306; + }; + in + lib.mkOption { + type = lib.types.port; + default = dbPorts.${cfg.databaseType}; + description = '' + Port of the database to connect to. + ''; + }; + + databaseUseSSL = lib.mkOption { + type = lib.types.bool; + default = cfg.databaseHost != "localhost"; + description = '' + Whether the database connection should be secured by SSL / + TLS. + ''; + }; + + databaseCaCert = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + The SSL / TLS CA certificate that verifies the identity of the + database server. + + Required when PostgreSQL is used and SSL is turned on. + + For MySQL, if left at null, the default + Java keystore is used, which should suffice if the server + certificate is issued by an official CA. ''; }; @@ -208,6 +256,12 @@ in let # We only want to create a database if we're actually going to connect to it. databaseActuallyCreateLocally = cfg.databaseCreateLocally && cfg.databaseHost == "localhost"; + createLocalPostgreSQL = databaseActuallyCreateLocally && cfg.databaseType == "postgresql"; + createLocalMySQL = databaseActuallyCreateLocally && cfg.databaseType == "mysql"; + + mySqlCaKeystore = pkgs.runCommandNoCC "mysql-ca-keystore" {} '' + ${pkgs.jre}/bin/keytool -importcert -trustcacerts -alias MySQLCACert -file ${cfg.databaseCaCert} -keystore $out -storepass notsosecretpassword -noprompt + ''; keycloakConfig' = builtins.foldl' lib.recursiveUpdate { "interface=public".inet-address = cfg.bindAddress; @@ -220,19 +274,52 @@ in }; }; }; - "subsystem=datasources"."jdbc-driver=postgresql" = { - driver-module-name = "org.postgresql"; - driver-name = "postgresql"; - driver-xa-datasource-class-name = "org.postgresql.xa.PGXADataSource"; - }; "subsystem=datasources"."data-source=KeycloakDS" = { - connection-url = "jdbc:postgresql://${cfg.databaseHost}/keycloak"; - driver-name = "postgresql"; max-pool-size = "20"; user-name = if databaseActuallyCreateLocally then "keycloak" else cfg.databaseUsername; password = "@db-password@"; }; } [ + (lib.optionalAttrs (cfg.databaseType == "postgresql") { + "subsystem=datasources" = { + "jdbc-driver=postgresql" = { + driver-module-name = "org.postgresql"; + driver-name = "postgresql"; + driver-xa-datasource-class-name = "org.postgresql.xa.PGXADataSource"; + }; + "data-source=KeycloakDS" = { + connection-url = "jdbc:postgresql://${cfg.databaseHost}:${builtins.toString cfg.databasePort}/keycloak"; + driver-name = "postgresql"; + "connection-properties=ssl".value = lib.boolToString cfg.databaseUseSSL; + } // (lib.optionalAttrs (cfg.databaseCaCert != null) { + "connection-properties=sslrootcert".value = cfg.databaseCaCert; + "connection-properties=sslmode".value = "verify-ca"; + }); + }; + }) + (lib.optionalAttrs (cfg.databaseType == "mysql") { + "subsystem=datasources" = { + "jdbc-driver=mysql" = { + driver-module-name = "com.mysql"; + driver-name = "mysql"; + driver-class-name = "com.mysql.jdbc.Driver"; + }; + "data-source=KeycloakDS" = { + connection-url = "jdbc:mysql://${cfg.databaseHost}:${builtins.toString cfg.databasePort}/keycloak"; + driver-name = "mysql"; + "connection-properties=useSSL".value = lib.boolToString cfg.databaseUseSSL; + "connection-properties=requireSSL".value = lib.boolToString cfg.databaseUseSSL; + "connection-properties=verifyServerCertificate".value = lib.boolToString cfg.databaseUseSSL; + "connection-properties=characterEncoding".value = "UTF-8"; + valid-connection-checker-class-name = "org.jboss.jca.adapters.jdbc.extensions.mysql.MySQLValidConnectionChecker"; + validate-on-match = true; + exception-sorter-class-name = "org.jboss.jca.adapters.jdbc.extensions.mysql.MySQLExceptionSorter"; + } // (lib.optionalAttrs (cfg.databaseCaCert != null) { + "connection-properties=trustCertificateKeyStoreUrl".value = "file:${mySqlCaKeystore}"; + "connection-properties=trustCertificateKeyStorePassword".value = "notsosecretpassword"; + }); + }; + }) (lib.optionalAttrs (cfg.certificatePrivateKeyBundle != null) { "socket-binding-group=standard-sockets"."socket-binding=https".port = cfg.httpsPort; "core-service=management"."security-realm=UndertowRealm"."server-identity=ssl" = { @@ -444,7 +531,7 @@ in jbossCliScript = pkgs.writeText "jboss-cli-script" (mkJbossScript keycloakConfig'); - keycloakConfig = pkgs.runCommand "keycloak-config" {} '' + keycloakConfig = pkgs.runCommandNoCC "keycloak-config" {} '' export JBOSS_BASE_DIR="$(pwd -P)"; export JBOSS_MODULEPATH="${cfg.package}/modules"; export JBOSS_LOG_DIR="$JBOSS_BASE_DIR/log"; @@ -475,9 +562,16 @@ in in lib.mkIf cfg.enable { + assertions = [ + { + assertion = (cfg.databaseUseSSL && cfg.databaseType == "postgresql") -> (cfg.databaseCaCert != null); + message = ''A CA certificate must be specified (in 'services.keycloak.databaseCaCert') when PostgreSQL is used with SSL''; + } + ]; + environment.systemPackages = [ cfg.package ]; - systemd.services.keycloakDatabaseInit = lib.mkIf databaseActuallyCreateLocally { + systemd.services.keycloakPostgreSQLInit = lib.mkIf createLocalPostgreSQL { after = [ "postgresql.service" ]; before = [ "keycloak.service" ]; bindsTo = [ "postgresql.service" ]; @@ -498,71 +592,100 @@ in ''; }; - systemd.services.keycloak = { - after = lib.optionals databaseActuallyCreateLocally [ - "keycloakDatabaseInit.service" "postgresql.service" - ]; - bindsTo = lib.optionals databaseActuallyCreateLocally [ - "keycloakDatabaseInit.service" "postgresql.service" - ]; - wantedBy = [ "multi-user.target" ]; - environment = { - JBOSS_LOG_DIR = "/var/log/keycloak"; - JBOSS_BASE_DIR = "/run/keycloak"; - JBOSS_MODULEPATH = "${cfg.package}/modules"; - }; + systemd.services.keycloakMySQLInit = lib.mkIf createLocalMySQL { + after = [ "mysql.service" ]; + before = [ "keycloak.service" ]; + bindsTo = [ "mysql.service" ]; serviceConfig = { - ExecStartPre = let - startPreFullPrivileges = '' - set -eu - - install -T -m 0400 -o keycloak -g keycloak '${cfg.databasePasswordFile}' /run/keycloak/secrets/db_password - '' + lib.optionalString (cfg.certificatePrivateKeyBundle != null) '' - install -T -m 0400 -o keycloak -g keycloak '${cfg.certificatePrivateKeyBundle}' /run/keycloak/secrets/ssl_cert_pk_bundle - ''; - startPre = '' - set -eu - - install -m 0600 ${cfg.package}/standalone/configuration/*.properties /run/keycloak/configuration - install -T -m 0600 ${keycloakConfig} /run/keycloak/configuration/standalone.xml - - db_password="$( allcerts.pem - ${pkgs.openssl}/bin/openssl pkcs12 -export -in /run/keycloak/secrets/ssl_cert_pk_bundle -chain \ - -name "${cfg.frontendUrl}" -out certificate_private_key_bundle.p12 \ - -CAfile allcerts.pem -passout pass:notsosecretpassword - popd - ''; - in [ - "+${pkgs.writeShellScript "keycloak-start-pre-full-privileges" startPreFullPrivileges}" - "${pkgs.writeShellScript "keycloak-start-pre" startPre}" - ]; - ExecStart = "${cfg.package}/bin/standalone.sh"; - User = "keycloak"; - Group = "keycloak"; - DynamicUser = true; - RuntimeDirectory = map (p: "keycloak/" + p) [ - "secrets" - "configuration" - "deployments" - "data" - "ssl" - "log" - "tmp" - ]; - RuntimeDirectoryMode = 0700; - LogsDirectory = "keycloak"; - AmbientCapabilities = "CAP_NET_BIND_SERVICE"; + Type = "oneshot"; + RemainAfterExit = true; + User = config.services.mysql.user; + Group = config.services.mysql.group; }; + script = '' + set -eu + + db_password="$(<'${cfg.databasePasswordFile}')" + ( echo "CREATE USER IF NOT EXISTS 'keycloak'@'localhost' IDENTIFIED BY '$db_password';" + echo "CREATE DATABASE keycloak CHARACTER SET utf8 COLLATE utf8_unicode_ci;" + echo "GRANT ALL PRIVILEGES ON keycloak.* TO 'keycloak'@'localhost';" + ) | ${config.services.mysql.package}/bin/mysql -N + ''; }; - services.postgresql.enable = lib.mkDefault databaseActuallyCreateLocally; + systemd.services.keycloak = + let + databaseServices = + if createLocalPostgreSQL then [ + "keycloakPostgreSQLInit.service" "postgresql.service" + ] + else if createLocalMySQL then [ + "keycloakMySQLInit.service" "mysql.service" + ] + else [ ]; + in { + after = databaseServices; + bindsTo = databaseServices; + wantedBy = [ "multi-user.target" ]; + environment = { + JBOSS_LOG_DIR = "/var/log/keycloak"; + JBOSS_BASE_DIR = "/run/keycloak"; + JBOSS_MODULEPATH = "${cfg.package}/modules"; + }; + serviceConfig = { + ExecStartPre = let + startPreFullPrivileges = '' + set -eu + + install -T -m 0400 -o keycloak -g keycloak '${cfg.databasePasswordFile}' /run/keycloak/secrets/db_password + '' + lib.optionalString (cfg.certificatePrivateKeyBundle != null) '' + install -T -m 0400 -o keycloak -g keycloak '${cfg.certificatePrivateKeyBundle}' /run/keycloak/secrets/ssl_cert_pk_bundle + ''; + startPre = '' + set -eu + + install -m 0600 ${cfg.package}/standalone/configuration/*.properties /run/keycloak/configuration + install -T -m 0600 ${keycloakConfig} /run/keycloak/configuration/standalone.xml + + db_password="$( allcerts.pem + ${pkgs.openssl}/bin/openssl pkcs12 -export -in /run/keycloak/secrets/ssl_cert_pk_bundle -chain \ + -name "${cfg.frontendUrl}" -out certificate_private_key_bundle.p12 \ + -CAfile allcerts.pem -passout pass:notsosecretpassword + popd + ''; + in [ + "+${pkgs.writeShellScript "keycloak-start-pre-full-privileges" startPreFullPrivileges}" + "${pkgs.writeShellScript "keycloak-start-pre" startPre}" + ]; + ExecStart = "${cfg.package}/bin/standalone.sh"; + User = "keycloak"; + Group = "keycloak"; + DynamicUser = true; + RuntimeDirectory = map (p: "keycloak/" + p) [ + "secrets" + "configuration" + "deployments" + "data" + "ssl" + "log" + "tmp" + ]; + RuntimeDirectoryMode = 0700; + LogsDirectory = "keycloak"; + AmbientCapabilities = "CAP_NET_BIND_SERVICE"; + }; + }; + + services.postgresql.enable = lib.mkDefault createLocalPostgreSQL; + services.mysql.enable = lib.mkDefault createLocalMySQL; + services.mysql.package = lib.mkIf createLocalMySQL pkgs.mysql; }; meta.doc = ./keycloak.xml; diff --git a/nixos/modules/services/web-apps/keycloak.xml b/nixos/modules/services/web-apps/keycloak.xml index 6b97d48e0bd88..ca5e223eee467 100644 --- a/nixos/modules/services/web-apps/keycloak.xml +++ b/nixos/modules/services/web-apps/keycloak.xml @@ -37,15 +37,30 @@
Database access - Keycloak depends on - PostgreSQL and will automatically - enable it and create a database and role unless configured not - to, either by changing - from its default of localhost or setting - + Keycloak can be used with either + PostgreSQL or + MySQL. Which one is used can be + configured in . The selected + database will automatically be enabled and a database and role + created unless is changed from + its default of localhost or is set to false. + + External database access can also be configured by setting + , , and as + appropriate. Note that you need to manually create a database + called keycloak and allow the configured + database user full access to it. + + must be set to the path to a file containing the password used diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 5a10f60fc9a38..d49357cb463c6 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -175,7 +175,7 @@ in kernel-latest = handleTest ./kernel-latest.nix {}; kernel-lts = handleTest ./kernel-lts.nix {}; kernel-testing = handleTest ./kernel-testing.nix {}; - keycloak = handleTest ./keycloak.nix {}; + keycloak = discoverTests (import ./keycloak.nix); keymap = handleTest ./keymap.nix {}; knot = handleTest ./knot.nix {}; krb5 = discoverTests (import ./krb5 {}); diff --git a/nixos/tests/keycloak.nix b/nixos/tests/keycloak.nix index e5e31b038e9cf..f448a0f7095f6 100644 --- a/nixos/tests/keycloak.nix +++ b/nixos/tests/keycloak.nix @@ -2,12 +2,12 @@ # OIDC client and a user, and simulates the user logging in to the # client using their Keycloak login. -import ./make-test-python.nix ( - { pkgs, ... }: - let - frontendUrl = "http://keycloak/auth"; - initialAdminPassword = "h4IhoJFnt2iQIR9"; - in +let + frontendUrl = "http://keycloak/auth"; + initialAdminPassword = "h4IhoJFnt2iQIR9"; + + keycloakTest = import ./make-test-python.nix ( + { pkgs, databaseType, ... }: { name = "keycloak"; meta = with pkgs.stdenv.lib.maintainers; { @@ -19,7 +19,7 @@ import ./make-test-python.nix ( virtualisation.memorySize = 1024; services.keycloak = { enable = true; - inherit frontendUrl initialAdminPassword; + inherit frontendUrl databaseType initialAdminPassword; databasePasswordFile = pkgs.writeText "dbPassword" "wzf6vOCbPp6cqTH"; }; environment.systemPackages = with pkgs; [ @@ -136,4 +136,9 @@ import ./make-test-python.nix ( ) ''; } -) + ); +in +{ + postgres = keycloakTest { databaseType = "postgresql"; }; + mysql = keycloakTest { databaseType = "mysql"; }; +} diff --git a/pkgs/servers/keycloak/default.nix b/pkgs/servers/keycloak/default.nix index 95935ce8f8aec..67d3d9bd45a25 100644 --- a/pkgs/servers/keycloak/default.nix +++ b/pkgs/servers/keycloak/default.nix @@ -1,11 +1,11 @@ -{ stdenv, fetchzip, makeWrapper, jre, writeText, nixosTests -, postgresql_jdbc ? null +{ stdenv, lib, fetchzip, makeWrapper, jre, writeText, nixosTests +, postgresql_jdbc ? null, mysql_jdbc ? null }: let mkModuleXml = name: jarFile: writeText "module.xml" '' - + @@ -33,17 +33,22 @@ stdenv.mkDerivation rec { rm -rf $out/bin/*.{ps1,bat} - module_path=$out/modules/system/layers/keycloak/org + module_path=$out/modules/system/layers/keycloak if ! [[ -d $module_path ]]; then echo "The module path $module_path not found!" exit 1 fi - ${if postgresql_jdbc != null then '' - mkdir -p $module_path/postgresql/main - ln -s ${postgresql_jdbc}/share/java/postgresql-jdbc.jar $module_path/postgresql/main - ln -s ${mkModuleXml "postgresql" "postgresql-jdbc.jar"} $module_path/postgresql/main/module.xml - '' else ""} + ${lib.optionalString (postgresql_jdbc != null) '' + mkdir -p $module_path/org/postgresql/main + ln -s ${postgresql_jdbc}/share/java/postgresql-jdbc.jar $module_path/org/postgresql/main/ + ln -s ${mkModuleXml "org.postgresql" "postgresql-jdbc.jar"} $module_path/org/postgresql/main/module.xml + ''} + ${lib.optionalString (mysql_jdbc != null) '' + mkdir -p $module_path/com/mysql/main + ln -s ${mysql_jdbc}/share/java/mysql-connector-java.jar $module_path/com/mysql/main/ + ln -s ${mkModuleXml "com.mysql" "mysql-connector-java.jar"} $module_path/com/mysql/main/module.xml + ''} wrapProgram $out/bin/standalone.sh --set JAVA_HOME ${jre} wrapProgram $out/bin/add-user-keycloak.sh --set JAVA_HOME ${jre} -- cgit 1.4.1 From 819b0f4bb864b21327672a5b9f2d94a75c312078 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 2 Nov 2020 21:18:57 +0100 Subject: nixos/initrd-network-ssh: fix test The test relied on moving `initrd` secrets from the store into the `initrd` which was fine here as it's only an integration test and not a production environment. However, this broke in 20.09 when support for this was dropped[1]. To make sure that the snakeoil key used as hostkey for `sshd` here actually gets copied into the VM, I added a small script for this that takes care of this process while building the initial ramdisk. [1] d930466b7728e5515991cb73c060fc3f32bdc001 --- nixos/tests/initrd-network-ssh/default.nix | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'nixos/tests') diff --git a/nixos/tests/initrd-network-ssh/default.nix b/nixos/tests/initrd-network-ssh/default.nix index 017de6882081d..0ad0563b0ce15 100644 --- a/nixos/tests/initrd-network-ssh/default.nix +++ b/nixos/tests/initrd-network-ssh/default.nix @@ -22,6 +22,10 @@ import ../make-test-python.nix ({ lib, ... }: hostKeys = [ ./ssh_host_ed25519_key ]; }; }; + boot.initrd.extraUtilsCommands = '' + mkdir -p $out/secrets/etc/ssh + cat "${./ssh_host_ed25519_key}" > $out/secrets/etc/ssh/sh_host_ed25519_key + ''; boot.initrd.preLVMCommands = '' while true; do if [ -f fnord ]; then -- cgit 1.4.1 From 54217cac698fdcf5c047e53054b7d3b039337f02 Mon Sep 17 00:00:00 2001 From: Janne Heß Date: Mon, 2 Nov 2020 23:35:34 +0100 Subject: nixos/promtail: Add a promtail module --- nixos/modules/module-list.nix | 1 + nixos/modules/services/logging/promtail.nix | 95 +++++++++++++++++++++++++++++ nixos/tests/loki.nix | 31 +++++++--- 3 files changed, 118 insertions(+), 9 deletions(-) create mode 100644 nixos/modules/services/logging/promtail.nix (limited to 'nixos/tests') diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index aa3b71a612417..45fb75d23b2c2 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -394,6 +394,7 @@ ./services/logging/logcheck.nix ./services/logging/logrotate.nix ./services/logging/logstash.nix + ./services/logging/promtail.nix ./services/logging/rsyslogd.nix ./services/logging/syslog-ng.nix ./services/logging/syslogd.nix diff --git a/nixos/modules/services/logging/promtail.nix b/nixos/modules/services/logging/promtail.nix new file mode 100644 index 0000000000000..834bb99bb1d65 --- /dev/null +++ b/nixos/modules/services/logging/promtail.nix @@ -0,0 +1,95 @@ +{ config, lib, pkgs, ... }: with lib; +let + cfg = config.services.promtail; + + prettyJSON = conf: pkgs.runCommandLocal "promtail-config.json" {} '' + echo '${builtins.toJSON conf}' | ${pkgs.buildPackages.jq}/bin/jq 'del(._module)' > $out + ''; + +in { + options.services.promtail = with types; { + enable = mkEnableOption "the Promtail ingresser"; + + configuration = mkOption { + type = with lib.types; let + valueType = nullOr (oneOf [ + bool + int + float + str + (lazyAttrsOf valueType) + (listOf valueType) + ]) // { + description = "JSON value"; + emptyValue.value = {}; + deprecationMessage = null; + }; + in valueType; + description = '' + Specify the configuration for Promtail in Nix. + ''; + }; + + extraFlags = mkOption { + type = listOf str; + default = []; + example = [ "--server.http-listen-port=3101" ]; + description = '' + Specify a list of additional command line flags, + which get escaped and are then passed to Loki. + ''; + }; + }; + + config = mkIf cfg.enable { + services.promtail.configuration.positions.filename = mkDefault "/var/cache/promtail/positions.yaml"; + + systemd.services.promtail = { + description = "Promtail log ingress"; + wantedBy = [ "multi-user.target" ]; + stopIfChanged = false; + + serviceConfig = { + Restart = "on-failure"; + + ExecStart = "${pkgs.grafana-loki}/bin/promtail -config.file=${prettyJSON cfg.configuration} ${escapeShellArgs cfg.extraFlags}"; + + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + RestrictSUIDSGID = true; + PrivateMounts = true; + CacheDirectory = "promtail"; + + User = "promtail"; + Group = "promtail"; + + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + + ProtectKernelModules = true; + SystemCallArchitectures = "native"; + ProtectKernelLogs = true; + ProtectClock = true; + + LockPersonality = true; + ProtectHostname = true; + RestrictRealtime = true; + MemoryDenyWriteExecute = true; + PrivateUsers = true; + } // (optionalAttrs (!pkgs.stdenv.isAarch64) { # FIXME: figure out why this breaks on aarch64 + SystemCallFilter = "@system-service"; + }); + }; + + users.groups.promtail = {}; + users.users.promtail = { + description = "Promtail service user"; + isSystemUser = true; + group = "promtail"; + }; + }; +} diff --git a/nixos/tests/loki.nix b/nixos/tests/loki.nix index dbf1e8a650f5d..eaee717cf87d8 100644 --- a/nixos/tests/loki.nix +++ b/nixos/tests/loki.nix @@ -12,15 +12,28 @@ import ./make-test-python.nix ({ lib, pkgs, ... }: enable = true; configFile = "${pkgs.grafana-loki.src}/cmd/loki/loki-local-config.yaml"; }; - systemd.services.promtail = { - description = "Promtail service for Loki test"; - wantedBy = [ "multi-user.target" ]; - - serviceConfig = { - ExecStart = '' - ${pkgs.grafana-loki}/bin/promtail --config.file ${pkgs.grafana-loki.src}/cmd/promtail/promtail-local-config.yaml - ''; - DynamicUser = true; + services.promtail = { + enable = true; + configuration = { + server = { + http_listen_port = 9080; + grpc_listen_port = 0; + }; + clients = [ { url = "http://localhost:3100/loki/api/v1/push"; } ]; + scrape_configs = [ + { + job_name = "system"; + static_configs = [ + { + targets = [ "localhost" ]; + labels = { + job = "varlogs"; + __path__ = "/var/log/*log"; + }; + } + ]; + } + ]; }; }; }; -- cgit 1.4.1