diff options
| -rw-r--r-- | doc/hooks/index.xml | 10 | ||||
| -rw-r--r-- | doc/hooks/postgresql-test-hook.section.md | 59 | ||||
| -rw-r--r-- | doc/manual.xml | 1 | ||||
| -rw-r--r-- | nixos/doc/manual/from_md/release-notes/rl-2205.section.xml | 7 | ||||
| -rw-r--r-- | nixos/doc/manual/release-notes/rl-2205.section.md | 2 | ||||
| -rw-r--r-- | pkgs/build-support/setup-hooks/postgresql-test-hook/default.nix | 9 | ||||
| -rw-r--r-- | pkgs/build-support/setup-hooks/postgresql-test-hook/postgresql-test-hook.sh | 79 | ||||
| -rw-r--r-- | pkgs/build-support/setup-hooks/postgresql-test-hook/test.nix | 27 | ||||
| -rw-r--r-- | pkgs/development/haskell-modules/configuration-common.nix | 54 | ||||
| -rw-r--r-- | pkgs/top-level/all-packages.nix | 2 |
10 files changed, 245 insertions, 5 deletions
diff --git a/doc/hooks/index.xml b/doc/hooks/index.xml new file mode 100644 index 0000000000000..6a046eae28857 --- /dev/null +++ b/doc/hooks/index.xml @@ -0,0 +1,10 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + xml:id="chap-hooks"> + <title>Hooks reference</title> + <para> + Nixpkgs has several hook packages that augment the stdenv phases. + </para> + <xi:include href="./postgresql-test-hook.section.xml" /> +</chapter> diff --git a/doc/hooks/postgresql-test-hook.section.md b/doc/hooks/postgresql-test-hook.section.md new file mode 100644 index 0000000000000..077fac14ebbff --- /dev/null +++ b/doc/hooks/postgresql-test-hook.section.md @@ -0,0 +1,59 @@ + +# `postgresqlTestHook` {#sec-postgresqlTestHook} + +This hook starts a PostgreSQL server during the `checkPhase`. Example: + +```nix +{ stdenv, postgresql, postgresqlTestHook }: +stdenv.mkDerivation { + + # ... + + checkInputs = [ + postgresql + postgresqlTestHook + ]; +} +``` + +If you use a custom `checkPhase`, remember to add the `runHook` calls: +```nix + checkPhase '' + runHook preCheck + + # ... your tests + + runHook postCheck + '' +``` + +## Variables {#sec-postgresqlTestHook-variables} + +The hook logic will read a number of variables and set them to a default value if unset or empty. + +Exported variables: + + - `PGDATA`: location of server files. + - `PGHOST`: location of UNIX domain socket directory; the default `host` in a connection string. + - `PGUSER`: user to create / log in with, default: `test_user`. + - `PGDATABASE`: database name, default: `test_db`. + +Bash-only variables: + + - `postgresqlTestUserOptions`: SQL options to use when creating the `$PGUSER` role, default: `LOGIN`. + - `postgresqlTestSetupSQL`: SQL commands to run as database administrator after startup, default: statements that create `$PGUSER` and `$PGDATABASE`. + - `postgresqlTestSetupCommands`: bash commands to run after database start, defaults to running `$postgresqlTestSetupSQL` as database administrator. + - `postgresqlEnableTCP`: set to `1` to enable TCP listening. Flaky; not recommended. + - `postgresqlStartCommands`: defaults to `pg_ctl start`. + +## TCP and the Nix sandbox {#sec-postgresqlTestHook-tcp} + +`postgresqlEnableTCP` relies on network sandboxing, which is not available on macOS and some custom Nix installations, resulting in flaky tests. +For this reason, it is disabled by default. + +The preferred solution is to make the test suite use a UNIX domain socket connection. This is the default behavior when no `host` connection parameter is provided. +Some test suites hardcode a value for `host` though, so a patch may be required. If you can upstream the patch, you can make `host` default to the `PGHOST` environment variable when set. Otherwise, you can patch it locally to omit the `host` connection string parameter altogether. + +::: {.note} +The error `libpq: failed (could not receive data from server: Connection refused` is generally an indication that the test suite is trying to connect through TCP. +::: diff --git a/doc/manual.xml b/doc/manual.xml index b43021d85ca56..e49ae67ec943a 100644 --- a/doc/manual.xml +++ b/doc/manual.xml @@ -27,6 +27,7 @@ <xi:include href="builders/trivial-builders.chapter.xml" /> <xi:include href="builders/special.xml" /> <xi:include href="builders/images.xml" /> + <xi:include href="hooks/index.xml" /> <xi:include href="languages-frameworks/index.xml" /> <xi:include href="builders/packages/index.xml" /> </part> diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml index 543853afd5b6f..cf5dd6c091639 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml @@ -75,6 +75,13 @@ </listitem> <listitem> <para> + The new + <link xlink:href="https://nixos.org/manual/nixpkgs/stable/#sec-postgresqlTestHook"><literal>postgresqlTestHook</literal></link> + runs a PostgreSQL server for the duration of package checks. + </para> + </listitem> + <listitem> + <para> <link xlink:href="https://kops.sigs.k8s.io"><literal>kops</literal></link> defaults to 1.22.4, which will enable <link xlink:href="https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html">Instance diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md index da36fbbb2e5ba..675a53996efd5 100644 --- a/nixos/doc/manual/release-notes/rl-2205.section.md +++ b/nixos/doc/manual/release-notes/rl-2205.section.md @@ -27,6 +27,8 @@ In addition to numerous new and upgraded packages, this release has the followin - Systemd has been upgraded to the version 250. +- The new [`postgresqlTestHook`](https://nixos.org/manual/nixpkgs/stable/#sec-postgresqlTestHook) runs a PostgreSQL server for the duration of package checks. + - [`kops`](https://kops.sigs.k8s.io) defaults to 1.22.4, which will enable [Instance Metadata Service Version 2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html) and require tokens on new clusters with Kubernetes 1.22. This will increase security by default, but may break some types of workloads. See the [release notes](https://kops.sigs.k8s.io/releases/1.22-notes/) for details. - Module authors can use `mkRenamedOptionModuleWith` to automate the deprecation cycle without annoying out-of-tree module authors and their users. diff --git a/pkgs/build-support/setup-hooks/postgresql-test-hook/default.nix b/pkgs/build-support/setup-hooks/postgresql-test-hook/default.nix new file mode 100644 index 0000000000000..d0031c93c10cb --- /dev/null +++ b/pkgs/build-support/setup-hooks/postgresql-test-hook/default.nix @@ -0,0 +1,9 @@ +{ callPackage, makeSetupHook }: + +(makeSetupHook { + name = "postgresql-test-hook"; +} ./postgresql-test-hook.sh).overrideAttrs (o: { + passthru.tests = { + simple = callPackage ./test.nix { }; + }; +}) diff --git a/pkgs/build-support/setup-hooks/postgresql-test-hook/postgresql-test-hook.sh b/pkgs/build-support/setup-hooks/postgresql-test-hook/postgresql-test-hook.sh new file mode 100644 index 0000000000000..041a3f5653325 --- /dev/null +++ b/pkgs/build-support/setup-hooks/postgresql-test-hook/postgresql-test-hook.sh @@ -0,0 +1,79 @@ +preCheckHooks+=('postgresqlStart') +postCheckHooks+=('postgresqlStop') + + +postgresqlStart() { + + # Add default environment variable values + # + # Client variables: + # - https://www.postgresql.org/docs/current/libpq-envars.html + # + # Server variables: + # - only PGDATA: https://www.postgresql.org/docs/current/creating-cluster.html + + if [[ "${PGDATA:-}" == "" ]]; then + PGDATA="$NIX_BUILD_TOP/postgresql" + fi + export PGDATA + + if [[ "${PGHOST:-}" == "" ]]; then + mkdir -p "$NIX_BUILD_TOP/run/postgresql" + PGHOST="$NIX_BUILD_TOP/run/postgresql" + fi + export PGHOST + + if [[ "${PGUSER:-}" == "" ]]; then + PGUSER="test_user" + fi + export PGUSER + + if [[ "${PGDATABASE:-}" == "" ]]; then + PGDATABASE="test_db" + fi + export PGDATABASE + + if [[ "${postgresqlTestUserOptions:-}" == "" ]]; then + postgresqlTestUserOptions="LOGIN" + fi + + if [[ "${postgresqlTestSetupSQL:-}" == "" ]]; then + postgresqlTestSetupSQL="$(cat <<EOF + CREATE ROLE "$PGUSER" $postgresqlTestUserOptions; + CREATE DATABASE "$PGDATABASE" OWNER '$PGUSER'; +EOF + )" + fi + + if [[ "${postgresqlTestSetupCommands:-}" == "" ]]; then + postgresqlTestSetupCommands='echo "$postgresqlTestSetupSQL" | PGUSER=postgres psql postgres' + fi + + if ! type initdb >/dev/null; then + echo >&2 'initdb not found. Did you add postgresql to the checkInputs?' + false + fi + header 'initializing postgresql' + initdb -U postgres + + # Move the socket + echo "unix_socket_directories = '$NIX_BUILD_TOP/run/postgresql'" >>"$PGDATA/postgresql.conf" + + # TCP ports can be a problem in some sandboxes, + # so we disable tcp listening by default + if ! [[ "${postgresqlEnableTCP:-}" = 1 ]]; then + echo "listen_addresses = ''" >>"$PGDATA/postgresql.conf" + fi + + header 'starting postgresql' + eval "${postgresqlStartCommands:-pg_ctl start}" + + header 'setting up postgresql' + eval "$postgresqlTestSetupCommands" + +} + +postgresqlStop() { + header 'stopping postgresql' + pg_ctl stop +} diff --git a/pkgs/build-support/setup-hooks/postgresql-test-hook/test.nix b/pkgs/build-support/setup-hooks/postgresql-test-hook/test.nix new file mode 100644 index 0000000000000..6d8ad6c8c7e33 --- /dev/null +++ b/pkgs/build-support/setup-hooks/postgresql-test-hook/test.nix @@ -0,0 +1,27 @@ +{ postgresql, postgresqlTestHook, stdenv }: + +stdenv.mkDerivation { + name = "postgresql-test-hook-test"; + buildInputs = [ postgresqlTestHook ]; + checkInputs = [ postgresql ]; + dontUnpack = true; + doCheck = true; + passAsFile = ["sql"]; + sql = '' + CREATE TABLE hello ( + message text + ); + INSERT INTO hello VALUES ('it '||'worked'); + SELECT * FROM hello; + ''; + checkPhase = '' + runHook preCheck + psql <$sqlPath | grep 'it worked' + TEST_RAN=1 + runHook postCheck + ''; + installPhase = '' + [[ $TEST_RAN == 1 ]] + touch $out + ''; +} diff --git a/pkgs/development/haskell-modules/configuration-common.nix b/pkgs/development/haskell-modules/configuration-common.nix index 7bc823b3e84c6..66bd34bdbc07f 100644 --- a/pkgs/development/haskell-modules/configuration-common.nix +++ b/pkgs/development/haskell-modules/configuration-common.nix @@ -1190,8 +1190,29 @@ self: super: { # Fix build with attr-2.4.48 (see #53716) xattr = appendPatch ./patches/xattr-fix-build.patch super.xattr; - # Some tests depend on a postgresql instance - esqueleto = dontCheck super.esqueleto; + esqueleto = + overrideCabal + (drv: { + postPatch = drv.postPatch or "" + '' + # patch out TCP usage: https://nixos.org/manual/nixpkgs/stable/#sec-postgresqlTestHook-tcp + sed -i test/PostgreSQL/Test.hs \ + -e s^host=localhost^^ + ''; + # Match the test suite defaults (or hardcoded values?) + preCheck = drv.preCheck or "" + '' + PGUSER=esqutest + PGDATABASE=esqutest + ''; + testFlags = drv.testFlags or [] ++ [ + # We don't have a MySQL test hook yet + "--skip=/Esqueleto/MySQL" + ]; + testToolDepends = drv.testToolDepends or [] ++ [ + pkgs.postgresql + pkgs.postgresqlTestHook + ]; + }) + super.esqueleto; # Requires API keys to run tests algolia = dontCheck super.algolia; @@ -1310,7 +1331,25 @@ self: super: { # Test suite requires database persistent-mysql = dontCheck super.persistent-mysql; - persistent-postgresql = dontCheck super.persistent-postgresql; + persistent-postgresql = + overrideCabal + (drv: { + postPatch = drv.postPath or "" + '' + # patch out TCP usage: https://nixos.org/manual/nixpkgs/stable/#sec-postgresqlTestHook-tcp + # NOTE: upstream host variable takes only two values... + sed -i test/PgInit.hs \ + -e s^'host=" <> host <> "'^^ + ''; + preCheck = drv.preCheck or "" + '' + PGDATABASE=test + PGUSER=test + ''; + testToolDepends = drv.testToolDepends or [] ++ [ + pkgs.postgresql + pkgs.postgresqlTestHook + ]; + }) + super.persistent-postgresql; # Fix EdisonAPI and EdisonCore for GHC 8.8: # https://github.com/robdockins/edison/pull/16 @@ -1515,8 +1554,13 @@ self: super: { }; pg-client = overrideCabal (drv: { librarySystemDepends = with pkgs; [ postgresql krb5.dev openssl.dev ]; - # wants a running DB to check against - doCheck = false; + testToolDepends = drv.testToolDepends or [] ++ [ + pkgs.postgresql pkgs.postgresqlTestHook + ]; + preCheck = drv.preCheck or "" + '' + # empty string means use default connection + export DATABASE_URL="" + ''; }) (super.pg-client.override { resource-pool = self.hasura-resource-pool; ekg-core = self.hasura-ekg-core; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index a65d42f532d47..5f4fc54285df9 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -22181,6 +22181,8 @@ with pkgs; postgresql_jdbc = callPackage ../development/java-modules/postgresql_jdbc { }; + postgresqlTestHook = callPackage ../build-support/setup-hooks/postgresql-test-hook { }; + prom2json = callPackage ../servers/monitoring/prometheus/prom2json.nix { }; prometheus = callPackage ../servers/monitoring/prometheus { }; prometheus-alertmanager = callPackage ../servers/monitoring/prometheus/alertmanager.nix { }; |