diff options
author | Sarah Brofeldt <sbrofeldt@gmail.com> | 2018-01-14 10:40:50 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-01-14 10:40:50 +0100 |
commit | ee4e6ebbfaad09a20f8b447e084444449583f505 (patch) | |
tree | e072452a8b823e6e25fa238bfd03bc9df923cc75 /nixos | |
parent | 08492b31d128501927862844b370b9facca17273 (diff) | |
parent | e8e88c43f61fd367e5c856bc29166cab5b29e048 (diff) |
Merge pull request #32822 from LumiGuide/elk6
ELK: 5.6.1 -> 5.6.5 & add ELK 6.1.0
Diffstat (limited to 'nixos')
-rw-r--r-- | nixos/modules/services/search/elasticsearch.nix | 8 | ||||
-rw-r--r-- | nixos/release.nix | 2 | ||||
-rw-r--r-- | nixos/tests/elk.nix | 172 |
3 files changed, 98 insertions, 84 deletions
diff --git a/nixos/modules/services/search/elasticsearch.nix b/nixos/modules/services/search/elasticsearch.nix index c51dd5d946550..adef500b7b5cd 100644 --- a/nixos/modules/services/search/elasticsearch.nix +++ b/nixos/modules/services/search/elasticsearch.nix @@ -6,6 +6,7 @@ let cfg = config.services.elasticsearch; es5 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "5" >= 0; + es6 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "6" >= 0; esConfig = '' network.host: ${cfg.listenAddress} @@ -92,8 +93,6 @@ in { node.name: "elasticsearch" node.master: true node.data: false - index.number_of_shards: 5 - index.number_of_replicas: 1 ''; }; @@ -165,7 +164,10 @@ in { path = [ pkgs.inetutils ]; environment = { ES_HOME = cfg.dataDir; - ES_JAVA_OPTS = toString ([ "-Des.path.conf=${configDir}" ] ++ cfg.extraJavaOptions); + ES_JAVA_OPTS = toString ( optional (!es6) [ "-Des.path.conf=${configDir}" ] + ++ cfg.extraJavaOptions); + } // optionalAttrs es6 { + ES_PATH_CONF = configDir; }; serviceConfig = { ExecStart = "${cfg.package}/bin/elasticsearch ${toString cfg.extraCmdLineOptions}"; diff --git a/nixos/release.nix b/nixos/release.nix index cf3fe6abd48c5..33916243e97bb 100644 --- a/nixos/release.nix +++ b/nixos/release.nix @@ -260,7 +260,7 @@ in rec { tests.etcd = hydraJob (import tests/etcd.nix { system = "x86_64-linux"; }); tests.ec2-nixops = hydraJob (import tests/ec2.nix { system = "x86_64-linux"; }).boot-ec2-nixops; tests.ec2-config = hydraJob (import tests/ec2.nix { system = "x86_64-linux"; }).boot-ec2-config; - tests.elk = hydraJob (import tests/elk.nix { system = "x86_64-linux"; }); + tests.elk = callSubTests tests/elk.nix { system = "x86_64-linux"; }; tests.env = callTest tests/env.nix {}; tests.ferm = callTest tests/ferm.nix {}; tests.firefox = callTest tests/firefox.nix {}; diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix index 65ff1cac070bd..ed656b3628b9e 100644 --- a/nixos/tests/elk.nix +++ b/nixos/tests/elk.nix @@ -1,95 +1,107 @@ -# Test the ELK stack: Elasticsearch, Logstash and Kibana. - -import ./make-test.nix ({ pkgs, ...} : +{ system ? builtins.currentSystem }: +with import ../lib/testing.nix { inherit system; }; +with pkgs.lib; let esUrl = "http://localhost:9200"; -in { - name = "ELK"; - meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ eelco chaoflow offline basvandijk ]; - }; - nodes = { - one = - { config, pkgs, ... }: { - # Not giving the machine at least 2060MB results in elasticsearch failing with the following error: - # - # OpenJDK 64-Bit Server VM warning: - # INFO: os::commit_memory(0x0000000085330000, 2060255232, 0) - # failed; error='Cannot allocate memory' (errno=12) - # - # There is insufficient memory for the Java Runtime Environment to continue. - # Native memory allocation (mmap) failed to map 2060255232 bytes for committing reserved memory. - # - # When setting this to 2500 I got "Kernel panic - not syncing: Out of - # memory: compulsory panic_on_oom is enabled" so lets give it even a - # bit more room: - virtualisation.memorySize = 3000; + mkElkTest = name : elk : makeTest { + inherit name; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ eelco chaoflow offline basvandijk ]; + }; + nodes = { + one = + { config, pkgs, ... }: { + # Not giving the machine at least 2060MB results in elasticsearch failing with the following error: + # + # OpenJDK 64-Bit Server VM warning: + # INFO: os::commit_memory(0x0000000085330000, 2060255232, 0) + # failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (mmap) failed to map 2060255232 bytes for committing reserved memory. + # + # When setting this to 2500 I got "Kernel panic - not syncing: Out of + # memory: compulsory panic_on_oom is enabled" so lets give it even a + # bit more room: + virtualisation.memorySize = 3000; - # For querying JSON objects returned from elasticsearch and kibana. - environment.systemPackages = [ pkgs.jq ]; + # For querying JSON objects returned from elasticsearch and kibana. + environment.systemPackages = [ pkgs.jq ]; - services = { - logstash = { - enable = true; - package = pkgs.logstash5; - inputConfig = '' - exec { command => "echo -n flowers" interval => 1 type => "test" } - exec { command => "echo -n dragons" interval => 1 type => "test" } - ''; - filterConfig = '' - if [message] =~ /dragons/ { - drop {} - } - ''; - outputConfig = '' - file { - path => "/tmp/logstash.out" - codec => line { format => "%{message}" } - } - elasticsearch { - hosts => [ "${esUrl}" ] - } - ''; - }; + services = { + logstash = { + enable = true; + package = elk.logstash; + inputConfig = '' + exec { command => "echo -n flowers" interval => 1 type => "test" } + exec { command => "echo -n dragons" interval => 1 type => "test" } + ''; + filterConfig = '' + if [message] =~ /dragons/ { + drop {} + } + ''; + outputConfig = '' + file { + path => "/tmp/logstash.out" + codec => line { format => "%{message}" } + } + elasticsearch { + hosts => [ "${esUrl}" ] + } + ''; + }; - elasticsearch = { - enable = true; - package = pkgs.elasticsearch5; - }; + elasticsearch = { + enable = true; + package = elk.elasticsearch; + }; - kibana = { - enable = true; - package = pkgs.kibana5; - elasticsearch.url = esUrl; + kibana = { + enable = true; + package = elk.kibana; + elasticsearch.url = esUrl; + }; }; }; - }; - }; + }; - testScript = '' - startAll; + testScript = '' + startAll; - $one->waitForUnit("elasticsearch.service"); + $one->waitForUnit("elasticsearch.service"); - # Continue as long as the status is not "red". The status is probably - # "yellow" instead of "green" because we are using a single elasticsearch - # node which elasticsearch considers risky. - # - # TODO: extend this test with multiple elasticsearch nodes and see if the status turns "green". - $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"); + # Continue as long as the status is not "red". The status is probably + # "yellow" instead of "green" because we are using a single elasticsearch + # node which elasticsearch considers risky. + # + # TODO: extend this test with multiple elasticsearch nodes and see if the status turns "green". + $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"); - # Perform some simple logstash tests. - $one->waitForUnit("logstash.service"); - $one->waitUntilSucceeds("cat /tmp/logstash.out | grep flowers"); - $one->waitUntilSucceeds("cat /tmp/logstash.out | grep -v dragons"); + # Perform some simple logstash tests. + $one->waitForUnit("logstash.service"); + $one->waitUntilSucceeds("cat /tmp/logstash.out | grep flowers"); + $one->waitUntilSucceeds("cat /tmp/logstash.out | grep -v dragons"); - # See if kibana is healthy. - $one->waitForUnit("kibana.service"); - $one->waitUntilSucceeds("curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"); + # See if kibana is healthy. + $one->waitForUnit("kibana.service"); + $one->waitUntilSucceeds("curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"); - # See if logstash messages arive in elasticsearch. - $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0"); - $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0"); - ''; -}) + # See if logstash messages arive in elasticsearch. + $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0"); + $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0"); + ''; + }; +in mapAttrs mkElkTest { + "ELK-5" = { + elasticsearch = pkgs.elasticsearch5; + logstash = pkgs.logstash5; + kibana = pkgs.kibana5; + }; + "ELK-6" = { + elasticsearch = pkgs.elasticsearch6; + logstash = pkgs.logstash6; + kibana = pkgs.kibana6; + }; +} |