about summary refs log tree commit diff
path: root/nixos/tests/consul.nix
blob: 6233234ff083b78580df9d8a1e2d5d8c5c3c5d48 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
import ./make-test-python.nix ({pkgs, lib, ...}:

let
  # Settings for both servers and agents
  webUi = true;
  retry_interval = "1s";
  raft_multiplier = 1;

  defaultExtraConfig = {
    inherit retry_interval;
    performance = {
      inherit raft_multiplier;
    };
  };

  allConsensusServerHosts = [
    "192.168.1.1"
    "192.168.1.2"
    "192.168.1.3"
  ];

  allConsensusClientHosts = [
    "192.168.2.1"
    "192.168.2.2"
  ];

  firewallSettings = {
    # See https://www.consul.io/docs/install/ports.html
    allowedTCPPorts = [ 8301 8302 8600 8500 8300 ];
    allowedUDPPorts = [ 8301 8302 8600 ];
  };

  client = index: { pkgs, ... }:
    let
      ip = builtins.elemAt allConsensusClientHosts index;
    in
      {
        environment.systemPackages = [ pkgs.consul ];

        networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
          { address = ip; prefixLength = 16; }
        ];
        networking.firewall = firewallSettings;

        services.consul = {
          enable = true;
          inherit webUi;
          extraConfig = defaultExtraConfig // {
            server = false;
            retry_join = allConsensusServerHosts;
            bind_addr = ip;
          };
        };
      };

  server = index: { pkgs, ... }:
    let
      numConsensusServers = builtins.length allConsensusServerHosts;
      thisConsensusServerHost = builtins.elemAt allConsensusServerHosts index;
      ip = thisConsensusServerHost; # since we already use IPs to identify servers
    in
      {
        networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
          { address = ip; prefixLength = 16; }
        ];
        networking.firewall = firewallSettings;

        services.consul =
          assert builtins.elem thisConsensusServerHost allConsensusServerHosts;
          {
            enable = true;
            inherit webUi;
            extraConfig = defaultExtraConfig // {
              server = true;
              bootstrap_expect = numConsensusServers;
              # Tell Consul that we never intend to drop below this many servers.
              # Ensures to not permanently lose consensus after temporary loss.
              # See https://github.com/hashicorp/consul/issues/8118#issuecomment-645330040
              autopilot.min_quorum = numConsensusServers;
              retry_join =
                # If there's only 1 node in the network, we allow self-join;
                # otherwise, the node must not try to join itself, and join only the other servers.
                # See https://github.com/hashicorp/consul/issues/2868
                if numConsensusServers == 1
                  then allConsensusServerHosts
                  else builtins.filter (h: h != thisConsensusServerHost) allConsensusServerHosts;
              bind_addr = ip;
            };
          };
      };
in {
  name = "consul";

  nodes = {
    server1 = server 0;
    server2 = server 1;
    server3 = server 2;

    client1 = client 0;
    client2 = client 1;
  };

  testScript = ''
    servers = [server1, server2, server3]
    machines = [server1, server2, server3, client1, client2]

    for m in machines:
        m.wait_for_unit("consul.service")


    def wait_for_healthy_servers():
        # See https://github.com/hashicorp/consul/issues/8118#issuecomment-645330040
        # for why the `Voter` column of `list-peers` has that info.
        # TODO: The `grep true` relies on the fact that currently in
        #       the output like
        #           # consul operator raft list-peers
        #           Node     ID   Address           State     Voter  RaftProtocol
        #           server3  ...  192.168.1.3:8300  leader    true   3
        #           server2  ...  192.168.1.2:8300  follower  true   3
        #           server1  ...  192.168.1.1:8300  follower  false  3
        #       `Voter`is the only boolean column.
        #       Change this to the more reliable way to be defined by
        #       https://github.com/hashicorp/consul/issues/8118
        #       once that ticket is closed.
        for m in machines:
            m.wait_until_succeeds(
                "[ $(consul operator raft list-peers | grep true | wc -l) == 3 ]"
            )


    def wait_for_all_machines_alive():
        """
        Note that Serf-"alive" does not mean "Raft"-healthy;
        see `wait_for_healthy_servers()` for that instead.
        """
        for m in machines:
            m.wait_until_succeeds("[ $(consul members | grep -o alive | wc -l) == 5 ]")


    wait_for_healthy_servers()
    # Also wait for clients to be alive.
    wait_for_all_machines_alive()

    client1.succeed("consul kv put testkey 42")
    client2.succeed("[ $(consul kv get testkey) == 42 ]")


    def rolling_restart_test(proper_rolling_procedure=True):
        """
        Tests that the cluster can tolearate failures of any single server,
        following the recommended rolling upgrade procedure from
        https://www.consul.io/docs/upgrading#standard-upgrades.

        Optionally, `proper_rolling_procedure=False` can be given
        to wait only for each server to be back `Healthy`, not `Stable`
        in the Raft consensus, see Consul setting `ServerStabilizationTime` and
        https://github.com/hashicorp/consul/issues/8118#issuecomment-645330040.
        """

        for server in servers:
            server.block()
            server.systemctl("stop consul")

            # Make sure the stopped peer is recognized as being down
            client1.wait_until_succeeds(
              f"[ $(consul members | grep {server.name} | grep -o -E 'failed|left' | wc -l) == 1 ]"
            )

            # For each client, wait until they have connection again
            # using `kv get -recurse` before issuing commands.
            client1.wait_until_succeeds("consul kv get -recurse")
            client2.wait_until_succeeds("consul kv get -recurse")

            # Do some consul actions while one server is down.
            client1.succeed("consul kv put testkey 43")
            client2.succeed("[ $(consul kv get testkey) == 43 ]")
            client2.succeed("consul kv delete testkey")

            server.unblock()
            server.systemctl("start consul")

            if proper_rolling_procedure:
                # Wait for recovery.
                wait_for_healthy_servers()
            else:
                # NOT proper rolling upgrade procedure, see above.
                wait_for_all_machines_alive()

            # Wait for client connections.
            client1.wait_until_succeeds("consul kv get -recurse")
            client2.wait_until_succeeds("consul kv get -recurse")

            # Do some consul actions with server back up.
            client1.succeed("consul kv put testkey 44")
            client2.succeed("[ $(consul kv get testkey) == 44 ]")
            client2.succeed("consul kv delete testkey")


    def all_servers_crash_simultaneously_test():
        """
        Tests that the cluster will eventually come back after all
        servers crash simultaneously.
        """

        for server in servers:
            server.block()
            server.systemctl("stop --no-block consul")

        for server in servers:
            # --no-block is async, so ensure it has been stopped by now
            server.wait_until_fails("systemctl is-active --quiet consul")
            server.unblock()
            server.systemctl("start consul")

        # Wait for recovery.
        wait_for_healthy_servers()

        # Wait for client connections.
        client1.wait_until_succeeds("consul kv get -recurse")
        client2.wait_until_succeeds("consul kv get -recurse")

        # Do some consul actions with servers back up.
        client1.succeed("consul kv put testkey 44")
        client2.succeed("[ $(consul kv get testkey) == 44 ]")
        client2.succeed("consul kv delete testkey")


    # Run the tests.

    print("rolling_restart_test()")
    rolling_restart_test()

    print("all_servers_crash_simultaneously_test()")
    all_servers_crash_simultaneously_test()

    print("rolling_restart_test(proper_rolling_procedure=False)")
    rolling_restart_test(proper_rolling_procedure=False)
  '';
})