diff options
author | Mogeko <zhengjunyi@live.com> | 2024-05-11 06:50:18 +0200 |
---|---|---|
committer | Mogeko <zhengjunyi@live.com> | 2024-05-28 12:39:30 +0800 |
commit | a642efcdab1402e45db5aa2bbff7b24322679219 (patch) | |
tree | a590cf07ca710ea1cf1614096ebf9f8bb647b0d4 /nixos/modules/services/cluster | |
parent | 398992986d223d333d5f40422b978512427bca0a (diff) |
nixos/rke2: add rke2 service
Set assertions to avoid obvious errors. Eliminate the conflict between default CNI (`cana`) and `NetworkManager`. Determine whether optional can be used for agent. Add the option `cisHardening` to enable CIS Hardening. Set kernel parameters by `boot.kernel.sysctl`. Using `lib.escapeShellArgs` to make `ExecStart` more resilient to escaping issues. Using a list of `str` to extra flags.
Diffstat (limited to 'nixos/modules/services/cluster')
-rw-r--r-- | nixos/modules/services/cluster/rke2/default.nix | 311 |
1 files changed, 311 insertions, 0 deletions
diff --git a/nixos/modules/services/cluster/rke2/default.nix b/nixos/modules/services/cluster/rke2/default.nix new file mode 100644 index 0000000000000..9ddbd299fdf8d --- /dev/null +++ b/nixos/modules/services/cluster/rke2/default.nix @@ -0,0 +1,311 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.rke2; +in +{ + imports = [ ]; + + options.services.rke2 = { + enable = mkEnableOption "rke2"; + + package = mkPackageOption pkgs "rke2" { }; + + role = mkOption { + type = types.enum [ "server" "agent" ]; + description = '' + Whether rke2 should run as a server or agent. + + If it's a server: + + - By default it also runs workloads as an agent. + - any optionals is allowed. + + If it's an agent: + + - `serverAddr` is required. + - `token` or `tokenFile` is required. + - `agentToken` or `agentTokenFile` or `disable` or `cni` are not allowed. + ''; + default = "server"; + }; + + configPath = mkOption { + type = types.path; + description = "Load configuration from FILE."; + default = "/etc/rancher/rke2/config.yaml"; + }; + + debug = mkOption { + type = types.bool; + description = "Turn on debug logs."; + default = false; + }; + + dataDir = mkOption { + type = types.path; + description = "The folder to hold state in."; + default = "/var/lib/rancher/rke2"; + }; + + token = mkOption { + type = types.str; + description = '' + Shared secret used to join a server or agent to a cluster. + + > WARNING: This option will expose store your token unencrypted world-readable in the nix store. + If this is undesired use the `tokenFile` option instead. + ''; + default = ""; + }; + + tokenFile = mkOption { + type = types.nullOr types.path; + description = "File path containing rke2 token to use when connecting to the server."; + default = null; + }; + + disable = mkOption { + type = types.listOf types.str; + description = "Do not deploy packaged components and delete any deployed components."; + default = [ ]; + }; + + nodeName = mkOption { + type = types.nullOr types.str; + description = "Node name."; + default = null; + }; + + nodeLabel = mkOption { + type = types.listOf types.str; + description = "Registering and starting kubelet with set of labels."; + default = [ ]; + }; + + nodeTaint = mkOption { + type = types.listOf types.str; + description = "Registering kubelet with set of taints."; + default = [ ]; + }; + + nodeIP = mkOption { + type = types.nullOr types.str; + description = "IPv4/IPv6 addresses to advertise for node."; + default = null; + }; + + agentToken = mkOption { + type = types.str; + description = '' + Shared secret used to join agents to the cluster, but not servers. + + > **WARNING**: This option will expose store your token unencrypted world-readable in the nix store. + If this is undesired use the `agentTokenFile` option instead. + ''; + default = ""; + }; + + agentTokenFile = mkOption { + type = types.nullOr types.path; + description = "File path containing rke2 agent token to use when connecting to the server."; + default = null; + }; + + serverAddr = mkOption { + type = types.str; + description = "The rke2 server to connect to, used to join a cluster."; + example = "https://10.0.0.10:6443"; + default = ""; + }; + + selinux = mkOption { + type = types.bool; + description = "Enable SELinux in containerd."; + default = false; + }; + + cni = mkOption { + type = types.enum [ "none" "canal" "cilium" "calico" "flannel" ]; + description = '' + CNI Plugins to deploy, one of `none`, `calico`, `canal`, `cilium` or `flannel`. + + All CNI plugins get installed via a helm chart after the main components are up and running + and can be [customized by modifying the helm chart options](https://docs.rke2.io/helm). + + [Learn more about RKE2 and CNI plugins](https://docs.rke2.io/networking/basic_network_options) + + > **WARNING**: Flannel support in RKE2 is currently experimental. + ''; + default = "canal"; + }; + + cisHardening = mkOption { + type = types.bool; + description = '' + Enable CIS Hardening for RKE2. + + It will set the configurations and controls required to address Kubernetes benchmark controls + from the Center for Internet Security (CIS). + + Learn more about [CIS Hardening for RKE2](https://docs.rke2.io/security/hardening_guide). + + > **NOTICE**: + > + > You may need restart the `systemd-sysctl` muaually by: + > + > ```shell + > sudo systemctl restart systemd-sysctl + > ``` + ''; + default = false; + }; + + extraFlags = mkOption { + type = types.listOf types.str; + description = '' + Extra flags to pass to the rke2 service/agent. + + Here you can find all the available flags: + + - [Server Configuration Reference](https://docs.rke2.io/reference/server_config) + - [Agent Configuration Reference](https://docs.rke2.io/reference/linux_agent_config) + ''; + example = [ "--disable-kube-proxy" "--cluster-cidr=10.24.0.0/16" ]; + default = [ ]; + }; + + environmentVars = mkOption { + type = types.attrsOf types.str; + description = '' + Environment variables for configuring the rke2 service/agent. + + Here you can find all the available environment variables: + + - [Server Configuration Reference](https://docs.rke2.io/reference/server_config) + - [Agent Configuration Reference](https://docs.rke2.io/reference/linux_agent_config) + + Besides the options above, you can also active environment variables by edit/create those files: + + - `/etc/default/rke2` + - `/etc/sysconfig/rke2` + - `/usr/local/lib/systemd/system/rke2.env` + ''; + # See: https://github.com/rancher/rke2/blob/master/bundle/lib/systemd/system/rke2-server.env#L1 + default = { + HOME = "/root"; + }; + }; + }; + + config = mkIf cfg.enable { + assertions = [ + { + assertion = cfg.role == "agent" -> (builtins.pathExists cfg.configPath || cfg.serverAddr != ""); + message = "serverAddr or configPath (with 'server' key) should be set if role is 'agent'"; + } + { + assertion = cfg.role == "agent" -> (builtins.pathExists cfg.configPath || cfg.tokenFile != null || cfg.token != ""); + message = "token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"; + } + { + assertion = cfg.role == "agent" -> ! (cfg.agentTokenFile != null || cfg.agentToken != ""); + message = "agentToken or agentTokenFile should be set if role is 'agent'"; + } + { + assertion = cfg.role == "agent" -> ! (cfg.disable != [ ]); + message = "disable should not be set if role is 'agent'"; + } + { + assertion = cfg.role == "agent" -> ! (cfg.cni != "canal"); + message = "cni should not be set if role is 'agent'"; + } + ]; + + environment.systemPackages = [ config.services.rke2.package ]; + # To configure NetworkManager to ignore calico/flannel related network interfaces. + # See: https://docs.rke2.io/known_issues#networkmanager + environment.etc."NetworkManager/conf.d/rke2-canal.conf" = { + enable = config.networking.networkmanager.enable; + text = '' + [keyfile] + unmanaged-devices=interface-name:cali*;interface-name:flannel* + ''; + }; + # See: https://docs.rke2.io/security/hardening_guide#set-kernel-parameters + boot.kernel.sysctl = mkIf cfg.cisHardening { + "vm.panic_on_oom" = 0; + "vm.overcommit_memory" = 1; + "kernel.panic" = 10; + "kernel.panic_on_oops" = 1; + }; + + systemd.services.rke2 = { + description = "Rancher Kubernetes Engine v2"; + documentation = [ "https://github.com/rancher/rke2#readme" ]; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = if cfg.role == "agent" then "exec" else "notify"; + EnvironmentFile = [ + "-/etc/default/%N" + "-/etc/sysconfig/%N" + "-/usr/local/lib/systemd/system/%N.env" + ]; + Environment = mapAttrsToList (k: v: "${k}=${v}") cfg.environmentVars; + KillMode = "process"; + Delegate = "yes"; + LimitNOFILE = 1048576; + LimitNPROC = "infinity"; + LimitCORE = "infinity"; + TasksMax = "infinity"; + TimeoutStartSec = 0; + Restart = "always"; + RestartSec = "5s"; + ExecStartPre = [ + # There is a conflict between RKE2 and `nm-cloud-setup.service`. This service add a routing table that + # interfere with the CNI plugin's configuration. This script checks if the service is enabled and if so, + # failed the RKE2 start. + # See: https://github.com/rancher/rke2/issues/1053 + (pkgs.writeScript "check-nm-cloud-setup.sh" '' + #! ${pkgs.runtimeShell} + set -x + ! /run/current-system/systemd/bin/systemctl is-enabled --quiet nm-cloud-setup.service + '') + "-${pkgs.kmod}/bin/modprobe br_netfilter" + "-${pkgs.kmod}/bin/modprobe overlay" + ]; + ExecStart = "${cfg.package}/bin/rke2 '${cfg.role}' ${escapeShellArgs ( + (optional (cfg.configPath != "/etc/rancher/rke2/config.yaml") "--config=${cfg.configPath}") + ++ (optional cfg.debug "--debug") + ++ (optional (cfg.dataDir != "/var/lib/rancher/rke2") "--data-dir=${cfg.dataDir}") + ++ (optional (cfg.token != "") "--token=${cfg.token}") + ++ (optional (cfg.tokenFile != null) "--token-file=${cfg.tokenFile}") + ++ (optionals (cfg.role == "server" && cfg.disable != [ ]) (map (d: "--disable=${d}") cfg.disable)) + ++ (optional (cfg.nodeName != null) "--node-name=${cfg.nodeName}") + ++ (optionals (cfg.nodeLabel != [ ]) (map (l: "--node-label=${l}") cfg.nodeLabel)) + ++ (optionals (cfg.nodeTaint != [ ]) (map (t: "--node-taint=${t}") cfg.nodeTaint)) + ++ (optional (cfg.nodeIP != null) "--node-ip=${cfg.nodeIP}") + ++ (optional (cfg.role == "server" && cfg.agentToken != "") "--agent-token=${cfg.agentToken}") + ++ (optional (cfg.role == "server" && cfg.agentTokenFile != null) "--agent-token-file=${cfg.agentTokenFile}") + ++ (optional (cfg.serverAddr != "") "--server=${cfg.serverAddr}") + ++ (optional cfg.selinux "--selinux") + ++ (optional (cfg.role == "server" && cfg.cni != "canal") "--cni=${cfg.cni}") + ++ (optional cfg.cisHardening "--profile=${if cfg.package.version >= "1.25" then "cis-1.23" else "cis-1.6"}") + ++ cfg.extraFlags + )}"; + ExecStopPost = let + killProcess = pkgs.writeScript "kill-process.sh" '' + #! ${pkgs.runtimeShell} + /run/current-system/systemd/bin/systemd-cgls /system.slice/$1 | \ + ${pkgs.gnugrep}/bin/grep -Eo '[0-9]+ (containerd|kubelet)' | \ + ${pkgs.gawk}/bin/awk '{print $1}' | \ + ${pkgs.findutils}/bin/xargs -r ${pkgs.util-linux}/bin/kill + ''; + in "-${killProcess} %n"; + }; + }; + }; +} |