diff options
Diffstat (limited to 'nixos/tests/hadoop/hdfs.nix')
-rw-r--r-- | nixos/tests/hadoop/hdfs.nix | 61 |
1 files changed, 43 insertions, 18 deletions
diff --git a/nixos/tests/hadoop/hdfs.nix b/nixos/tests/hadoop/hdfs.nix index b63cbf4803271..9415500463dec 100644 --- a/nixos/tests/hadoop/hdfs.nix +++ b/nixos/tests/hadoop/hdfs.nix @@ -1,32 +1,46 @@ # Test a minimal HDFS cluster with no HA -import ../make-test-python.nix ({...}: { - nodes = { - namenode = {pkgs, ...}: { +import ../make-test-python.nix ({ package, lib, ... }: +with lib; +{ + name = "hadoop-hdfs"; + + nodes = let + coreSite = { + "fs.defaultFS" = "hdfs://namenode:8020"; + "hadoop.proxyuser.httpfs.groups" = "*"; + "hadoop.proxyuser.httpfs.hosts" = "*"; + }; + in { + namenode = { pkgs, ... }: { services.hadoop = { - package = pkgs.hadoop; + inherit package; hdfs = { namenode = { enable = true; + openFirewall = true; formatOnInit = true; }; - httpfs.enable = true; - }; - coreSite = { - "fs.defaultFS" = "hdfs://namenode:8020"; - "hadoop.proxyuser.httpfs.groups" = "*"; - "hadoop.proxyuser.httpfs.hosts" = "*"; + httpfs = { + # The NixOS hadoop module only support webHDFS on 3.3 and newer + enable = mkIf (versionAtLeast package.version "3.3") true; + openFirewall = true; + }; }; + inherit coreSite; }; }; - datanode = {pkgs, ...}: { + datanode = { pkgs, ... }: { services.hadoop = { - package = pkgs.hadoop; - hdfs.datanode.enable = true; - coreSite = { - "fs.defaultFS" = "hdfs://namenode:8020"; - "hadoop.proxyuser.httpfs.groups" = "*"; - "hadoop.proxyuser.httpfs.hosts" = "*"; + inherit package; + hdfs.datanode = { + enable = true; + openFirewall = true; + dataDirs = [{ + type = "DISK"; + path = "/tmp/dn1"; + }]; }; + inherit coreSite; }; }; }; @@ -37,21 +51,32 @@ import ../make-test-python.nix ({...}: { namenode.wait_for_unit("hdfs-namenode") namenode.wait_for_unit("network.target") namenode.wait_for_open_port(8020) + namenode.succeed("ss -tulpne | systemd-cat") + namenode.succeed("cat /etc/hadoop*/hdfs-site.xml | systemd-cat") namenode.wait_for_open_port(9870) datanode.wait_for_unit("hdfs-datanode") datanode.wait_for_unit("network.target") + '' + ( if versionAtLeast package.version "3" then '' datanode.wait_for_open_port(9864) datanode.wait_for_open_port(9866) datanode.wait_for_open_port(9867) - namenode.succeed("curl -f http://namenode:9870") datanode.succeed("curl -f http://datanode:9864") + '' else '' + datanode.wait_for_open_port(50075) + datanode.wait_for_open_port(50010) + datanode.wait_for_open_port(50020) + + datanode.succeed("curl -f http://datanode:50075") + '' ) + '' + namenode.succeed("curl -f http://namenode:9870") datanode.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait") datanode.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile") assert "testfilecontents" in datanode.succeed("sudo -u hdfs hdfs dfs -cat /testfile") + '' + optionalString ( versionAtLeast package.version "3.3" ) '' namenode.wait_for_unit("hdfs-httpfs") namenode.wait_for_open_port(14000) assert "testfilecontents" in datanode.succeed("curl -f \"http://namenode:14000/webhdfs/v1/testfile?user.name=hdfs&op=OPEN\" 2>&1") |