Merge pull request #54800 from nlewo/nova

Remove cloud-init from the Openstack image configuration

10 files changed, 261 insertions, 162 deletions

diff --git a/nixos/maintainers/scripts/openstack/nova-image.nix b/nixos/maintainers/scripts/openstack/nova-image.nix
deleted file mode 100644
index b6f3a5b152003..0000000000000
--- a/nixos/maintainers/scripts/openstack/nova-image.nix
+++ /dev/null
@@ -1,26 +0,0 @@
-# nix-build '<nixpkgs/nixos>' -A config.system.build.novaImage --arg configuration "{ imports = [ ./nixos/maintainers/scripts/openstack/nova-image.nix ]; }"
-
-{ config, lib, pkgs, ... }:
-
-with lib;
-
-{
-  imports =
-    [ ../../../modules/installer/cd-dvd/channel.nix
-      ../../../modules/virtualisation/nova-config.nix
-    ];
-
-  system.build.novaImage = import ../../../lib/make-disk-image.nix {
-    inherit lib config;
-    pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
-    diskSize = 8192;
-    format = "qcow2";
-    configFile = pkgs.writeText "configuration.nix"
-      ''
-        {
-          imports = [ <nixpkgs/nixos/modules/virtualisation/nova-config.nix> ];
-        }
-      '';
-  };
-
-}
diff --git a/nixos/maintainers/scripts/openstack/openstack-image.nix b/nixos/maintainers/scripts/openstack/openstack-image.nix
new file mode 100644
index 0000000000000..4c464f43f61d8
--- /dev/null
+++ b/nixos/maintainers/scripts/openstack/openstack-image.nix
@@ -0,0 +1,26 @@
+# nix-build '<nixpkgs/nixos>' -A config.system.build.openstackImage --arg configuration "{ imports = [ ./nixos/maintainers/scripts/openstack/openstack-image.nix ]; }"
+
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+{
+  imports =
+    [ ../../../modules/installer/cd-dvd/channel.nix
+      ../../../modules/virtualisation/openstack-config.nix
+    ];
+
+  system.build.openstackImage = import ../../../lib/make-disk-image.nix {
+    inherit lib config;
+    pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
+    diskSize = 8192;
+    format = "qcow2";
+    configFile = pkgs.writeText "configuration.nix"
+      ''
+        {
+          imports = [ <nixpkgs/nixos/modules/virtualisation/openstack-config.nix> ];
+        }
+      '';
+  };
+
+}
diff --git a/nixos/modules/virtualisation/amazon-image.nix b/nixos/modules/virtualisation/amazon-image.nix
index 9015200beeadb..6f4f99caa6f13 100644
--- a/nixos/modules/virtualisation/amazon-image.nix
+++ b/nixos/modules/virtualisation/amazon-image.nix
@@ -8,7 +8,13 @@
 
 with lib;
 
-let cfg = config.ec2; in
+let
+  cfg = config.ec2;
+  metadataFetcher = import ./ec2-metadata-fetcher.nix {
+    targetRoot = "$targetRoot/";
+    wgetExtraOptions = "-q";
+  };
+in
 
 {
   imports = [ ../profiles/headless.nix ./ec2-data.nix ./amazon-init.nix ];
@@ -61,26 +67,7 @@ let cfg = config.ec2; in
     # Nix operations.
     boot.initrd.postMountCommands =
       ''
-        metaDir=$targetRoot/etc/ec2-metadata
-        mkdir -m 0755 -p "$metaDir"
-
-        echo "getting EC2 instance metadata..."
-
-        if ! [ -e "$metaDir/ami-manifest-path" ]; then
-          wget -q -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
-        fi
-
-        if ! [ -e "$metaDir/user-data" ]; then
-          wget -q -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data"
-        fi
-
-        if ! [ -e "$metaDir/hostname" ]; then
-          wget -q -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname
-        fi
-
-        if ! [ -e "$metaDir/public-keys-0-openssh-key" ]; then
-          wget -q -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
-        fi
+        ${metadataFetcher}
 
         diskNr=0
         diskForUnionfs=
diff --git a/nixos/modules/virtualisation/ec2-metadata-fetcher.nix b/nixos/modules/virtualisation/ec2-metadata-fetcher.nix
new file mode 100644
index 0000000000000..b531787c31a29
--- /dev/null
+++ b/nixos/modules/virtualisation/ec2-metadata-fetcher.nix
@@ -0,0 +1,23 @@
+{ targetRoot, wgetExtraOptions }:
+''
+  metaDir=${targetRoot}etc/ec2-metadata
+  mkdir -m 0755 -p "$metaDir"
+
+  echo "getting EC2 instance metadata..."
+
+  if ! [ -e "$metaDir/ami-manifest-path" ]; then
+    wget ${wgetExtraOptions} -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
+  fi
+
+  if ! [ -e "$metaDir/user-data" ]; then
+    wget ${wgetExtraOptions} -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data"
+  fi
+
+  if ! [ -e "$metaDir/hostname" ]; then
+    wget ${wgetExtraOptions} -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname
+  fi
+
+  if ! [ -e "$metaDir/public-keys-0-openssh-key" ]; then
+    wget ${wgetExtraOptions} -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
+  fi
+''
diff --git a/nixos/modules/virtualisation/nova-config.nix b/nixos/modules/virtualisation/nova-config.nix
deleted file mode 100644
index cecf2a3f144c1..0000000000000
--- a/nixos/modules/virtualisation/nova-config.nix
+++ /dev/null
@@ -1,60 +0,0 @@
-{ lib, ... }:
-
-with lib;
-
-{
-  imports = [
-    ../profiles/qemu-guest.nix
-    ../profiles/headless.nix
-  ];
-
-  config = {
-    fileSystems."/" = {
-      device = "/dev/disk/by-label/nixos";
-      autoResize = true;
-    };
-
-    boot.growPartition = true;
-    boot.kernelParams = [ "console=ttyS0" ];
-    boot.loader.grub.device = "/dev/vda";
-    boot.loader.timeout = 0;
-
-    # Allow root logins
-    services.openssh = {
-      enable = true;
-      permitRootLogin = "prohibit-password";
-      passwordAuthentication = mkDefault false;
-    };
-
-    services.cloud-init.enable = true;
-
-    # Put /tmp and /var on /ephemeral0, which has a lot more space.
-    # Unfortunately we can't do this with the `fileSystems' option
-    # because it has no support for creating the source of a bind
-    # mount.  Also, "move" /nix to /ephemeral0 by layering a unionfs-fuse
-    # mount on top of it so we have a lot more space for Nix operations.
-
-    /*
-    boot.initrd.postMountCommands =
-      ''
-        mkdir -m 1777 -p $targetRoot/ephemeral0/tmp
-        mkdir -m 1777 -p $targetRoot/tmp
-        mount --bind $targetRoot/ephemeral0/tmp $targetRoot/tmp
-
-        mkdir -m 755 -p $targetRoot/ephemeral0/var
-        mkdir -m 755 -p $targetRoot/var
-        mount --bind $targetRoot/ephemeral0/var $targetRoot/var
-
-        mkdir -p /unionfs-chroot/ro-nix
-        mount --rbind $targetRoot/nix /unionfs-chroot/ro-nix
-
-        mkdir -p /unionfs-chroot/rw-nix
-        mkdir -m 755 -p $targetRoot/ephemeral0/nix
-        mount --rbind $targetRoot/ephemeral0/nix /unionfs-chroot/rw-nix
-        unionfs -o allow_other,cow,nonempty,chroot=/unionfs-chroot,max_files=32768 /rw-nix=RW:/ro-nix=RO $targetRoot/nix
-      '';
-
-      boot.initrd.supportedFilesystems = [ "unionfs-fuse" ];
-    */
-  };
-}
diff --git a/nixos/modules/virtualisation/openstack-config.nix b/nixos/modules/virtualisation/openstack-config.nix
new file mode 100644
index 0000000000000..d5e862da0ea48
--- /dev/null
+++ b/nixos/modules/virtualisation/openstack-config.nix
@@ -0,0 +1,57 @@
+{ pkgs, lib, ... }:
+
+with lib;
+
+let
+  metadataFetcher = import ./ec2-metadata-fetcher.nix {
+    targetRoot = "/";
+    wgetExtraOptions = "--retry-connrefused";
+  };
+in
+{
+  imports = [
+    ../profiles/qemu-guest.nix
+    ../profiles/headless.nix
+    # The Openstack Metadata service exposes data on an EC2 API also.
+    ./ec2-data.nix
+    ./amazon-init.nix
+  ];
+
+  config = {
+    fileSystems."/" = {
+      device = "/dev/disk/by-label/nixos";
+      autoResize = true;
+    };
+
+    boot.growPartition = true;
+    boot.kernelParams = [ "console=ttyS0" ];
+    boot.loader.grub.device = "/dev/vda";
+    boot.loader.timeout = 0;
+
+    # Allow root logins
+    services.openssh = {
+      enable = true;
+      permitRootLogin = "prohibit-password";
+      passwordAuthentication = mkDefault false;
+    };
+
+    # Force getting the hostname from Openstack metadata.
+    networking.hostName = mkDefault "";
+
+    systemd.services.openstack-init = {
+      path = [ pkgs.wget ];
+      description = "Fetch Metadata on startup";
+      wantedBy = [ "multi-user.target" ];
+      before = [ "apply-ec2-data.service" "amazon-init.service"];
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      script = metadataFetcher;
+      restartIfChanged = false;
+      unitConfig.X-StopOnRemoval = false;
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+    };
+  };
+}
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index a847abdc542c1..7e207fa419f83 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -163,6 +163,9 @@ in
   openldap = handleTest ./openldap.nix {};
   opensmtpd = handleTest ./opensmtpd.nix {};
   openssh = handleTest ./openssh.nix {};
+  # openstack-image-userdata doesn't work in a sandbox as the simulated openstack instance needs network access
+  #openstack-image-userdata = (handleTestOn ["x86_64-linux"] ./openstack-image.nix {}).userdata or {};
+  openstack-image-metadata = (handleTestOn ["x86_64-linux"] ./openstack-image.nix {}).metadata or {};
   osquery = handleTest ./osquery.nix {};
   osrm-backend = handleTest ./osrm-backend.nix {};
   ostree = handleTest ./ostree.nix {};
diff --git a/nixos/tests/common/ec2.nix b/nixos/tests/common/ec2.nix
new file mode 100644
index 0000000000000..1e69b63191a70
--- /dev/null
+++ b/nixos/tests/common/ec2.nix
@@ -0,0 +1,49 @@
+{ pkgs, makeTest }:
+
+with pkgs.lib;
+
+{
+  makeEc2Test = { name, image, userData, script, hostname ? "ec2-instance", sshPublicKey ? null }:
+    let
+      metaData = pkgs.stdenv.mkDerivation {
+        name = "metadata";
+        buildCommand = ''
+          mkdir -p $out/1.0/meta-data
+          ln -s ${pkgs.writeText "userData" userData} $out/1.0/user-data
+          echo "${hostname}" > $out/1.0/meta-data/hostname
+          echo "(unknown)" > $out/1.0/meta-data/ami-manifest-path
+        '' + optionalString (sshPublicKey != null) ''
+          mkdir -p $out/1.0/meta-data/public-keys/0
+          ln -s ${pkgs.writeText "sshPublicKey" sshPublicKey} $out/1.0/meta-data/public-keys/0/openssh-key
+        '';
+      };
+    in makeTest {
+      name = "ec2-" + name;
+      nodes = {};
+      testScript =
+        ''
+          my $imageDir = ($ENV{'TMPDIR'} // "/tmp") . "/vm-state-machine";
+          mkdir $imageDir, 0700;
+          my $diskImage = "$imageDir/machine.qcow2";
+          system("qemu-img create -f qcow2 -o backing_file=${image}/nixos.qcow2 $diskImage") == 0 or die;
+          system("qemu-img resize $diskImage 10G") == 0 or die;
+
+          # Note: we use net=169.0.0.0/8 rather than
+          # net=169.254.0.0/16 to prevent dhcpcd from getting horribly
+          # confused. (It would get a DHCP lease in the 169.254.*
+          # range, which it would then configure and prompty delete
+          # again when it deletes link-local addresses.) Ideally we'd
+          # turn off the DHCP server, but qemu does not have an option
+          # to do that.
+          my $startCommand = "qemu-kvm -m 768";
+          $startCommand .= " -device virtio-net-pci,netdev=vlan0";
+          $startCommand .= " -netdev 'user,id=vlan0,net=169.0.0.0/8,guestfwd=tcp:169.254.169.254:80-cmd:${pkgs.micro-httpd}/bin/micro_httpd ${metaData}'";
+          $startCommand .= " -drive file=$diskImage,if=virtio,werror=report";
+          $startCommand .= " \$QEMU_OPTS";
+
+          my $machine = createMachine({ startCommand => $startCommand });
+
+          ${script}
+        '';
+    };
+}
diff --git a/nixos/tests/ec2.nix b/nixos/tests/ec2.nix
index ed6bf7da988ca..384fce67c227d 100644
--- a/nixos/tests/ec2.nix
+++ b/nixos/tests/ec2.nix
@@ -6,6 +6,8 @@
 with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
 
+with import common/ec2.nix { inherit makeTest pkgs; };
+
 let
   image =
     (import ../lib/eval-config.nix {
@@ -39,65 +41,14 @@ let
       ];
     }).config.system.build.amazonImage;
 
-  makeEc2Test = { name, userData, script, hostname ? "ec2-instance", sshPublicKey ? null }:
-    let
-      metaData = pkgs.stdenv.mkDerivation {
-        name = "metadata";
-        buildCommand = ''
-          mkdir -p $out/1.0/meta-data
-          ln -s ${pkgs.writeText "userData" userData} $out/1.0/user-data
-          echo "${hostname}" > $out/1.0/meta-data/hostname
-          echo "(unknown)" > $out/1.0/meta-data/ami-manifest-path
-        '' + optionalString (sshPublicKey != null) ''
-          mkdir -p $out/1.0/meta-data/public-keys/0
-          ln -s ${pkgs.writeText "sshPublicKey" sshPublicKey} $out/1.0/meta-data/public-keys/0/openssh-key
-        '';
-      };
-    in makeTest {
-      name = "ec2-" + name;
-      nodes = {};
-      testScript =
-        ''
-          my $imageDir = ($ENV{'TMPDIR'} // "/tmp") . "/vm-state-machine";
-          mkdir $imageDir, 0700;
-          my $diskImage = "$imageDir/machine.qcow2";
-          system("qemu-img create -f qcow2 -o backing_file=${image}/nixos.qcow2 $diskImage") == 0 or die;
-          system("qemu-img resize $diskImage 10G") == 0 or die;
-
-          # Note: we use net=169.0.0.0/8 rather than
-          # net=169.254.0.0/16 to prevent dhcpcd from getting horribly
-          # confused. (It would get a DHCP lease in the 169.254.*
-          # range, which it would then configure and prompty delete
-          # again when it deletes link-local addresses.) Ideally we'd
-          # turn off the DHCP server, but qemu does not have an option
-          # to do that.
-          my $startCommand = "qemu-kvm -m 768";
-          $startCommand .= " -device virtio-net-pci,netdev=vlan0";
-          $startCommand .= " -netdev 'user,id=vlan0,net=169.0.0.0/8,guestfwd=tcp:169.254.169.254:80-cmd:${pkgs.micro-httpd}/bin/micro_httpd ${metaData}'";
-          $startCommand .= " -drive file=$diskImage,if=virtio,werror=report";
-          $startCommand .= " \$QEMU_OPTS";
-
-          my $machine = createMachine({ startCommand => $startCommand });
-
-          ${script}
-        '';
-    };
-
-  snakeOilPrivateKey = ''
-    -----BEGIN OPENSSH PRIVATE KEY-----
-    b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
-    QyNTUxOQAAACDEPmwZv5dDPrMUaq0dDP+6eBTTe+QNrz14KBEIdhHd1QAAAJDufJ4S7nye
-    EgAAAAtzc2gtZWQyNTUxOQAAACDEPmwZv5dDPrMUaq0dDP+6eBTTe+QNrz14KBEIdhHd1Q
-    AAAECgwbDlYATM5/jypuptb0GF/+zWZcJfoVIFBG3LQeRyGsQ+bBm/l0M+sxRqrR0M/7p4
-    FNN75A2vPXgoEQh2Ed3VAAAADEVDMiB0ZXN0IGtleQE=
-    -----END OPENSSH PRIVATE KEY-----
-  '';
-
-  snakeOilPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMQ+bBm/l0M+sxRqrR0M/7p4FNN75A2vPXgoEQh2Ed3V EC2 test key";
+  sshKeys = import ./ssh-keys.nix pkgs;
+  snakeOilPrivateKey = sshKeys.snakeOilPrivateKey.text;
+  snakeOilPublicKey = sshKeys.snakeOilPublicKey;
 
 in {
   boot-ec2-nixops = makeEc2Test {
     name         = "nixops-userdata";
+    inherit image;
     sshPublicKey = snakeOilPublicKey; # That's right folks! My user's key is also the host key!
 
     userData = ''
@@ -142,6 +93,7 @@ in {
 
   boot-ec2-config = makeEc2Test {
     name         = "config-userdata";
+    inherit image;
     sshPublicKey = snakeOilPublicKey;
 
     # ### http://nixos.org/channels/nixos-unstable nixos
diff --git a/nixos/tests/openstack-image.nix b/nixos/tests/openstack-image.nix
new file mode 100644
index 0000000000000..d0225016ab762
--- /dev/null
+++ b/nixos/tests/openstack-image.nix
@@ -0,0 +1,88 @@
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
+
+with import ../lib/testing.nix { inherit system pkgs; };
+with pkgs.lib;
+
+with import common/ec2.nix { inherit makeTest pkgs; };
+
+let
+  image =
+    (import ../lib/eval-config.nix {
+      inherit system;
+      modules = [
+        ../maintainers/scripts/openstack/openstack-image.nix
+        ../modules/testing/test-instrumentation.nix
+        ../modules/profiles/qemu-guest.nix
+      ];
+    }).config.system.build.openstackImage;
+
+  sshKeys = import ./ssh-keys.nix pkgs;
+  snakeOilPrivateKey = sshKeys.snakeOilPrivateKey.text;
+  snakeOilPublicKey = sshKeys.snakeOilPublicKey;
+
+in {
+  metadata = makeEc2Test {
+    name = "openstack-ec2-metadata";
+    inherit image;
+    sshPublicKey = snakeOilPublicKey;
+    userData = ''
+      SSH_HOST_ED25519_KEY_PUB:${snakeOilPublicKey}
+      SSH_HOST_ED25519_KEY:${replaceStrings ["\n"] ["|"] snakeOilPrivateKey}
+    '';
+    script = ''
+      $machine->start;
+      $machine->waitForFile("/etc/ec2-metadata/user-data");
+      $machine->waitForUnit("sshd.service");
+
+      $machine->succeed("grep unknown /etc/ec2-metadata/ami-manifest-path");
+
+      # We have no keys configured on the client side yet, so this should fail
+      $machine->fail("ssh -o BatchMode=yes localhost exit");
+
+      # Let's install our client private key
+      $machine->succeed("mkdir -p ~/.ssh");
+
+      $machine->succeed("echo '${snakeOilPrivateKey}' > ~/.ssh/id_ed25519");
+      $machine->succeed("chmod 600 ~/.ssh/id_ed25519");
+
+      # We haven't configured the host key yet, so this should still fail
+      $machine->fail("ssh -o BatchMode=yes localhost exit");
+
+      # Add the host key; ssh should finally succeed
+      $machine->succeed("echo localhost,127.0.0.1 ${snakeOilPublicKey} > ~/.ssh/known_hosts");
+      $machine->succeed("ssh -o BatchMode=yes localhost exit");
+
+      # Just to make sure resizing is idempotent.
+      $machine->shutdown;
+      $machine->start;
+      $machine->waitForFile("/etc/ec2-metadata/user-data");
+    '';
+  };
+
+  userdata = makeEc2Test {
+    name = "openstack-ec2-metadata";
+    inherit image;
+    sshPublicKey = snakeOilPublicKey;
+    userData = ''
+      { pkgs, ... }:
+      {
+        imports = [
+          <nixpkgs/nixos/modules/virtualisation/openstack-config.nix>
+          <nixpkgs/nixos/modules/testing/test-instrumentation.nix>
+          <nixpkgs/nixos/modules/profiles/qemu-guest.nix>
+        ];
+        environment.etc.testFile = {
+          text = "whoa";
+        };
+      }
+    '';
+    script = ''
+      $machine->start;
+      $machine->waitForFile("/etc/testFile");
+      $machine->succeed("cat /etc/testFile | grep -q 'whoa'");
+    '';
+  };
+}