nixos/qemu-vm: Allow building a Nix store image instead of using 9p

Add the `useNixStoreImage` option, allowing a disk image with the
necessary contents from the Nix store to be built using
make-disk-image.nix. The image will be mounted at `/nix/store` and
acts as a drop-in replacement for the usual 9p mounting of the host's
Nix store.

This removes the performance penalty of 9p, drastically improving
execution speed of applications which do lots of reads from the Nix
store. The caveats are increased disk space usage and image build
time.

1 files changed, 56 insertions, 3 deletions

diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix
index c5f71b249a6d0..6af5848b16828 100644
--- a/nixos/modules/virtualisation/qemu-vm.nix
+++ b/nixos/modules/virtualisation/qemu-vm.nix
@@ -122,6 +122,12 @@ let
           TMPDIR=$(mktemp -d nix-vm.XXXXXXXXXX --tmpdir)
       fi
 
+      ${lib.optionalString cfg.useNixStoreImage
+      ''
+        # Create a writable copy/snapshot of the store image.
+        ${qemu}/bin/qemu-img create -f qcow2 -F qcow2 -b ${storeImage}/nixos.qcow2 "$TMPDIR"/store.img
+      ''}
+
       # Create a directory for exchanging data with the VM.
       mkdir -p "$TMPDIR/xchg"
 
@@ -263,6 +269,18 @@ let
         '' # */
     );
 
+  storeImage = import ../../lib/make-disk-image.nix {
+    inherit pkgs config lib;
+    additionalPaths = [ regInfo ];
+    format = "qcow2";
+    onlyNixStore = true;
+    partitionTableType = "none";
+    installBootLoader = false;
+    diskSize = "auto";
+    additionalSpace = "0M";
+    copyChannel = false;
+  };
+
 in
 
 {
@@ -608,6 +626,20 @@ in
         };
     };
 
+    virtualisation.useNixStoreImage =
+      mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Build and use a disk image for the Nix store, instead of
+          accessing the host's one through 9p.
+
+          For applications which do a lot of reads from the store,
+          this can drastically improve performance, but at the cost of
+          disk space and image build time.
+        '';
+      };
+
     virtualisation.useBootLoader =
       mkOption {
         type = types.bool;
@@ -762,9 +794,18 @@ in
     virtualisation.pathsInNixDB = [ config.system.build.toplevel ];
 
     virtualisation.sharedDirectories = {
-      nix-store = { source = "/nix/store"; target = "/nix/store"; };
-      xchg      = { source = ''"$TMPDIR"/xchg''; target = "/tmp/xchg"; };
-      shared    = { source = ''"''${SHARED_DIR:-$TMPDIR/xchg}"''; target = "/tmp/shared"; };
+      nix-store = mkIf (!cfg.useNixStoreImage) {
+        source = builtins.storeDir;
+        target = "/nix/store";
+      };
+      xchg = {
+        source = ''"$TMPDIR"/xchg'';
+        target = "/tmp/xchg";
+      };
+      shared = {
+        source = ''"''${SHARED_DIR:-$TMPDIR/xchg}"'';
+        target = "/tmp/shared";
+      };
     };
 
     virtualisation.qemu.networkingOptions =
@@ -815,6 +856,11 @@ in
         driveExtraOpts.cache = "writeback";
         driveExtraOpts.werror = "report";
       }]
+      (mkIf cfg.useNixStoreImage [{
+        name = "nix-store";
+        file = ''"$TMPDIR"/store.img'';
+        deviceExtraOpts.bootindex = if cfg.useBootLoader then "3" else "2";
+      }])
       (mkIf cfg.useBootLoader [
         # The order of this list determines the device names, see
         # note [Disk layout with `useBootLoader`].
@@ -865,6 +911,13 @@ in
             options = [ "mode=1777" "strictatime" "nosuid" "nodev" "size=${toString config.boot.tmpOnTmpfsSize}" ];
           };
 
+        "/nix/${if cfg.writableStore then ".ro-store" else "store"}" =
+          mkIf cfg.useNixStoreImage
+            { device = "${lookupDriveDeviceName "nix-store" cfg.qemu.drives}";
+              neededForBoot = true;
+              options = [ "ro" ];
+            };
+
         "/nix/.rw-store" = mkIf (cfg.writableStore && cfg.writableStoreUseTmpfs)
           { fsType = "tmpfs";
             options = [ "mode=0755" ];