Merge pull request #49 (support programs.sqlite)

This started with pull request #43, which was about disabling the
command-not-found handler. The follow-up pull request, which is what I'm
merging now, however re-uses the programs.sqlite that is shipped as part
of the official nixpkgs channels.

While ideally the programs.sqlite would be in par with the jobset input
for nixpkgsSrc on Hydra, it's not easily possible since we'd need extra
configuration on Hydra, which in turn would make it harder to reproduce
the configuration on other systems. There also is no "tarball" jobset
input, which would at least make it easier to do via declarative
jobsets.

So until we have a definitive solution to this, we currently use
fetchurl to get the programs.sqlite based on a fixed snapshots and in
addition to that, @sternenseeman also wrote a handy updater to make
updating a breeze.

While I didn't test this on one of my machines yet, I did however add a
Hydra jobset for the branch and it so far didn't introduce new eval
errors or build failures, so I consider this safe to merge.

Thanks to @sternenseeman for implementing both #43 and #49.

5 files changed, 147 insertions, 6 deletions

diff --git a/machines/sternenseemann/base.nix b/machines/sternenseemann/base.nix
index 75add239..4678b8b1 100644
--- a/machines/sternenseemann/base.nix
+++ b/machines/sternenseemann/base.nix
@@ -18,9 +18,6 @@ in {
     nix.extraOptions = "gc-keep-derivations = false";
     nixpkgs.config.allowUnfree = true;
 
-    # doesn't work with vuizvui atm
-    programs.command-not-found.enable = false;
-
     services.journald.extraConfig = lib.mkDefault "SystemMaxUse=500M";
 
     console.keyMap = lib.mkDefault "de-latin1";
diff --git a/modules/core/common.nix b/modules/core/common.nix
index 9c9c7a67..3f9f4f5f 100644
--- a/modules/core/common.nix
+++ b/modules/core/common.nix
@@ -2,7 +2,11 @@
 
 with lib;
 
-{
+let
+  rootChannelsPath = "/nix/var/nix/profiles/per-user/root/channels";
+  channelPath = "${rootChannelsPath}/${config.vuizvui.channelName}";
+
+in {
   options.vuizvui = {
     modifyNixPath = mkOption {
       type = types.bool;
@@ -55,8 +59,6 @@ with lib;
     in mkIf config.vuizvui.enableGlobalNixpkgsConfig (mkForce nixpkgsCfg);
 
     nix.nixPath = let
-      rootChannelsPath = "/nix/var/nix/profiles/per-user/root/channels";
-      channelPath = "${rootChannelsPath}/${config.vuizvui.channelName}";
       nixosConfig = "/etc/nixos/configuration.nix";
       nixpkgsConfig = "nixpkgs-config=${pkgs.writeText "nixpkgs-config.nix" ''
         (import ${pkgs.path}/nixos/lib/eval-config.nix {
@@ -70,5 +72,9 @@ with lib;
         rootChannelsPath
       ] ++ optional config.vuizvui.enableGlobalNixpkgsConfig nixpkgsConfig;
     in mkIf config.vuizvui.modifyNixPath (mkOverride 90 nixPath);
+
+    # correct path used by command-not-found which is enabled by default
+    programs.command-not-found.dbPath =
+      mkDefault "${channelPath}/nixpkgs/programs.sqlite";
   };
 }
diff --git a/pkgs/sternenseemann/default.nix b/pkgs/sternenseemann/default.nix
index c41cb3fa..e0c1bb16 100644
--- a/pkgs/sternenseemann/default.nix
+++ b/pkgs/sternenseemann/default.nix
@@ -116,4 +116,9 @@ lib.fix (self: {
   };
 
   unicode_clock = python3Packages.callPackage ./unicode_clock { };
+
+  vuizvui-update-programs-sqlite = python3Packages.callPackage ./vuizvui-update-programs-sqlite {
+    inherit (pkgs.writers) writePython3;
+    inherit (profpatsch) getBins;
+  };
 })
diff --git a/pkgs/sternenseemann/vuizvui-update-programs-sqlite/default.nix b/pkgs/sternenseemann/vuizvui-update-programs-sqlite/default.nix
new file mode 100644
index 00000000..6e14b948
--- /dev/null
+++ b/pkgs/sternenseemann/vuizvui-update-programs-sqlite/default.nix
@@ -0,0 +1,113 @@
+{ writePython3
+, getBins
+, requests
+, nix
+, gnutar
+}:
+
+let
+
+  bins = (getBins nix [ "nix-hash" ])
+      // (getBins gnutar [ "tar" ]);
+
+in
+
+writePython3 "vuizvui-update-programs-sqlite" {
+  flakeIgnore = [
+    # whitespaces around { }
+    "E201" "E202"
+    # fuck 4-space indentation
+    "E121" "E111"
+    # who cares about blank lines
+    "W391" "E302" "E305"
+    # URLs are long
+    "E501"
+  ];
+  libraries = [ requests ];
+} ''
+  from pathlib import Path
+  import re
+  import requests
+  import subprocess
+  import sys
+  from tempfile import TemporaryDirectory
+
+  def latest_nixexprs_url():
+    r = requests.head('https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz')
+
+    assert r.status_code == 301
+    return r.headers['location']
+
+  def nixos_version_for_url(url):
+    match = re.match(r"https://releases\.nixos\.org/nixos/unstable/nixos-([0-9a-fpre.]+)/nixexprs\.tar\.xz", url)
+    return match.group(1)
+
+  def download(url: str, filename: Path) -> Path:
+    with requests.get(url, stream=True) as r:
+      r.raise_for_status()
+      with open(filename, 'wb') as f:
+        for c in r.iter_content(chunk_size=16384):
+          f.write(c)
+
+    return filename
+
+  def main():
+    if len(sys.argv) > 2:
+        print(f'Usage: {sys.argv[0]} /path/to/release.nix', file=sys.stderr)
+        raise SystemExit(64)
+
+    url = latest_nixexprs_url()
+    version = nixos_version_for_url(url)
+
+    print(f'Updating programs.sqlite to {version}', file=sys.stderr)
+
+    with TemporaryDirectory(prefix="vuizvui-update-programs-sqlite") as dir:
+      nixexprs = download(url, dir / Path('nixexprs.tar.xz'))
+      programs_sqlite = dir / Path('programs.sqlite')
+
+      with open(programs_sqlite, 'wb') as f:
+        subprocess.run([
+            '${bins.tar}',
+            '-xJOf',
+            nixexprs,
+            f'nixos-{version}/programs.sqlite'
+          ], stdout=f, check=True)
+
+      hash = subprocess.run([
+          '${bins.nix-hash}',
+          '--base32',
+          '--type', 'sha256',
+          '--flat',
+          programs_sqlite
+        ],
+        check=True,
+        capture_output=True).stdout.decode('utf-8').strip()
+
+      print(f'New hash: {hash}', file=sys.stderr)
+
+      if len(sys.argv) == 1:
+        print('Doing nothing (dry run)', file=sys.stderr)
+      elif len(sys.argv) == 2:
+        release_nix = Path(sys.argv[1])
+
+        with open(release_nix, 'r+') as f:
+          text = f.read()
+          # base32 alphabet as per nix-rust/src/util/base32.rs
+          new_text = re.sub(r'programsSqliteSha256\s*=\s*"[0-9a-fg-np-sv-z]+"',
+                            f'programsSqliteSha256 = "{hash}"',
+                            text)
+          new_text = re.sub(r'programsSqliteVersion\s*=\s*"[0-9a-fpre.]+"',
+                            f'programsSqliteVersion = "{version}"',
+                            new_text)
+
+          if text == new_text:
+            print('Already up to date')
+          else:
+            f.seek(0)
+            f.write(new_text)
+
+            print(f'Wrote to {release_nix}', file=sys.stderr)
+
+  if __name__ == '__main__':
+    main()
+  ''
diff --git a/release.nix b/release.nix
index 21fc1e6e..b82cf0ce 100644
--- a/release.nix
+++ b/release.nix
@@ -14,6 +14,20 @@ let
   vuizvuiShortRev = vuizvuiSrc.shortRev or "abcdefg";
   vuizvuiVersion = "pre${toString vuizvuiRevCount}.${vuizvuiShortRev}";
 
+  # version of the nixos-unstable channel to get programs.sqlite from.
+  # Use pkgs.sternenseemann.vuizvui-update-programs-sqlite to update.
+  programsSqliteVersion = "21.05pre270709.6b1057b452c";
+  programsSqliteSha256 = "1qa3jir0r4mqijw694hi3dba34n1chisha01fzmvsfk4bgc98xqc";
+  programsSqlite = pkgsUpstream.fetchurl {
+    name = "programs.sqlite-${programsSqliteVersion}";
+    url = "https://releases.nixos.org/nixos/unstable/nixos-${programsSqliteVersion}/nixexprs.tar.xz";
+    sha256 = programsSqliteSha256;
+    downloadToTemp = true;
+    postFetch = ''
+      tar -xOJf $downloadedFile nixos-${programsSqliteVersion}/programs.sqlite > "$out"
+    '';
+  };
+
   vuizvui = let
     patchedVuizvui = (import nixpkgs {}).stdenv.mkDerivation {
       name = "vuizvui-${vuizvuiVersion}";
@@ -23,6 +37,12 @@ let
       installPhase = ''
         cp -r --no-preserve=ownership "${nixpkgs}/" nixpkgs
         chmod -R u+w nixpkgs
+        # since we fetch nixpkgsSrc using git, we don't get programs.sqlite
+        # for programs.command-not-found which is normally included in the
+        # channel. Building this ourselves is not desireable as it requires
+        # to build and index the whole of nixpkgs. Therefore we just inject
+        # it from a nixos channel (which possibly is a different version).
+        cp --no-preserve=ownership "${programsSqlite}" nixpkgs/programs.sqlite
         echo -n "$nixpkgsVersion" > nixpkgs/.version-suffix
         echo "echo '$nixpkgsVersion'" \
           > nixpkgs/nixos/modules/installer/tools/get-version-suffix