sapling: simplify dependency generation script

5 files changed, 122 insertions, 197 deletions

diff --git a/pkgs/applications/version-management/sapling/default.nix b/pkgs/applications/version-management/sapling/default.nix
index c51454d5c5c0c..d964a9b5167ce 100644
--- a/pkgs/applications/version-management/sapling/default.nix
+++ b/pkgs/applications/version-management/sapling/default.nix
@@ -1,9 +1,7 @@
 { lib, stdenv, python3Packages, fetchFromGitHub, fetchurl, sd, curl, pkg-config, openssl, rustPlatform, fetchYarnDeps, yarn, nodejs, fixup_yarn_lock }:
 
 let
-  deps = import ./deps.nix { inherit lib fetchurl; };
-  version = deps.version;
-  versionHash = deps.versionHash;
+  inherit (lib.importJSON ./deps.json) links version versionHash;
 
   src = fetchFromGitHub {
     owner = "facebook";
@@ -83,9 +81,7 @@ let
     # with filesystem paths for the curl calls.
     postUnpack = ''
       mkdir $sourceRoot/hack_pydeps
-      ( cd $sourceRoot/hack_pydeps
-        ${deps.links}
-      )
+      ${lib.concatStrings (map (li: "ln -s ${fetchurl li} $sourceRoot/hack_pydeps/${baseNameOf li.url}\n") links)}
       sed -i "s|https://files.pythonhosted.org/packages/[[:alnum:]]*/[[:alnum:]]*/[[:alnum:]]*/|file://$NIX_BUILD_TOP/$sourceRoot/hack_pydeps/|g" $sourceRoot/setup.py
     '';
 
diff --git a/pkgs/applications/version-management/sapling/deps.json b/pkgs/applications/version-management/sapling/deps.json
new file mode 100644
index 0000000000000..06751b51b15e3
--- /dev/null
+++ b/pkgs/applications/version-management/sapling/deps.json
@@ -0,0 +1,78 @@
+{
+  "links": [
+    {
+      "sha256": "0dgg5x4nvdpfiz552diy11xg72y14s38hjz9qxygafnfgybg6hab",
+      "url": "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl"
+    },
+    {
+      "sha256": "0dq9f7irmml1nm9a2rx8dd6m2dqmzjj1x40mk0rg619wrdfsaj0b",
+      "url": "https://files.pythonhosted.org/packages/39/7b/88dbb785881c28a102619d46423cb853b46dbccc70d3ac362d99773a78ce/pexpect-4.8.0-py2.py3-none-any.whl"
+    },
+    {
+      "sha256": "0r4xy2sqwyhwlwj81zvhzbnx7a7r4xdz9xyzzkjczlx7gk1cig1d",
+      "url": "https://files.pythonhosted.org/packages/23/6a/210816c943c9aeeb29e4e18a298f14bf0e118fe222a23e13bfcc2d41b0a4/ipython-7.16.1-py3-none-any.whl"
+    },
+    {
+      "sha256": "0wwi1c6md4vkbcsfsf8dklf3vr4mcdj4mpxkanwgb6jb1432x5yw",
+      "url": "https://files.pythonhosted.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip"
+    },
+    {
+      "sha256": "16sgpg57kxx5jh467d9qwc2hwshfvdbl0xkafdp3qspvbfp46qc0",
+      "url": "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl"
+    },
+    {
+      "sha256": "1qn3bnyd7gdwkyk8nvlhiy3c6zbwjd49fjxj0gp8xxi9faiysiwz",
+      "url": "https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl"
+    },
+    {
+      "sha256": "0mfj5d1bgpy1clfgwrkmjqm2pv70pm01jp4iyyhphc96kyifdg7v",
+      "url": "https://files.pythonhosted.org/packages/4c/1c/ff6546b6c12603d8dd1070aa3c3d273ad4c07f5771689a7b69a550e8c951/backcall-0.2.0-py2.py3-none-any.whl"
+    },
+    {
+      "sha256": "0yxz45fzjsq6zh5f9cjl0gf4vfg1l7rd79zyb3ih544layjg3ff8",
+      "url": "https://files.pythonhosted.org/packages/4e/78/56aa1b5f4d8ac548755ae767d84f0be54fdd9d404197a3d9e4659d272348/setuptools-57.0.0-py3-none-any.whl"
+    },
+    {
+      "sha256": "1177pfa343r378020a85l3b16ak479qgyvh8k5719fgbkhm81d5y",
+      "url": "https://files.pythonhosted.org/packages/59/7c/e39aca596badaf1b78e8f547c807b04dae603a433d3e7a7e04d67f2ef3e5/wcwidth-0.2.5-py2.py3-none-any.whl"
+    },
+    {
+      "sha256": "1r55ffffaq4q3dpvha7iipgxlqwvjg5cklf9izr42xj5rr226r26",
+      "url": "https://files.pythonhosted.org/packages/87/61/2dfea88583d5454e3a64f9308a686071d58d59a55db638268a6413e1eb6d/prompt_toolkit-2.0.10-py3-none-any.whl"
+    },
+    {
+      "sha256": "08v36wa0kckc892bk4nypl6sszbysarm8jhslviz1agp2sf1jp3f",
+      "url": "https://files.pythonhosted.org/packages/6a/36/b1b9bfdf28690ae01d9ca0aa5b0d07cb4448ac65fb91dc7e2d094e3d992f/decorator-5.0.9-py3-none-any.whl"
+    },
+    {
+      "sha256": "0mnzcb714ynl1qlv9dwnh50rv75mmj18ywaxbl8xzm3l9m0syjcn",
+      "url": "https://files.pythonhosted.org/packages/9a/41/220f49aaea88bc6fa6cba8d05ecf24676326156c23b991e80b3f2fc24c77/pickleshare-0.7.5-py2.py3-none-any.whl"
+    },
+    {
+      "sha256": "13iv73575lilrm01ffhr8y8sxn8kxhvfqsgwckbr919725280vnn",
+      "url": "https://files.pythonhosted.org/packages/a6/c9/be11fce9810793676017f79ffab3c6cb18575844a6c7b8d4ed92f95de604/Pygments-2.9.0-py3-none-any.whl"
+    },
+    {
+      "sha256": "0i7ycyjad9kq6lgq5ih7j8xsm639z24250s6d17pp781v6hwdd3h",
+      "url": "https://files.pythonhosted.org/packages/ca/ab/872a23e29cec3cf2594af7e857f18b687ad21039c1f9b922fac5b9b142d5/traitlets-4.3.3-py2.py3-none-any.whl"
+    },
+    {
+      "sha256": "0m02dsi8lvrjf4bi20ab6lm7rr6krz7pg6lzk3xjs2l9hqfjzfwa",
+      "url": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl"
+    },
+    {
+      "sha256": "1f7sc4ydjj33gadcgfz8fcx02d1wm2frlqwzdik1krlr6wikgpbj",
+      "url": "https://files.pythonhosted.org/packages/fa/bc/9bd3b5c2b4774d5f33b2d544f1460be9df7df2fe42f352135381c347c69a/ipython_genutils-0.2.0-py2.py3-none-any.whl"
+    },
+    {
+      "sha256": "1ibql99agjf2gj7y0svzd5m0h81hailf4p3sj3yl9i1i8ykdj6wm",
+      "url": "https://files.pythonhosted.org/packages/fc/56/9f67dcd4a4b9960373173a31be1b8c47fe351a1c9385677a7bdd82810e57/ipdb-0.13.9.tar.gz"
+    },
+    {
+      "sha256": "04m31z011arz2b70rwwkhvzkb9d4yxcfbxpw27d6fa3n79a7sdxg",
+      "url": "https://files.pythonhosted.org/packages/bc/fa/8604d92ef753e0036d807f1b3179813ab2fa283e3b19c926e11673c8205b/Cython-0.29.26.tar.gz"
+    }
+  ],
+  "version": "0.1.20221118-210929-cfbb68aa",
+  "versionHash": "5535144625961033752"
+}
diff --git a/pkgs/applications/version-management/sapling/deps.nix b/pkgs/applications/version-management/sapling/deps.nix
deleted file mode 100644
index 8cf89ec392881..0000000000000
--- a/pkgs/applications/version-management/sapling/deps.nix
+++ /dev/null
@@ -1,104 +0,0 @@
-# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
-# Generated by `gen-deps`
-
-{ lib, fetchurl }:
-let
-  ptyprocess = fetchurl {
-    url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl";
-    sha256 = "0dgg5x4nvdpfiz552diy11xg72y14s38hjz9qxygafnfgybg6hab";
-  };
-  pexpect = fetchurl {
-    url = "https://files.pythonhosted.org/packages/39/7b/88dbb785881c28a102619d46423cb853b46dbccc70d3ac362d99773a78ce/pexpect-4.8.0-py2.py3-none-any.whl";
-    sha256 = "0dq9f7irmml1nm9a2rx8dd6m2dqmzjj1x40mk0rg619wrdfsaj0b";
-  };
-  ipython = fetchurl {
-    url = "https://files.pythonhosted.org/packages/23/6a/210816c943c9aeeb29e4e18a298f14bf0e118fe222a23e13bfcc2d41b0a4/ipython-7.16.1-py3-none-any.whl";
-    sha256 = "0r4xy2sqwyhwlwj81zvhzbnx7a7r4xdz9xyzzkjczlx7gk1cig1d";
-  };
-  simplegeneric = fetchurl {
-    url = "https://files.pythonhosted.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip";
-    sha256 = "0wwi1c6md4vkbcsfsf8dklf3vr4mcdj4mpxkanwgb6jb1432x5yw";
-  };
-  toml = fetchurl {
-    url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl";
-    sha256 = "16sgpg57kxx5jh467d9qwc2hwshfvdbl0xkafdp3qspvbfp46qc0";
-  };
-  colorama = fetchurl {
-    url = "https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl";
-    sha256 = "1qn3bnyd7gdwkyk8nvlhiy3c6zbwjd49fjxj0gp8xxi9faiysiwz";
-  };
-  backcall = fetchurl {
-    url = "https://files.pythonhosted.org/packages/4c/1c/ff6546b6c12603d8dd1070aa3c3d273ad4c07f5771689a7b69a550e8c951/backcall-0.2.0-py2.py3-none-any.whl";
-    sha256 = "0mfj5d1bgpy1clfgwrkmjqm2pv70pm01jp4iyyhphc96kyifdg7v";
-  };
-  setuptools = fetchurl {
-    url = "https://files.pythonhosted.org/packages/4e/78/56aa1b5f4d8ac548755ae767d84f0be54fdd9d404197a3d9e4659d272348/setuptools-57.0.0-py3-none-any.whl";
-    sha256 = "0yxz45fzjsq6zh5f9cjl0gf4vfg1l7rd79zyb3ih544layjg3ff8";
-  };
-  wcwidth = fetchurl {
-    url = "https://files.pythonhosted.org/packages/59/7c/e39aca596badaf1b78e8f547c807b04dae603a433d3e7a7e04d67f2ef3e5/wcwidth-0.2.5-py2.py3-none-any.whl";
-    sha256 = "1177pfa343r378020a85l3b16ak479qgyvh8k5719fgbkhm81d5y";
-  };
-  prompt_toolkit = fetchurl {
-    url = "https://files.pythonhosted.org/packages/87/61/2dfea88583d5454e3a64f9308a686071d58d59a55db638268a6413e1eb6d/prompt_toolkit-2.0.10-py3-none-any.whl";
-    sha256 = "1r55ffffaq4q3dpvha7iipgxlqwvjg5cklf9izr42xj5rr226r26";
-  };
-  decorator = fetchurl {
-    url = "https://files.pythonhosted.org/packages/6a/36/b1b9bfdf28690ae01d9ca0aa5b0d07cb4448ac65fb91dc7e2d094e3d992f/decorator-5.0.9-py3-none-any.whl";
-    sha256 = "08v36wa0kckc892bk4nypl6sszbysarm8jhslviz1agp2sf1jp3f";
-  };
-  pickleshare = fetchurl {
-    url = "https://files.pythonhosted.org/packages/9a/41/220f49aaea88bc6fa6cba8d05ecf24676326156c23b991e80b3f2fc24c77/pickleshare-0.7.5-py2.py3-none-any.whl";
-    sha256 = "0mnzcb714ynl1qlv9dwnh50rv75mmj18ywaxbl8xzm3l9m0syjcn";
-  };
-  Pygments = fetchurl {
-    url = "https://files.pythonhosted.org/packages/a6/c9/be11fce9810793676017f79ffab3c6cb18575844a6c7b8d4ed92f95de604/Pygments-2.9.0-py3-none-any.whl";
-    sha256 = "13iv73575lilrm01ffhr8y8sxn8kxhvfqsgwckbr919725280vnn";
-  };
-  traitlets = fetchurl {
-    url = "https://files.pythonhosted.org/packages/ca/ab/872a23e29cec3cf2594af7e857f18b687ad21039c1f9b922fac5b9b142d5/traitlets-4.3.3-py2.py3-none-any.whl";
-    sha256 = "0i7ycyjad9kq6lgq5ih7j8xsm639z24250s6d17pp781v6hwdd3h";
-  };
-  six = fetchurl {
-    url = "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl";
-    sha256 = "0m02dsi8lvrjf4bi20ab6lm7rr6krz7pg6lzk3xjs2l9hqfjzfwa";
-  };
-  ipython_genutils = fetchurl {
-    url = "https://files.pythonhosted.org/packages/fa/bc/9bd3b5c2b4774d5f33b2d544f1460be9df7df2fe42f352135381c347c69a/ipython_genutils-0.2.0-py2.py3-none-any.whl";
-    sha256 = "1f7sc4ydjj33gadcgfz8fcx02d1wm2frlqwzdik1krlr6wikgpbj";
-  };
-  ipdb = fetchurl {
-    url = "https://files.pythonhosted.org/packages/fc/56/9f67dcd4a4b9960373173a31be1b8c47fe351a1c9385677a7bdd82810e57/ipdb-0.13.9.tar.gz";
-    sha256 = "1ibql99agjf2gj7y0svzd5m0h81hailf4p3sj3yl9i1i8ykdj6wm";
-  };
-  Cython = fetchurl {
-    url = "https://files.pythonhosted.org/packages/bc/fa/8604d92ef753e0036d807f1b3179813ab2fa283e3b19c926e11673c8205b/Cython-0.29.26.tar.gz";
-    sha256 = "04m31z011arz2b70rwwkhvzkb9d4yxcfbxpw27d6fa3n79a7sdxg";
-  };
-in
-{
-  links = ''
-    ln -s ${ptyprocess} ptyprocess-0.7.0-py2.py3-none-any.whl
-    ln -s ${pexpect} pexpect-4.8.0-py2.py3-none-any.whl
-    ln -s ${ipython} ipython-7.16.1-py3-none-any.whl
-    ln -s ${simplegeneric} simplegeneric-0.8.1.zip
-    ln -s ${toml} toml-0.10.2-py2.py3-none-any.whl
-    ln -s ${colorama} colorama-0.4.4-py2.py3-none-any.whl
-    ln -s ${backcall} backcall-0.2.0-py2.py3-none-any.whl
-    ln -s ${setuptools} setuptools-57.0.0-py3-none-any.whl
-    ln -s ${wcwidth} wcwidth-0.2.5-py2.py3-none-any.whl
-    ln -s ${prompt_toolkit} prompt_toolkit-2.0.10-py3-none-any.whl
-    ln -s ${decorator} decorator-5.0.9-py3-none-any.whl
-    ln -s ${pickleshare} pickleshare-0.7.5-py2.py3-none-any.whl
-    ln -s ${Pygments} Pygments-2.9.0-py3-none-any.whl
-    ln -s ${traitlets} traitlets-4.3.3-py2.py3-none-any.whl
-    ln -s ${six} six-1.16.0-py2.py3-none-any.whl
-    ln -s ${ipython_genutils} ipython_genutils-0.2.0-py2.py3-none-any.whl
-    ln -s ${ipdb} ipdb-0.13.9.tar.gz
-    ln -s ${Cython} Cython-0.29.26.tar.gz
-  '';
-
-  version = "0.1.20221118-210929-cfbb68aa";
-
-  versionHash = "5535144625961033752";
-}
diff --git a/pkgs/applications/version-management/sapling/gen-deps b/pkgs/applications/version-management/sapling/gen-deps
deleted file mode 100755
index 3ff7e24916c1b..0000000000000
--- a/pkgs/applications/version-management/sapling/gen-deps
+++ /dev/null
@@ -1,87 +0,0 @@
-#!/usr/bin/env nix-shell
-#!nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests ])" nixpkgs-fmt
-
-# vim: set syntax=python:
-
-import hashlib
-import json
-import re
-import struct
-import subprocess
-
-import requests
-
-gen_file = "deps.nix"
-repo = "facebook/sapling"
-source_file = "eden/scm/setup.py"
-
-# Fetch the latest stable release metadata from GitHub
-latest = requests.get(f"https://api.github.com/repos/{repo}/releases/latest").text
-latest = json.loads(latest)
-
-# Find latest's git tag which corresponds to the Sapling version. Also needed
-# is a hash of the version, so calculate that here. Taken from Sapling source
-# `$root/eden/scm/setup_with_version.py`.
-version = str(latest["tag_name"])
-version_hash = str(
-    struct.unpack(">Q", hashlib.sha1(version.encode("ascii")).digest()[:8])[0]
-)
-
-# Fetch the `setup.py` source and look for instances of assets being downloaded
-# from files.pythonhosted.org.
-source = requests.get(
-    f"https://raw.githubusercontent.com/{repo}/{version}/{source_file}"
-).text
-matches = re.findall(
-    r'(https://files\.pythonhosted\.org/packages/.*?/.*?/.*?/)(.*?)(-.*?)"', source
-)
-
-# For each matched asset URL, prefetch said asset and build the corresponding
-# Nix code needed to call `fetchurl` and perform symlinks in the actual build.
-assets = ""
-links = ""
-for m in matches:
-    sha256 = (
-        subprocess.Popen(
-            ["nix-prefetch-url", "--type", "sha256", f"{m[0]}{m[1]}{m[2]}"],
-            text=True,
-            stdout=subprocess.PIPE,
-        )
-        .stdout.read()
-        .strip()
-    )
-    assets += """  {} = fetchurl {{
-    url = "{}{}{}";
-    sha256 = "{}";
-  }};\n""".format(
-        m[1], m[0], m[1], m[2], sha256
-    )
-    links += "    ln -s ${{{}}} {}{}\n".format(m[1], m[1], m[2])
-assets = assets.strip()
-links = links.strip()
-
-# Render an autogenerated Nix file with the fetched dependency and version
-# information for import in the main derivation.
-out = f"""
-# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
-# Generated by `gen-deps`
-
-{{ lib, fetchurl }}:
-let
-  {assets}
-in
-{{
-  links = ''
-    {links}
-  '';
-
-  version = "{version}";
-
-  versionHash = "{version_hash}";
-}}
-"""
-with open(gen_file, "w+") as f:
-    f.write(out.strip())
-
-# Formamt the generated Nix file, which also serves as a syntax sanity check.
-subprocess.run(["nixpkgs-fmt", gen_file], check=True)
diff --git a/pkgs/applications/version-management/sapling/gen-deps.py b/pkgs/applications/version-management/sapling/gen-deps.py
new file mode 100755
index 0000000000000..e3fe56f73221b
--- /dev/null
+++ b/pkgs/applications/version-management/sapling/gen-deps.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests ])"
+import json
+import re
+from hashlib import sha1
+from struct import unpack
+from subprocess import run
+
+from requests import get
+
+# Fetch the latest stable release metadata from GitHub
+latestTag = get("https://api.github.com/repos/facebook/sapling/releases/latest").json()[
+    "tag_name"
+]
+
+
+def nixPrefetchUrl(url):
+    return run(
+        ["nix-prefetch-url", "--type", "sha256", url],
+        check=True,
+        text=True,
+        capture_output=True,
+    ).stdout.rstrip()
+
+
+# Fetch the `setup.py` source and look for instances of assets being downloaded
+# from files.pythonhosted.org.
+setupPy = get(
+    f"https://github.com/facebook/sapling/raw/{latestTag}/eden/scm/setup.py"
+).text
+foundUrls = re.findall(r'(https://files\.pythonhosted\.org/packages/[^\s]+)"', setupPy)
+
+dataDeps = {
+    "links": [{"url": url, "sha256": nixPrefetchUrl(url)} for url in foundUrls],
+    "version": latestTag,
+    # Find latest's git tag which corresponds to the Sapling version. Also
+    # needed is a hash of the version, so calculate that here. Taken from
+    # Sapling source `$root/eden/scm/setup_with_version.py`.
+    "versionHash": str(unpack(">Q", sha1(latestTag.encode("ascii")).digest()[:8])[0]),
+}
+
+open("deps.json", "w").write(json.dumps(dataDeps, indent=2, sort_keys=True) + "\n")