diff options
author | Yureka <yuka@yuka.dev> | 2021-10-06 04:52:35 +0200 |
---|---|---|
committer | Yuka <yuka@yuka.dev> | 2021-10-20 11:39:16 +0200 |
commit | d6e0195ccd478ab6c7f46cf93b08804ad3ec6592 (patch) | |
tree | 12898be8973b09025542549401d4559de3341fc5 /pkgs/build-support | |
parent | 7141eb9c5790417da38ce5ea5b273cd1aa994307 (diff) |
prefetch-yarn-deps, fetchYarnDeps: init
Diffstat (limited to 'pkgs/build-support')
6 files changed, 280 insertions, 0 deletions
diff --git a/pkgs/build-support/node/fetch-yarn-deps/default.nix b/pkgs/build-support/node/fetch-yarn-deps/default.nix new file mode 100644 index 0000000000000..03be881311fbe --- /dev/null +++ b/pkgs/build-support/node/fetch-yarn-deps/default.nix @@ -0,0 +1,74 @@ +{ stdenv, lib, makeWrapper, coreutils, nix-prefetch-git, fetchurl, nodejs-slim, prefetch-yarn-deps, cacert, callPackage }: + +let + yarnpkg-lockfile-tar = fetchurl { + url = "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz"; + sha512 = "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ=="; + }; + +in { + prefetch-yarn-deps = stdenv.mkDerivation { + name = "prefetch-yarn-deps"; + + dontUnpack = true; + + nativeBuildInputs = [ makeWrapper ]; + buildInputs = [ coreutils nix-prefetch-git nodejs-slim ]; + + buildPhase = '' + runHook preBuild + + mkdir libexec + tar --strip-components=1 -xf ${yarnpkg-lockfile-tar} package/index.js + mv index.js libexec/yarnpkg-lockfile.js + cp ${./index.js} libexec/index.js + patchShebangs libexec/index.js + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + mkdir -p $out/bin + cp -r libexec $out + makeWrapper $out/libexec/index.js $out/bin/prefetch-yarn-deps \ + --prefix PATH : ${lib.makeBinPath [ coreutils nix-prefetch-git ]} + + runHook postInstall + ''; + }; + + fetchYarnDeps = let + f = { + name ? "offline", + yarnLock, + hash ? "", + sha256 ? "", + }: let + hash_ = + if hash != "" then { outputHashAlgo = null; outputHash = hash; } + else if sha256 != "" then { outputHashAlgo = "sha256"; outputHash = sha256; } + else throw "fetchYarnDeps requires a hash"; + in stdenv.mkDerivation { + inherit name; + + dontUnpack = true; + dontInstall = true; + + nativeBuildInputs = [ prefetch-yarn-deps ]; + GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + + buildPhase = '' + mkdir -p $out + (cd $out; prefetch-yarn-deps --verbose --builder ${yarnLock}) + ''; + + outputHashMode = "recursive"; + inherit (hash_) outputHashAlgo outputHash; + }; + + in lib.setFunctionArgs f (lib.functionArgs f) // { + tests = callPackage ./tests {}; + }; +} diff --git a/pkgs/build-support/node/fetch-yarn-deps/index.js b/pkgs/build-support/node/fetch-yarn-deps/index.js new file mode 100755 index 0000000000000..0952235ee0682 --- /dev/null +++ b/pkgs/build-support/node/fetch-yarn-deps/index.js @@ -0,0 +1,168 @@ +#!/usr/bin/env node +'use strict' + +const fs = require('fs') +const crypto = require('crypto') +const process = require('process') +const https = require('https') +const child_process = require('child_process') +const path = require('path') +const lockfile = require('./yarnpkg-lockfile.js') +const { promisify } = require('util') + +const execFile = promisify(child_process.execFile) + +const exec = async (...args) => { + const res = await execFile(...args) + if (res.error) throw new Error(res.stderr) + return res +} + +const downloadFileHttps = (fileName, url, expectedHash) => { + return new Promise((resolve, reject) => { + https.get(url, (res) => { + const file = fs.createWriteStream(fileName) + const hash = crypto.createHash('sha1') + res.pipe(file) + res.pipe(hash).setEncoding('hex') + res.on('end', () => { + file.close() + const h = hash.read() + if (h != expectedHash) return reject(new Error(`hash mismatch, expected ${expectedHash}, got ${h}`)) + resolve() + }) + res.on('error', e => reject(e)) + }) + }) +} + +const downloadGit = async (fileName, url, rev) => { + await exec('nix-prefetch-git', [ + '--out', fileName + '.tmp', + '--url', url, + '--rev', rev, + '--builder' + ]) + + await exec('tar', [ + // hopefully make it reproducible across runs and systems + '--owner=0', '--group=0', '--numeric-owner', '--format=gnu', '--sort=name', '--mtime=@1', + + // Set u+w because tar-fs can't unpack archives with read-only dirs: https://github.com/mafintosh/tar-fs/issues/79 + '--mode', 'u+w', + + '-C', fileName + '.tmp', + '-cf', fileName, '.' + ]) + + await exec('rm', [ '-rf', fileName + '.tmp', ]) +} + +const downloadPkg = (pkg, verbose) => { + const [ url, hash ] = pkg.resolved.split('#') + if (verbose) console.log('downloading ' + url) + if (url.startsWith('https://codeload.github.com/') && url.includes('/tar.gz/')) { + const fileName = path.basename(url) + const s = url.split('/') + downloadGit(fileName, `https://github.com/${s[3]}/${s[4]}.git`, s[6]) + } else if (url.startsWith('https://')) { + const fileName = url + .replace(/https:\/\/(.)*(.com)\//g, '') // prevents having long directory names + .replace(/[@/%:-]/g, '_') // replace @ and : and - and % characters with underscore + + return downloadFileHttps(fileName, url, hash) + } else if (url.startsWith('git+')) { + const fileName = path.basename(url) + return downloadGit(fileName, url.replace(/^git\+/, ''), hash) + } else { + throw new Error('don\'t know how to download "' + url + '"') + } +} + +const performParallel = tasks => { + const worker = async () => { + while (tasks.length > 0) await tasks.shift()() + } + + const workers = [] + for (let i = 0; i < 4; i++) { + workers.push(worker()) + } + + return Promise.all(workers) +} + +const prefetchYarnDeps = async (lockData, verbose) => { + const tasks = Object.values( + Object.entries(lockData.object) + .map(([key, value]) => { + return { key, ...value } + }) + .reduce((out, pkg) => { + out[pkg.resolved] = pkg + return out + }, {}) + ) + .map(pkg => () => downloadPkg(pkg, verbose)) + + await performParallel(tasks) + if (verbose) console.log('Done') +} + +const showUsage = async () => { + process.stderr.write(` +syntax: prefetch-yarn-deps [path to yarn.lock] [options] + +Options: + -h --help Show this help + -v --verbose Verbose output + --builder Only perform the download to current directory, then exit +`) + process.exit(1) +} + +const main = async () => { + const args = process.argv.slice(2) + let next, lockFile, verbose, isBuilder + while (next = args.shift()) { + if (next == '--builder') { + isBuilder = true + } else if (next == '--verbose' || next == '-v') { + verbose = true + } else if (next == '--help' || next == '-h') { + showUsage() + } else if (!lockFile) { + lockFile = next + } else { + showUsage() + } + } + let lockContents + try { + lockContents = await fs.promises.readFile(lockFile || 'yarn.lock', 'utf-8') + } catch(e) { + showUsage() + } + const lockData = lockfile.parse(lockContents) + + if (isBuilder) { + await prefetchYarnDeps(lockData, verbose) + } else { + const { stdout: tmpDir } = await exec('mktemp', [ '-d' ]) + + try { + process.chdir(tmpDir.trim()) + await prefetchYarnDeps(lockData, verbose) + const { stdout: hash } = await exec('nix-hash', [ '--type', 'sha256', '--base32', tmpDir.trim() ]) + console.log(hash) + } finally { + await exec('rm', [ '-rf', tmpDir.trim() ]) + } + } +} + +main() + .catch(e => { + console.error(e) + process.exit(1) + }) diff --git a/pkgs/build-support/node/fetch-yarn-deps/tests/default.nix b/pkgs/build-support/node/fetch-yarn-deps/tests/default.nix new file mode 100644 index 0000000000000..a781dad830725 --- /dev/null +++ b/pkgs/build-support/node/fetch-yarn-deps/tests/default.nix @@ -0,0 +1,16 @@ +{ invalidateFetcherByDrvHash, fetchYarnDeps, ... }: + +{ + simple = invalidateFetcherByDrvHash fetchYarnDeps { + yarnLock = ./simple.lock; + sha256 = "sha256-Erdkw2E8wWT09jFNLXGkrdwKl0HuSZWnUDJUrV95vSE="; + }; + gitDep = invalidateFetcherByDrvHash fetchYarnDeps { + yarnLock = ./git.lock; + sha256 = "sha256-lAqN4LpoE+jgsQO1nDtuORwcVEO7ogEV53jCu2jFJUI="; + }; + githubDep = invalidateFetcherByDrvHash fetchYarnDeps { + yarnLock = ./github.lock; + sha256 = "sha256-Tsfgyjxz8x6gNmfN0xR7G/NQNoEs4svxRN/N+26vfJU="; + }; +} diff --git a/pkgs/build-support/node/fetch-yarn-deps/tests/git.lock b/pkgs/build-support/node/fetch-yarn-deps/tests/git.lock new file mode 100644 index 0000000000000..9eda5b2c409db --- /dev/null +++ b/pkgs/build-support/node/fetch-yarn-deps/tests/git.lock @@ -0,0 +1,7 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"async@git+https://github.com/caolan/async": + version "3.2.1" + resolved "git+https://github.com/caolan/async#fc9ba651341af5ab974aade6b1640e345912be83" diff --git a/pkgs/build-support/node/fetch-yarn-deps/tests/github.lock b/pkgs/build-support/node/fetch-yarn-deps/tests/github.lock new file mode 100644 index 0000000000000..057e043a53908 --- /dev/null +++ b/pkgs/build-support/node/fetch-yarn-deps/tests/github.lock @@ -0,0 +1,7 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"async@github:caolan/async": + version "3.2.1" + resolved "https://codeload.github.com/caolan/async/tar.gz/fc9ba651341af5ab974aade6b1640e345912be83" diff --git a/pkgs/build-support/node/fetch-yarn-deps/tests/simple.lock b/pkgs/build-support/node/fetch-yarn-deps/tests/simple.lock new file mode 100644 index 0000000000000..db2f4b2be4b79 --- /dev/null +++ b/pkgs/build-support/node/fetch-yarn-deps/tests/simple.lock @@ -0,0 +1,8 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +lit-html@1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/lit-html/-/lit-html-1.4.1.tgz#0c6f3ee4ad4eb610a49831787f0478ad8e9ae5e0" + integrity sha512-B9btcSgPYb1q4oSOb/PrOT6Z/H+r6xuNzfH4lFli/AWhYwdtrgQkQWBbIc6mdnf6E2IL3gDXdkkqNktpU0OZQA== |