diff options
Diffstat (limited to 'pkgs/build-support/node/fetch-npm-deps/src/main.rs')
| -rw-r--r-- | pkgs/build-support/node/fetch-npm-deps/src/main.rs | 425 |
1 files changed, 73 insertions, 352 deletions
diff --git a/pkgs/build-support/node/fetch-npm-deps/src/main.rs b/pkgs/build-support/node/fetch-npm-deps/src/main.rs index 3d2204071a66a..57725a922dfd8 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/main.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/main.rs @@ -1,250 +1,18 @@ #![warn(clippy::pedantic)] use crate::cacache::Cache; -use anyhow::{anyhow, Context}; +use anyhow::anyhow; use rayon::prelude::*; -use serde::Deserialize; use serde_json::{Map, Value}; use std::{ - collections::{HashMap, HashSet}, - env, fmt, fs, io, + env, fs, path::Path, - process::{self, Command, Stdio}, + process::{self, Command}, }; use tempfile::tempdir; -use url::Url; mod cacache; -#[cfg(test)] -mod tests; - -#[derive(Deserialize)] -struct PackageLock { - #[serde(rename = "lockfileVersion")] - version: u8, - dependencies: Option<HashMap<String, OldPackage>>, - packages: Option<HashMap<String, Package>>, -} - -#[derive(Deserialize)] -struct OldPackage { - version: UrlOrString, - #[serde(default)] - bundled: bool, - resolved: Option<UrlOrString>, - integrity: Option<String>, - dependencies: Option<HashMap<String, OldPackage>>, -} - -#[derive(Debug, Deserialize, PartialEq, Eq)] -struct Package { - resolved: Option<UrlOrString>, - integrity: Option<String>, -} - -#[derive(Debug, Deserialize, PartialEq, Eq)] -#[serde(untagged)] -enum UrlOrString { - Url(Url), - String(String), -} - -impl fmt::Display for UrlOrString { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - UrlOrString::Url(url) => url.fmt(f), - UrlOrString::String(string) => string.fmt(f), - } - } -} - -#[allow(clippy::case_sensitive_file_extension_comparisons)] -fn to_new_packages( - old_packages: HashMap<String, OldPackage>, - initial_url: &Url, -) -> anyhow::Result<HashMap<String, Package>> { - let mut new = HashMap::new(); - - for (name, mut package) in old_packages { - // In some cases, a bundled dependency happens to have the same version as a non-bundled one, causing - // the bundled one without a URL to override the entry for the non-bundled instance, which prevents the - // dependency from being downloaded. - if package.bundled { - continue; - } - - if let UrlOrString::Url(v) = &package.version { - for (scheme, host) in [ - ("github", "github.com"), - ("bitbucket", "bitbucket.org"), - ("gitlab", "gitlab.com"), - ] { - if v.scheme() == scheme { - package.version = { - let mut new_url = initial_url.clone(); - - new_url.set_host(Some(host))?; - - if v.path().ends_with(".git") { - new_url.set_path(v.path()); - } else { - new_url.set_path(&format!("{}.git", v.path())); - } - - new_url.set_fragment(v.fragment()); - - UrlOrString::Url(new_url) - }; - - break; - } - } - } - - new.insert( - format!("{name}-{}", package.version), - Package { - resolved: if matches!(package.version, UrlOrString::Url(_)) { - Some(package.version) - } else { - package.resolved - }, - integrity: package.integrity, - }, - ); - - if let Some(dependencies) = package.dependencies { - new.extend(to_new_packages(dependencies, initial_url)?); - } - } - - Ok(new) -} - -#[allow(clippy::case_sensitive_file_extension_comparisons)] -fn get_hosted_git_url(url: &Url) -> Option<Url> { - if ["git", "http", "git+ssh", "git+https", "ssh", "https"].contains(&url.scheme()) { - let mut s = url.path_segments()?; - - match url.host_str()? { - "github.com" => { - let user = s.next()?; - let mut project = s.next()?; - let typ = s.next(); - let mut commit = s.next(); - - if typ.is_none() { - commit = url.fragment(); - } else if typ.is_some() && typ != Some("tree") { - return None; - } - - if project.ends_with(".git") { - project = project.strip_suffix(".git")?; - } - - let commit = commit.unwrap(); - - Some( - Url::parse(&format!( - "https://codeload.github.com/{user}/{project}/tar.gz/{commit}" - )) - .ok()?, - ) - } - "bitbucket.org" => { - let user = s.next()?; - let mut project = s.next()?; - let aux = s.next(); - - if aux == Some("get") { - return None; - } - - if project.ends_with(".git") { - project = project.strip_suffix(".git")?; - } - - let commit = url.fragment()?; - - Some( - Url::parse(&format!( - "https://bitbucket.org/{user}/{project}/get/{commit}.tar.gz" - )) - .ok()?, - ) - } - "gitlab.com" => { - let path = &url.path()[1..]; - - if path.contains("/~/") || path.contains("/archive.tar.gz") { - return None; - } - - let user = s.next()?; - let mut project = s.next()?; - - if project.ends_with(".git") { - project = project.strip_suffix(".git")?; - } - - let commit = url.fragment()?; - - Some( - Url::parse(&format!( - "https://gitlab.com/{user}/{project}/repository/archive.tar.gz?ref={commit}" - )) - .ok()?, - ) - } - "git.sr.ht" => { - let user = s.next()?; - let mut project = s.next()?; - let aux = s.next(); - - if aux == Some("archive") { - return None; - } - - if project.ends_with(".git") { - project = project.strip_suffix(".git")?; - } - - let commit = url.fragment()?; - - Some( - Url::parse(&format!( - "https://git.sr.ht/{user}/{project}/archive/{commit}.tar.gz" - )) - .ok()?, - ) - } - _ => None, - } - } else { - None - } -} - -fn get_ideal_hash(integrity: &str) -> anyhow::Result<&str> { - let split: Vec<_> = integrity.split_ascii_whitespace().collect(); - - if split.len() == 1 { - Ok(split[0]) - } else { - for hash in ["sha512-", "sha1-"] { - if let Some(h) = split.iter().find(|s| s.starts_with(hash)) { - return Ok(h); - } - } - - Err(anyhow!("not sure which hash to select out of {split:?}")) - } -} - -fn get_initial_url() -> anyhow::Result<Url> { - Url::parse("git+ssh://git@a.b").context("initial url should be valid") -} +mod parse; /// `fixup_lockfile` removes the `integrity` field from Git dependencies. /// @@ -294,7 +62,6 @@ fn fixup_lockfile(mut lock: Map<String, Value>) -> anyhow::Result<Option<Map<Str } } -#[allow(clippy::too_many_lines)] fn main() -> anyhow::Result<()> { let args = env::args().collect::<Vec<_>>(); @@ -319,7 +86,6 @@ fn main() -> anyhow::Result<()> { } let lock_content = fs::read_to_string(&args[1])?; - let lock: PackageLock = serde_json::from_str(&lock_content)?; let out_tempdir; @@ -331,137 +97,92 @@ fn main() -> anyhow::Result<()> { (out_tempdir.path(), true) }; - let agent = ureq::agent(); - - eprintln!("lockfile version: {}", lock.version); - - let packages = match lock.version { - 1 => { - let initial_url = get_initial_url()?; - - lock.dependencies - .map(|p| to_new_packages(p, &initial_url)) - .transpose()? - } - 2 | 3 => lock.packages, - _ => panic!( - "We don't support lockfile version {}, please file an issue.", - lock.version - ), - }; - - if packages.is_none() { - return Ok(()); - } - - let packages = { - let mut seen = HashSet::new(); - let mut new_packages = HashMap::new(); - - for (dep, package) in packages.unwrap().drain() { - if let (false, Some(UrlOrString::Url(resolved))) = (dep.is_empty(), &package.resolved) { - if !seen.contains(resolved) { - seen.insert(resolved.clone()); - new_packages.insert(dep, package); - } - } - } - - new_packages - }; + let packages = parse::lockfile(&lock_content, env::var("FORCE_GIT_DEPS").is_ok())?; let cache = Cache::new(out.join("_cacache")); - packages - .into_par_iter() - .try_for_each(|(dep, mut package)| { - eprintln!("{dep}"); + packages.into_par_iter().try_for_each(|package| { + eprintln!("{}", package.name); - let mut resolved = match package.resolved { - Some(UrlOrString::Url(url)) => url, - _ => unreachable!(), - }; + let tarball = package.tarball()?; + let integrity = package.integrity(); - let mut hosted = false; + cache + .put( + format!("make-fetch-happen:request-cache:{}", package.url), + package.url, + &tarball, + integrity, + ) + .map_err(|e| anyhow!("couldn't insert cache entry for {}: {e:?}", package.name))?; - if let Some(hosted_git_url) = get_hosted_git_url(&resolved) { - resolved = hosted_git_url; - package.integrity = None; - hosted = true; - } + Ok::<_, anyhow::Error>(()) + })?; - let mut data = Vec::new(); - - let mut body = agent.get(resolved.as_str()).call()?.into_reader(); - - if hosted { - let workdir = tempdir()?; + fs::write(out.join("package-lock.json"), lock_content)?; - let tar_path = workdir.path().join("package"); + if print_hash { + Command::new("nix") + .args(["--experimental-features", "nix-command", "hash", "path"]) + .arg(out.as_os_str()) + .status()?; + } - fs::create_dir(&tar_path)?; + Ok(()) +} - let mut cmd = Command::new("tar") - .args(["--extract", "--gzip", "--strip-components=1", "-C"]) - .arg(&tar_path) - .stdin(Stdio::piped()) - .spawn()?; +#[cfg(test)] +mod tests { + use super::fixup_lockfile; + use serde_json::json; + + #[test] + fn lockfile_fixup() -> anyhow::Result<()> { + let input = json!({ + "lockfileVersion": 2, + "name": "foo", + "packages": { + "": { - io::copy(&mut body, &mut cmd.stdin.take().unwrap())?; + }, + "foo": { + "resolved": "https://github.com/NixOS/nixpkgs", + "integrity": "aaa" + }, + "bar": { + "resolved": "git+ssh://git@github.com/NixOS/nixpkgs.git", + "integrity": "bbb" + } + } + }); - let exit = cmd.wait()?; + let expected = json!({ + "lockfileVersion": 2, + "name": "foo", + "packages": { + "": { - if !exit.success() { - return Err(anyhow!( - "failed to extract tarball for {dep}: tar exited with status code {}", - exit.code().unwrap() - )); + }, + "foo": { + "resolved": "https://github.com/NixOS/nixpkgs", + "integrity": "aaa" + }, + "bar": { + "resolved": "git+ssh://git@github.com/NixOS/nixpkgs.git", } - - data = Command::new("tar") - .args([ - "--sort=name", - "--mtime=@0", - "--owner=0", - "--group=0", - "--numeric-owner", - "--format=gnu", - "-I", - "gzip -n -9", - "--create", - "-C", - ]) - .arg(workdir.path()) - .arg("package") - .output()? - .stdout; - } else { - body.read_to_end(&mut data)?; } + }); - cache - .put( - format!("make-fetch-happen:request-cache:{resolved}"), - resolved, - &data, - package - .integrity - .map(|i| Ok::<String, anyhow::Error>(get_ideal_hash(&i)?.to_string())) - .transpose()?, - ) - .map_err(|e| anyhow!("couldn't insert cache entry for {dep}: {e:?}"))?; - - Ok::<_, anyhow::Error>(()) - })?; + assert_eq!( + fixup_lockfile(input.as_object().unwrap().clone())?, + Some(expected.as_object().unwrap().clone()) + ); - fs::write(out.join("package-lock.json"), lock_content)?; + assert_eq!( + fixup_lockfile(json!({"lockfileVersion": 1}).as_object().unwrap().clone())?, + None + ); - if print_hash { - Command::new("nix") - .args(["--experimental-features", "nix-command", "hash", "path"]) - .arg(out.as_os_str()) - .status()?; + Ok(()) } - - Ok(()) } |