diff options
author | Winter <winter@winter.cafe> | 2022-12-05 20:15:49 -0500 |
---|---|---|
committer | Lily Foster <lily@lily.flowers> | 2023-04-06 11:32:23 -0400 |
commit | 3e68a27822eb3e2a27282e32289a13f5b988c08d (patch) | |
tree | 2e8d54b3b2f388324e35e0f81ccb9ff1d5409fe3 | |
parent | 9c8ff8b426a8b07b9e0a131ac3218740dc85ba1e (diff) |
prefetch-npm-deps: refactor
This splits prefetch-npm-deps into multiple files, as well as making a few small changes along the way, such as going from a `HashMap` to a `Vec` as the container for packages, to deduplicate them more efficently.
-rw-r--r-- | pkgs/build-support/node/fetch-npm-deps/src/main.rs | 425 | ||||
-rw-r--r-- | pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs | 191 | ||||
-rw-r--r-- | pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs | 311 | ||||
-rw-r--r-- | pkgs/build-support/node/fetch-npm-deps/src/tests.rs | 141 |
4 files changed, 575 insertions, 493 deletions
diff --git a/pkgs/build-support/node/fetch-npm-deps/src/main.rs b/pkgs/build-support/node/fetch-npm-deps/src/main.rs index 3d2204071a66a..068a6d2bcd60b 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/main.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/main.rs @@ -1,250 +1,18 @@ #![warn(clippy::pedantic)] use crate::cacache::Cache; -use anyhow::{anyhow, Context}; +use anyhow::anyhow; use rayon::prelude::*; -use serde::Deserialize; use serde_json::{Map, Value}; use std::{ - collections::{HashMap, HashSet}, - env, fmt, fs, io, + env, fs, path::Path, - process::{self, Command, Stdio}, + process::{self, Command}, }; use tempfile::tempdir; -use url::Url; mod cacache; -#[cfg(test)] -mod tests; - -#[derive(Deserialize)] -struct PackageLock { - #[serde(rename = "lockfileVersion")] - version: u8, - dependencies: Option<HashMap<String, OldPackage>>, - packages: Option<HashMap<String, Package>>, -} - -#[derive(Deserialize)] -struct OldPackage { - version: UrlOrString, - #[serde(default)] - bundled: bool, - resolved: Option<UrlOrString>, - integrity: Option<String>, - dependencies: Option<HashMap<String, OldPackage>>, -} - -#[derive(Debug, Deserialize, PartialEq, Eq)] -struct Package { - resolved: Option<UrlOrString>, - integrity: Option<String>, -} - -#[derive(Debug, Deserialize, PartialEq, Eq)] -#[serde(untagged)] -enum UrlOrString { - Url(Url), - String(String), -} - -impl fmt::Display for UrlOrString { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - UrlOrString::Url(url) => url.fmt(f), - UrlOrString::String(string) => string.fmt(f), - } - } -} - -#[allow(clippy::case_sensitive_file_extension_comparisons)] -fn to_new_packages( - old_packages: HashMap<String, OldPackage>, - initial_url: &Url, -) -> anyhow::Result<HashMap<String, Package>> { - let mut new = HashMap::new(); - - for (name, mut package) in old_packages { - // In some cases, a bundled dependency happens to have the same version as a non-bundled one, causing - // the bundled one without a URL to override the entry for the non-bundled instance, which prevents the - // dependency from being downloaded. - if package.bundled { - continue; - } - - if let UrlOrString::Url(v) = &package.version { - for (scheme, host) in [ - ("github", "github.com"), - ("bitbucket", "bitbucket.org"), - ("gitlab", "gitlab.com"), - ] { - if v.scheme() == scheme { - package.version = { - let mut new_url = initial_url.clone(); - - new_url.set_host(Some(host))?; - - if v.path().ends_with(".git") { - new_url.set_path(v.path()); - } else { - new_url.set_path(&format!("{}.git", v.path())); - } - - new_url.set_fragment(v.fragment()); - - UrlOrString::Url(new_url) - }; - - break; - } - } - } - - new.insert( - format!("{name}-{}", package.version), - Package { - resolved: if matches!(package.version, UrlOrString::Url(_)) { - Some(package.version) - } else { - package.resolved - }, - integrity: package.integrity, - }, - ); - - if let Some(dependencies) = package.dependencies { - new.extend(to_new_packages(dependencies, initial_url)?); - } - } - - Ok(new) -} - -#[allow(clippy::case_sensitive_file_extension_comparisons)] -fn get_hosted_git_url(url: &Url) -> Option<Url> { - if ["git", "http", "git+ssh", "git+https", "ssh", "https"].contains(&url.scheme()) { - let mut s = url.path_segments()?; - - match url.host_str()? { - "github.com" => { - let user = s.next()?; - let mut project = s.next()?; - let typ = s.next(); - let mut commit = s.next(); - - if typ.is_none() { - commit = url.fragment(); - } else if typ.is_some() && typ != Some("tree") { - return None; - } - - if project.ends_with(".git") { - project = project.strip_suffix(".git")?; - } - - let commit = commit.unwrap(); - - Some( - Url::parse(&format!( - "https://codeload.github.com/{user}/{project}/tar.gz/{commit}" - )) - .ok()?, - ) - } - "bitbucket.org" => { - let user = s.next()?; - let mut project = s.next()?; - let aux = s.next(); - - if aux == Some("get") { - return None; - } - - if project.ends_with(".git") { - project = project.strip_suffix(".git")?; - } - - let commit = url.fragment()?; - - Some( - Url::parse(&format!( - "https://bitbucket.org/{user}/{project}/get/{commit}.tar.gz" - )) - .ok()?, - ) - } - "gitlab.com" => { - let path = &url.path()[1..]; - - if path.contains("/~/") || path.contains("/archive.tar.gz") { - return None; - } - - let user = s.next()?; - let mut project = s.next()?; - - if project.ends_with(".git") { - project = project.strip_suffix(".git")?; - } - - let commit = url.fragment()?; - - Some( - Url::parse(&format!( - "https://gitlab.com/{user}/{project}/repository/archive.tar.gz?ref={commit}" - )) - .ok()?, - ) - } - "git.sr.ht" => { - let user = s.next()?; - let mut project = s.next()?; - let aux = s.next(); - - if aux == Some("archive") { - return None; - } - - if project.ends_with(".git") { - project = project.strip_suffix(".git")?; - } - - let commit = url.fragment()?; - - Some( - Url::parse(&format!( - "https://git.sr.ht/{user}/{project}/archive/{commit}.tar.gz" - )) - .ok()?, - ) - } - _ => None, - } - } else { - None - } -} - -fn get_ideal_hash(integrity: &str) -> anyhow::Result<&str> { - let split: Vec<_> = integrity.split_ascii_whitespace().collect(); - - if split.len() == 1 { - Ok(split[0]) - } else { - for hash in ["sha512-", "sha1-"] { - if let Some(h) = split.iter().find(|s| s.starts_with(hash)) { - return Ok(h); - } - } - - Err(anyhow!("not sure which hash to select out of {split:?}")) - } -} - -fn get_initial_url() -> anyhow::Result<Url> { - Url::parse("git+ssh://git@a.b").context("initial url should be valid") -} +mod parse; /// `fixup_lockfile` removes the `integrity` field from Git dependencies. /// @@ -294,7 +62,6 @@ fn fixup_lockfile(mut lock: Map<String, Value>) -> anyhow::Result<Option<Map<Str } } -#[allow(clippy::too_many_lines)] fn main() -> anyhow::Result<()> { let args = env::args().collect::<Vec<_>>(); @@ -319,7 +86,6 @@ fn main() -> anyhow::Result<()> { } let lock_content = fs::read_to_string(&args[1])?; - let lock: PackageLock = serde_json::from_str(&lock_content)?; let out_tempdir; @@ -331,137 +97,92 @@ fn main() -> anyhow::Result<()> { (out_tempdir.path(), true) }; - let agent = ureq::agent(); - - eprintln!("lockfile version: {}", lock.version); - - let packages = match lock.version { - 1 => { - let initial_url = get_initial_url()?; - - lock.dependencies - .map(|p| to_new_packages(p, &initial_url)) - .transpose()? - } - 2 | 3 => lock.packages, - _ => panic!( - "We don't support lockfile version {}, please file an issue.", - lock.version - ), - }; - - if packages.is_none() { - return Ok(()); - } - - let packages = { - let mut seen = HashSet::new(); - let mut new_packages = HashMap::new(); - - for (dep, package) in packages.unwrap().drain() { - if let (false, Some(UrlOrString::Url(resolved))) = (dep.is_empty(), &package.resolved) { - if !seen.contains(resolved) { - seen.insert(resolved.clone()); - new_packages.insert(dep, package); - } - } - } - - new_packages - }; + let packages = parse::lockfile(&lock_content)?; let cache = Cache::new(out.join("_cacache")); - packages - .into_par_iter() - .try_for_each(|(dep, mut package)| { - eprintln!("{dep}"); + packages.into_par_iter().try_for_each(|package| { + eprintln!("{}", package.name); - let mut resolved = match package.resolved { - Some(UrlOrString::Url(url)) => url, - _ => unreachable!(), - }; + let tarball = package.tarball()?; + let integrity = package.integrity(); - let mut hosted = false; + cache + .put( + format!("make-fetch-happen:request-cache:{}", package.url), + package.url, + &tarball, + integrity, + ) + .map_err(|e| anyhow!("couldn't insert cache entry for {}: {e:?}", package.name))?; - if let Some(hosted_git_url) = get_hosted_git_url(&resolved) { - resolved = hosted_git_url; - package.integrity = None; - hosted = true; - } + Ok::<_, anyhow::Error>(()) + })?; - let mut data = Vec::new(); - - let mut body = agent.get(resolved.as_str()).call()?.into_reader(); - - if hosted { - let workdir = tempdir()?; + fs::write(out.join("package-lock.json"), lock_content)?; - let tar_path = workdir.path().join("package"); + if print_hash { + Command::new("nix") + .args(["--experimental-features", "nix-command", "hash", "path"]) + .arg(out.as_os_str()) + .status()?; + } - fs::create_dir(&tar_path)?; + Ok(()) +} - let mut cmd = Command::new("tar") - .args(["--extract", "--gzip", "--strip-components=1", "-C"]) - .arg(&tar_path) - .stdin(Stdio::piped()) - .spawn()?; +#[cfg(test)] +mod tests { + use super::fixup_lockfile; + use serde_json::json; + + #[test] + fn lockfile_fixup() -> anyhow::Result<()> { + let input = json!({ + "lockfileVersion": 2, + "name": "foo", + "packages": { + "": { - io::copy(&mut body, &mut cmd.stdin.take().unwrap())?; + }, + "foo": { + "resolved": "https://github.com/NixOS/nixpkgs", + "integrity": "aaa" + }, + "bar": { + "resolved": "git+ssh://git@github.com/NixOS/nixpkgs.git", + "integrity": "bbb" + } + } + }); - let exit = cmd.wait()?; + let expected = json!({ + "lockfileVersion": 2, + "name": "foo", + "packages": { + "": { - if !exit.success() { - return Err(anyhow!( - "failed to extract tarball for {dep}: tar exited with status code {}", - exit.code().unwrap() - )); + }, + "foo": { + "resolved": "https://github.com/NixOS/nixpkgs", + "integrity": "aaa" + }, + "bar": { + "resolved": "git+ssh://git@github.com/NixOS/nixpkgs.git", } - - data = Command::new("tar") - .args([ - "--sort=name", - "--mtime=@0", - "--owner=0", - "--group=0", - "--numeric-owner", - "--format=gnu", - "-I", - "gzip -n -9", - "--create", - "-C", - ]) - .arg(workdir.path()) - .arg("package") - .output()? - .stdout; - } else { - body.read_to_end(&mut data)?; } + }); - cache - .put( - format!("make-fetch-happen:request-cache:{resolved}"), - resolved, - &data, - package - .integrity - .map(|i| Ok::<String, anyhow::Error>(get_ideal_hash(&i)?.to_string())) - .transpose()?, - ) - .map_err(|e| anyhow!("couldn't insert cache entry for {dep}: {e:?}"))?; - - Ok::<_, anyhow::Error>(()) - })?; + assert_eq!( + fixup_lockfile(input.as_object().unwrap().clone())?, + Some(expected.as_object().unwrap().clone()) + ); - fs::write(out.join("package-lock.json"), lock_content)?; + assert_eq!( + fixup_lockfile(json!({"lockfileVersion": 1}).as_object().unwrap().clone())?, + None + ); - if print_hash { - Command::new("nix") - .args(["--experimental-features", "nix-command", "hash", "path"]) - .arg(out.as_os_str()) - .status()?; + Ok(()) } - - Ok(()) } diff --git a/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs b/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs new file mode 100644 index 0000000000000..99bd3020b5237 --- /dev/null +++ b/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs @@ -0,0 +1,191 @@ +use anyhow::{bail, Context}; +use rayon::slice::ParallelSliceMut; +use serde::Deserialize; +use std::{collections::HashMap, fmt}; +use url::Url; + +pub(super) fn packages(content: &str) -> anyhow::Result<Vec<Package>> { + let lockfile: Lockfile = serde_json::from_str(content)?; + + let mut packages = match lockfile.version { + 1 => { + let initial_url = get_initial_url()?; + + lockfile + .dependencies + .map(|p| to_new_packages(p, &initial_url)) + .transpose()? + } + 2 | 3 => lockfile.packages.map(|pkgs| { + pkgs.into_iter() + .filter(|(n, p)| !n.is_empty() && matches!(p.resolved, Some(UrlOrString::Url(_)))) + .map(|(n, p)| Package { name: Some(n), ..p }) + .collect() + }), + _ => bail!( + "We don't support lockfile version {}, please file an issue.", + lockfile.version + ), + } + .expect("lockfile should have packages"); + + packages.par_sort_by(|x, y| { + x.resolved + .partial_cmp(&y.resolved) + .expect("resolved should be comparable") + }); + + packages.dedup_by(|x, y| x.resolved == y.resolved); + + Ok(packages) +} + +#[derive(Deserialize)] +struct Lockfile { + #[serde(rename = "lockfileVersion")] + version: u8, + dependencies: Option<HashMap<String, OldPackage>>, + packages: Option<HashMap<String, Package>>, +} + +#[derive(Deserialize)] +struct OldPackage { + version: UrlOrString, + #[serde(default)] + bundled: bool, + resolved: Option<UrlOrString>, + integrity: Option<String>, + dependencies: Option<HashMap<String, OldPackage>>, +} + +#[derive(Debug, Deserialize, PartialEq, Eq)] +pub(super) struct Package { + #[serde(default)] + pub(super) name: Option<String>, + pub(super) resolved: Option<UrlOrString>, + pub(super) integrity: Option<String>, +} + +#[derive(Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(untagged)] +pub(super) enum UrlOrString { + Url(Url), + String(String), +} + +impl fmt::Display for UrlOrString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + UrlOrString::Url(url) => url.fmt(f), + UrlOrString::String(string) => string.fmt(f), + } + } +} + +#[allow(clippy::case_sensitive_file_extension_comparisons)] +fn to_new_packages( + old_packages: HashMap<String, OldPackage>, + initial_url: &Url, +) -> anyhow::Result<Vec<Package>> { + let mut new = Vec::new(); + + for (name, mut package) in old_packages { + // In some cases, a bundled dependency happens to have the same version as a non-bundled one, causing + // the bundled one without a URL to override the entry for the non-bundled instance, which prevents the + // dependency from being downloaded. + if package.bundled { + continue; + } + + if let UrlOrString::Url(v) = &package.version { + for (scheme, host) in [ + ("github", "github.com"), + ("bitbucket", "bitbucket.org"), + ("gitlab", "gitlab.com"), + ] { + if v.scheme() == scheme { + package.version = { + let mut new_url = initial_url.clone(); + + new_url.set_host(Some(host))?; + + if v.path().ends_with(".git") { + new_url.set_path(v.path()); + } else { + new_url.set_path(&format!("{}.git", v.path())); + } + + new_url.set_fragment(v.fragment()); + + UrlOrString::Url(new_url) + }; + + break; + } + } + } + + new.push(Package { + name: Some(name), + resolved: if matches!(package.version, UrlOrString::Url(_)) { + Some(package.version) + } else { + package.resolved + }, + integrity: package.integrity, + }); + + if let Some(dependencies) = package.dependencies { + new.append(&mut to_new_packages(dependencies, initial_url)?); + } + } + + Ok(new) +} + +fn get_initial_url() -> anyhow::Result<Url> { + Url::parse("git+ssh://git@a.b").context("initial url should be valid") +} + +#[cfg(test)] +mod tests { + use super::{get_initial_url, to_new_packages, OldPackage, Package, UrlOrString}; + use std::collections::HashMap; + use url::Url; + + #[test] + fn git_shorthand_v1() -> anyhow::Result<()> { + let old = { + let mut o = HashMap::new(); + o.insert( + String::from("sqlite3"), + OldPackage { + version: UrlOrString::Url( + Url::parse( + "github:mapbox/node-sqlite3#593c9d498be2510d286349134537e3bf89401c4a", + ) + .unwrap(), + ), + bundled: false, + resolved: None, + integrity: None, + dependencies: None, + }, + ); + o + }; + + let initial_url = get_initial_url()?; + + let new = to_new_packages(old, &initial_url)?; + + assert_eq!(new.len(), 1, "new packages map should contain 1 value"); + assert_eq!(new[0], Package { + name: Some(String::from("sqlite3")), + resolved: Some(UrlOrString::Url(Url::parse("git+ssh://git@github.com/mapbox/node-sqlite3.git#593c9d498be2510d286349134537e3bf89401c4a").unwrap())), + integrity: None + }); + + Ok(()) + } +} diff --git a/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs b/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs new file mode 100644 index 0000000000000..95f876017bdd5 --- /dev/null +++ b/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs @@ -0,0 +1,311 @@ +use anyhow::{anyhow, bail, Context}; +use lock::UrlOrString; +use rayon::prelude::*; +use std::{ + fs, io, + process::{Command, Stdio}, +}; +use tempfile::{tempdir, TempDir}; +use url::Url; + +mod lock; + +pub fn lockfile(lockfile: &str) -> anyhow::Result<Vec<Package>> { + let packages = lock::packages(lockfile).context("failed to extract packages from lockfile")?; + + packages + .into_par_iter() + .map(|p| { + let n = p.name.clone().unwrap(); + + Package::from_lock(p).with_context(|| format!("failed to parse data for {n}")) + }) + .collect() +} + +#[derive(Debug)] +pub struct Package { + pub name: String, + pub url: Url, + specifics: Specifics, +} + +#[derive(Debug)] +enum Specifics { + Registry { integrity: String }, + Git { workdir: TempDir }, +} + +impl Package { + fn from_lock(pkg: lock::Package) -> anyhow::Result<Package> { + let mut resolved = match pkg + .resolved + .expect("at this point, packages should have URLs") + { + UrlOrString::Url(u) => u, + UrlOrString::String(_) => panic!("at this point, all packages should have URLs"), + }; + + let specifics = match get_hosted_git_url(&resolved) { + Some(hosted) => { + let mut body = ureq::get(hosted.as_str()).call()?.into_reader(); + + let workdir = tempdir()?; + + let tar_path = workdir.path().join("package"); + + fs::create_dir(&tar_path)?; + + let mut cmd = Command::new("tar") + .args(["--extract", "--gzip", "--strip-components=1", "-C"]) + .arg(&tar_path) + .stdin(Stdio::piped()) + .spawn()?; + + io::copy(&mut body, &mut cmd.stdin.take().unwrap())?; + + let exit = cmd.wait()?; + + if !exit.success() { + bail!( + "failed to extract tarball for {}: tar exited with status code {}", + pkg.name.unwrap(), + exit.code().unwrap() + ); + } + + resolved = hosted; + + Specifics::Git { workdir } + } + None => Specifics::Registry { + integrity: get_ideal_hash( + &pkg.integrity + .expect("non-git dependencies should have assosciated integrity"), + )? + .to_string(), + }, + }; + + Ok(Package { + name: pkg.name.unwrap(), + url: resolved, + specifics, + }) + } + + pub fn tarball(&self) -> anyhow::Result<Vec<u8>> { + match &self.specifics { + Specifics::Registry { .. } => { + let mut body = Vec::new(); + + ureq::get(self.url.as_str()) + .call()? + .into_reader() + .read_to_end(&mut body)?; + + Ok(body) + } + Specifics::Git { workdir } => Ok(Command::new("tar") + .args([ + "--sort=name", + "--mtime=@0", + "--owner=0", + "--group=0", + "--numeric-owner", + "--format=gnu", + "-I", + "gzip -n -9", + "--create", + "-C", + ]) + .arg(workdir.path()) + .arg("package") + .output()? + .stdout), + } + } + + pub fn integrity(&self) -> Option<String> { + match &self.specifics { + Specifics::Registry { integrity } => Some(integrity.clone()), + Specifics::Git { .. } => None, + } + } +} + +#[allow(clippy::case_sensitive_file_extension_comparisons)] +fn get_hosted_git_url(url: &Url) -> Option<Url> { + if ["git", "http", "git+ssh", "git+https", "ssh", "https"].contains(&url.scheme()) { + let mut s = url.path_segments()?; + + match url.host_str()? { + "github.com" => { + let user = s.next()?; + let mut project = s.next()?; + let typ = s.next(); + let mut commit = s.next(); + + if typ.is_none() { + commit = url.fragment(); + } else if typ.is_some() && typ != Some("tree") { + return None; + } + + if project.ends_with(".git") { + project = project.strip_suffix(".git")?; + } + + let commit = commit.unwrap(); + + Some( + Url::parse(&format!( + "https://codeload.github.com/{user}/{project}/tar.gz/{commit}" + )) + .ok()?, + ) + } + "bitbucket.org" => { + let user = s.next()?; + let mut project = s.next()?; + let aux = s.next(); + + if aux == Some("get") { + return None; + } + + if project.ends_with(".git") { + project = project.strip_suffix(".git")?; + } + + let commit = url.fragment()?; + + Some( + Url::parse(&format!( + "https://bitbucket.org/{user}/{project}/get/{commit}.tar.gz" + )) + .ok()?, + ) + } + "gitlab.com" => { + let path = &url.path()[1..]; + + if path.contains("/~/") || path.contains("/archive.tar.gz") { + return None; + } + + let user = s.next()?; + let mut project = s.next()?; + + if project.ends_with(".git") { + project = project.strip_suffix(".git")?; + } + + let commit = url.fragment()?; + + Some( + Url::parse(&format!( + "https://gitlab.com/{user}/{project}/repository/archive.tar.gz?ref={commit}" + )) + .ok()?, + ) + } + "git.sr.ht" => { + let user = s.next()?; + let mut project = s.next()?; + let aux = s.next(); + + if aux == Some("archive") { + return None; + } + + if project.ends_with(".git") { + project = project.strip_suffix(".git")?; + } + + let commit = url.fragment()?; + + Some( + Url::parse(&format!( + "https://git.sr.ht/{user}/{project}/archive/{commit}.tar.gz" + )) + .ok()?, + ) + } + _ => None, + } + } else { + None + } +} + +fn get_ideal_hash(integrity: &str) -> anyhow::Result<&str> { + let split: Vec<_> = integrity.split_ascii_whitespace().collect(); + + if split.len() == 1 { + Ok(split[0]) + } else { + for hash in ["sha512-", "sha1-"] { + if let Some(h) = split.iter().find(|s| s.starts_with(hash)) { + return Ok(h); + } + } + + Err(anyhow!("not sure which hash to select out of {split:?}")) + } +} + +#[cfg(test)] +mod tests { + use super::{get_hosted_git_url, get_ideal_hash}; + use url::Url; + + #[test] + fn hosted_git_urls() { + for (input, expected) in [ + ( + "git+ssh://git@github.com/castlabs/electron-releases.git#fc5f78d046e8d7cdeb66345a2633c383ab41f525", + Some("https://codeload.github.com/castlabs/electron-releases/tar.gz/fc5f78d046e8d7cdeb66345a2633c383ab41f525"), + ), + ( + "https://user@github.com/foo/bar#fix/bug", + Some("https://codeload.github.com/foo/bar/tar.gz/fix/bug") + ), + ( + "https://github.com/eligrey/classList.js/archive/1.2.20180112.tar.gz", + None + ), + ( + "git+ssh://bitbucket.org/foo/bar#branch", + Some("https://bitbucket.org/foo/bar/get/branch.tar.gz") + ), + ( + "ssh://git@gitlab.com/foo/bar.git#fix/bug", + Some("https://gitlab.com/foo/bar/repository/archive.tar.gz?ref=fix/bug") + ), + ( + "git+ssh://git.sr.ht/~foo/bar#branch", + Some("https://git.sr.ht/~foo/bar/archive/branch.tar.gz") + ), + ] { + assert_eq!( + get_hosted_git_url(&Url::parse(input).unwrap()), + expected.map(|u| Url::parse(u).unwrap()) + ); + } + } + + #[test] + fn ideal_hashes() { + for (input, expected) in [ + ("sha512-foo sha1-bar", Some("sha512-foo")), + ("sha1-bar md5-foo", Some("sha1-bar")), + ("sha1-bar", Some("sha1-bar")), + ("sha512-foo", Some("sha512-foo")), + ("foo-bar sha1-bar", Some("sha1-bar")), + ("foo-bar baz-foo", None), + ] { + assert_eq!(get_ideal_hash(input).ok(), expected); + } + } +} diff --git a/pkgs/build-support/node/fetch-npm-deps/src/tests.rs b/pkgs/build-support/node/fetch-npm-deps/src/tests.rs deleted file mode 100644 index a3317207c42e4..0000000000000 --- a/pkgs/build-support/node/fetch-npm-deps/src/tests.rs +++ /dev/null @@ -1,141 +0,0 @@ -use super::{ - fixup_lockfile, get_hosted_git_url, get_ideal_hash, get_initial_url, to_new_packages, - OldPackage, Package, UrlOrString, -}; -use serde_json::json; -use std::collections::HashMap; -use url::Url; - -#[test] -fn hosted_git_urls() { - for (input, expected) in [ - ( - "git+ssh://git@github.com/castlabs/electron-releases.git#fc5f78d046e8d7cdeb66345a2633c383ab41f525", - Some("https://codeload.github.com/castlabs/electron-releases/tar.gz/fc5f78d046e8d7cdeb66345a2633c383ab41f525"), - ), - ( - "https://user@github.com/foo/bar#fix/bug", - Some("https://codeload.github.com/foo/bar/tar.gz/fix/bug") - ), - ( - "https://github.com/eligrey/classList.js/archive/1.2.20180112.tar.gz", - None - ), - ( - "git+ssh://bitbucket.org/foo/bar#branch", - Some("https://bitbucket.org/foo/bar/get/branch.tar.gz") - ), - ( - "ssh://git@gitlab.com/foo/bar.git#fix/bug", - Some("https://gitlab.com/foo/bar/repository/archive.tar.gz?ref=fix/bug") - ), - ( - "git+ssh://git.sr.ht/~foo/bar#branch", - Some("https://git.sr.ht/~foo/bar/archive/branch.tar.gz") - ), - ] { - assert_eq!( - get_hosted_git_url(&Url::parse(input).unwrap()), - expected.map(|u| Url::parse(u).unwrap()) - ); - } -} - -#[test] -fn ideal_hashes() { - for (input, expected) in [ - ("sha512-foo sha1-bar", Some("sha512-foo")), - ("sha1-bar md5-foo", Some("sha1-bar")), - ("sha1-bar", Some("sha1-bar")), - ("sha512-foo", Some("sha512-foo")), - ("foo-bar sha1-bar", Some("sha1-bar")), - ("foo-bar baz-foo", None), - ] { - assert_eq!(get_ideal_hash(input).ok(), expected); - } -} - -#[test] -fn git_shorthand_v1() -> anyhow::Result<()> { - let old = { - let mut o = HashMap::new(); - o.insert( - String::from("sqlite3"), - OldPackage { - version: UrlOrString::Url( - Url::parse( - "github:mapbox/node-sqlite3#593c9d498be2510d286349134537e3bf89401c4a", - ) - .unwrap(), - ), - bundled: false, - resolved: None, - integrity: None, - dependencies: None, - }, - ); - o - }; - - let initial_url = get_initial_url()?; - - let new = to_new_packages(old, &initial_url)?; - - assert_eq!(new.len(), 1, "new packages map should contain 1 value"); - assert_eq!(new.into_values().next().unwrap(), Package { - resolved: Some(UrlOrString::Url(Url::parse("git+ssh://git@github.com/mapbox/node-sqlite3.git#593c9d498be2510d286349134537e3bf89401c4a").unwrap())), - integrity: None - }); - - Ok(()) -} - -#[test] -fn lockfile_fixup() -> anyhow::Result<()> { - let input = json!({ - "lockfileVersion": 2, - "name": "foo", - "packages": { - "": { - - }, - "foo": { - "resolved": "https://github.com/NixOS/nixpkgs", - "integrity": "aaa" - }, - "bar": { - "resolved": "git+ssh://git@github.com/NixOS/nixpkgs.git", - "integrity": "bbb" - } - } - }); - - let expected = json!({ - "lockfileVersion": 2, - "name": "foo", - "packages": { - "": { - - }, - "foo": { - "resolved": "https://github.com/NixOS/nixpkgs", - "integrity": "aaa" - }, - "bar": { - "resolved": "git+ssh://git@github.com/NixOS/nixpkgs.git", - } - } - }); - - assert_eq!( - fixup_lockfile(input.as_object().unwrap().clone())?, - Some(expected.as_object().unwrap().clone()) - ); - - assert_eq!( - fixup_lockfile(json!({"lockfileVersion": 1}).as_object().unwrap().clone())?, - None - ); - - Ok(()) -} |