diff options
author | Robert Hensing <roberth@users.noreply.github.com> | 2021-06-28 14:02:37 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-06-28 14:02:37 +0200 |
commit | 4c4c00e9f1373dee2dd294442b7bd1cde40bec65 (patch) | |
tree | 642eb074235b122851b3ab07bad1bb19230907f1 /lib | |
parent | 124bbaf311076d6e6b79bc2093fa4726a388cc16 (diff) | |
parent | 4cf56e56405a226d8b21e8643e32a10b60930ae9 (diff) |
Merge pull request #124875 from hercules-ci/lib-sources
lib.sources: docs, tests, refactoring
Diffstat (limited to 'lib')
-rw-r--r-- | lib/sources.nix | 117 | ||||
-rw-r--r-- | lib/tests/release.nix | 4 | ||||
-rwxr-xr-x | lib/tests/sources.sh | 59 |
3 files changed, 163 insertions, 17 deletions
diff --git a/lib/sources.nix b/lib/sources.nix index 1a821f55056bd..407829b547b08 100644 --- a/lib/sources.nix +++ b/lib/sources.nix @@ -1,6 +1,7 @@ # Functions for copying sources to the Nix store. { lib }: +# Tested in lib/tests/sources.sh let inherit (builtins) hasContext @@ -11,14 +12,13 @@ let tryEval ; inherit (lib) + boolToString filter getAttr isString pathExists readFile ; -in -rec { # Returns the type of a path: regular (for file), symlink, or directory pathType = p: getAttr (baseNameOf p) (readDir (dirOf p)); @@ -84,18 +84,36 @@ rec { # cleanSourceWith = { filter ? _path: _type: true, src, name ? null }: let - isFiltered = src ? _isLibCleanSourceWith; - origSrc = if isFiltered then src.origSrc else src; - filter' = if isFiltered then name: type: filter name type && src.filter name type else filter; - name' = if name != null then name else if isFiltered then src.name else "source"; - in { - inherit origSrc; - filter = filter'; - outPath = builtins.path { filter = filter'; path = origSrc; name = name'; }; - _isLibCleanSourceWith = true; - name = name'; + orig = toSourceAttributes src; + in fromSourceAttributes { + inherit (orig) origSrc; + filter = path: type: filter path type && orig.filter path type; + name = if name != null then name else orig.name; }; + /* + Add logging to a source, for troubleshooting the filtering behavior. + Type: + sources.trace :: sourceLike -> Source + */ + trace = + # Source to debug. The returned source will behave like this source, but also log its filter invocations. + src: + let + attrs = toSourceAttributes src; + in + fromSourceAttributes ( + attrs // { + filter = path: type: + let + r = attrs.filter path type; + in + builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r; + } + ) // { + satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant; + }; + # Filter sources by a list of regular expressions. # # E.g. `src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"]` @@ -110,14 +128,26 @@ rec { inherit src; }; - # Get all files ending with the specified suffices from the given - # directory or its descendants. E.g. `sourceFilesBySuffices ./dir - # [".xml" ".c"]'. - sourceFilesBySuffices = path: exts: + /* + Get all files ending with the specified suffices from the given + source directory or its descendants, omitting files that do not match + any suffix. The result of the example below will include files like + `./dir/module.c` and `./dir/subdir/doc.xml` if present. + + Type: sourceLike -> [String] -> Source + + Example: + sourceFilesBySuffices ./. [ ".xml" ".c" ] + */ + sourceFilesBySuffices = + # Path or source containing the files to be returned + src: + # A list of file suffix strings + exts: let filter = name: type: let base = baseNameOf (toString name); in type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts; - in cleanSourceWith { inherit filter; src = path; }; + in cleanSourceWith { inherit filter src; }; pathIsGitRepo = path: (tryEval (commitIdFromGitRepo path)).success; @@ -177,4 +207,57 @@ rec { pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir); canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src)); + + # -------------------------------------------------------------------------- # + # Internal functions + # + + # toSourceAttributes : sourceLike -> SourceAttrs + # + # Convert any source-like object into a simple, singular representation. + # We don't expose this representation in order to avoid having a fifth path- + # like class of objects in the wild. + # (Existing ones being: paths, strings, sources and x//{outPath}) + # So instead of exposing internals, we build a library of combinator functions. + toSourceAttributes = src: + let + isFiltered = src ? _isLibCleanSourceWith; + in + { + # The original path + origSrc = if isFiltered then src.origSrc else src; + filter = if isFiltered then src.filter else _: _: true; + name = if isFiltered then src.name else "source"; + }; + + # fromSourceAttributes : SourceAttrs -> Source + # + # Inverse of toSourceAttributes for Source objects. + fromSourceAttributes = { origSrc, filter, name }: + { + _isLibCleanSourceWith = true; + inherit origSrc filter name; + outPath = builtins.path { inherit filter name; path = origSrc; }; + }; + +in { + inherit + pathType + pathIsDirectory + pathIsRegularFile + + pathIsGitRepo + commitIdFromGitRepo + + cleanSource + cleanSourceWith + cleanSourceFilter + pathHasContext + canCleanSource + + sourceByRegex + sourceFilesBySuffices + + trace + ; } diff --git a/lib/tests/release.nix b/lib/tests/release.nix index 800d8a65c14f9..c3b05251f709d 100644 --- a/lib/tests/release.nix +++ b/lib/tests/release.nix @@ -26,7 +26,11 @@ pkgs.runCommandNoCC "nixpkgs-lib-tests" { nix-store --init cp -r ${../.} lib + echo "Running lib/tests/modules.sh" bash lib/tests/modules.sh + echo "Running lib/tests/sources.sh" + TEST_LIB=$PWD/lib bash lib/tests/sources.sh + touch $out '' diff --git a/lib/tests/sources.sh b/lib/tests/sources.sh new file mode 100755 index 0000000000000..71fee719cb21e --- /dev/null +++ b/lib/tests/sources.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Use +# || die +die() { + echo >&2 "test case failed: " "$@" + exit 1 +} + +if test -n "${TEST_LIB:-}"; then + export NIX_PATH=nixpkgs="$(dirname "$TEST_LIB")" +else + export NIX_PATH=nixpkgs="$(cd $(dirname ${BASH_SOURCE[0]})/../..; pwd)" +fi + +work="$(mktemp -d)" +clean_up() { + rm -rf "$work" +} +trap clean_up EXIT +cd $work + +touch {README.md,module.o,foo.bar} + +# nix-instantiate doesn't write out the source, only computing the hash, so +# this uses the experimental nix command instead. + +dir="$(nix eval --raw '(with import <nixpkgs/lib>; "${ + cleanSource ./. +}")')" +(cd $dir; find) | sort -f | diff -U10 - <(cat <<EOF +. +./foo.bar +./README.md +EOF +) || die "cleanSource 1" + + +dir="$(nix eval --raw '(with import <nixpkgs/lib>; "${ + cleanSourceWith { src = '"$work"'; filter = path: type: ! hasSuffix ".bar" path; } +}")')" +(cd $dir; find) | sort -f | diff -U10 - <(cat <<EOF +. +./module.o +./README.md +EOF +) || die "cleanSourceWith 1" + +dir="$(nix eval --raw '(with import <nixpkgs/lib>; "${ + cleanSourceWith { src = cleanSource '"$work"'; filter = path: type: ! hasSuffix ".bar" path; } +}")')" +(cd $dir; find) | sort -f | diff -U10 - <(cat <<EOF +. +./README.md +EOF +) || die "cleanSourceWith + cleanSource" + +echo >&2 tests ok |