diff options
Diffstat (limited to 'lib/fileset/default.nix')
| -rw-r--r-- | lib/fileset/default.nix | 590 |
1 files changed, 397 insertions, 193 deletions
diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index 81be1af9d8a1a..2cb361ec9ba19 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -1,3 +1,94 @@ +/* + <!-- This anchor is here for backwards compatibity --> + []{#sec-fileset} + + The [`lib.fileset`](#sec-functions-library-fileset) library allows you to work with _file sets_. + A file set is a (mathematical) set of local files that can be added to the Nix store for use in Nix derivations. + File sets are easy and safe to use, providing obvious and composable semantics with good error messages to prevent mistakes. + + ## Overview {#sec-fileset-overview} + + Basics: + - [Implicit coercion from paths to file sets](#sec-fileset-path-coercion) + + - [`lib.fileset.trace`](#function-library-lib.fileset.trace)/[`lib.fileset.traceVal`](#function-library-lib.fileset.trace): + + Pretty-print file sets for debugging. + + - [`lib.fileset.toSource`](#function-library-lib.fileset.toSource): + + Add files in file sets to the store to use as derivation sources. + + Combinators: + - [`lib.fileset.union`](#function-library-lib.fileset.union)/[`lib.fileset.unions`](#function-library-lib.fileset.unions): + + Create a larger file set from all the files in multiple file sets. + + - [`lib.fileset.intersection`](#function-library-lib.fileset.intersection): + + Create a smaller file set from only the files in both file sets. + + - [`lib.fileset.difference`](#function-library-lib.fileset.difference): + + Create a smaller file set containing all files that are in one file set, but not another one. + + Filtering: + - [`lib.fileset.fileFilter`](#function-library-lib.fileset.fileFilter): + + Create a file set from all files that satisisfy a predicate in a directory. + + Utilities: + - [`lib.fileset.fromSource`](#function-library-lib.fileset.fromSource): + + Create a file set from a `lib.sources`-based value. + + - [`lib.fileset.gitTracked`](#function-library-lib.fileset.gitTracked)/[`lib.fileset.gitTrackedWith`](#function-library-lib.fileset.gitTrackedWith): + + Create a file set from all tracked files in a local Git repository. + + If you need more file set functions, + see [this issue](https://github.com/NixOS/nixpkgs/issues/266356) to request it. + + + ## Implicit coercion from paths to file sets {#sec-fileset-path-coercion} + + All functions accepting file sets as arguments can also accept [paths](https://nixos.org/manual/nix/stable/language/values.html#type-path) as arguments. + Such path arguments are implicitly coerced to file sets containing all files under that path: + - A path to a file turns into a file set containing that single file. + - A path to a directory turns into a file set containing all files _recursively_ in that directory. + + If the path points to a non-existent location, an error is thrown. + + ::: {.note} + Just like in Git, file sets cannot represent empty directories. + Because of this, a path to a directory that contains no files (recursively) will turn into a file set containing no files. + ::: + + :::{.note} + File set coercion does _not_ add any of the files under the coerced paths to the store. + Only the [`toSource`](#function-library-lib.fileset.toSource) function adds files to the Nix store, and only those files contained in the `fileset` argument. + This is in contrast to using [paths in string interpolation](https://nixos.org/manual/nix/stable/language/values.html#type-path), which does add the entire referenced path to the store. + ::: + + ### Example {#sec-fileset-path-coercion-example} + + Assume we are in a local directory with a file hierarchy like this: + ``` + ├─ a/ + │ ├─ x (file) + │ └─ b/ + │ └─ y (file) + └─ c/ + └─ d/ + ``` + + Here's a listing of which files get included when different path expressions get coerced to file sets: + - `./.` as a file set contains both `a/x` and `a/b/y` (`c/` does not contain any files and is therefore omitted). + - `./a` as a file set contains both `a/x` and `a/b/y`. + - `./a/x` as a file set contains only `a/x`. + - `./a/b` as a file set contains only `a/b/y`. + - `./c` as a file set is empty, since neither `c` nor `c/d` contain any files. +*/ { lib }: let @@ -12,14 +103,18 @@ let _printFileset _intersection _difference + _mirrorStorePath + _fetchGitSubmodulesMinver ; inherit (builtins) + isBool isList isPath pathExists seq typeOf + nixVersion ; inherit (lib.lists) @@ -34,6 +129,7 @@ let inherit (lib.strings) isStringLike + versionOlder ; inherit (lib.filesystem) @@ -47,11 +143,102 @@ let inherit (lib.trivial) isFunction pipe + inPureEvalMode ; in { /* + Incrementally evaluate and trace a file set in a pretty way. + This function is only intended for debugging purposes. + The exact tracing format is unspecified and may change. + + This function takes a final argument to return. + In comparison, [`traceVal`](#function-library-lib.fileset.traceVal) returns + the given file set argument. + + This variant is useful for tracing file sets in the Nix repl. + + Type: + trace :: FileSet -> Any -> Any + + Example: + trace (unions [ ./Makefile ./src ./tests/run.sh ]) null + => + trace: /home/user/src/myProject + trace: - Makefile (regular) + trace: - src (all files in directory) + trace: - tests + trace: - run.sh (regular) + null + */ + trace = + /* + The file set to trace. + + This argument can also be a path, + which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). + */ + fileset: + let + # "fileset" would be a better name, but that would clash with the argument name, + # and we cannot change that because of https://github.com/nix-community/nixdoc/issues/76 + actualFileset = _coerce "lib.fileset.trace: Argument" fileset; + in + seq + (_printFileset actualFileset) + (x: x); + + /* + Incrementally evaluate and trace a file set in a pretty way. + This function is only intended for debugging purposes. + The exact tracing format is unspecified and may change. + + This function returns the given file set. + In comparison, [`trace`](#function-library-lib.fileset.trace) takes another argument to return. + + This variant is useful for tracing file sets passed as arguments to other functions. + + Type: + traceVal :: FileSet -> FileSet + + Example: + toSource { + root = ./.; + fileset = traceVal (unions [ + ./Makefile + ./src + ./tests/run.sh + ]); + } + => + trace: /home/user/src/myProject + trace: - Makefile (regular) + trace: - src (all files in directory) + trace: - tests + trace: - run.sh (regular) + "/nix/store/...-source" + */ + traceVal = + /* + The file set to trace and return. + + This argument can also be a path, + which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). + */ + fileset: + let + # "fileset" would be a better name, but that would clash with the argument name, + # and we cannot change that because of https://github.com/nix-community/nixdoc/issues/76 + actualFileset = _coerce "lib.fileset.traceVal: Argument" fileset; + in + seq + (_printFileset actualFileset) + # We could also return the original fileset argument here, + # but that would then duplicate work for consumers of the fileset, because then they have to coerce it again + actualFileset; + + /* Add the local files contained in `fileset` to the store as a single [store path](https://nixos.org/manual/nix/stable/glossary#gloss-store-path) rooted at `root`. The result is the store path as a string-like value, making it usable e.g. as the `src` of a derivation, or in string interpolation: @@ -196,75 +383,6 @@ in { }; /* - Create a file set with the same files as a `lib.sources`-based value. - This does not import any of the files into the store. - - This can be used to gradually migrate from `lib.sources`-based filtering to `lib.fileset`. - - A file set can be turned back into a source using [`toSource`](#function-library-lib.fileset.toSource). - - :::{.note} - File sets cannot represent empty directories. - Turning the result of this function back into a source using `toSource` will therefore not preserve empty directories. - ::: - - Type: - fromSource :: SourceLike -> FileSet - - Example: - # There's no cleanSource-like function for file sets yet, - # but we can just convert cleanSource to a file set and use it that way - toSource { - root = ./.; - fileset = fromSource (lib.sources.cleanSource ./.); - } - - # Keeping a previous sourceByRegex (which could be migrated to `lib.fileset.unions`), - # but removing a subdirectory using file set functions - difference - (fromSource (lib.sources.sourceByRegex ./. [ - "^README\.md$" - # This regex includes everything in ./doc - "^doc(/.*)?$" - ]) - ./doc/generated - - # Use cleanSource, but limit it to only include ./Makefile and files under ./src - intersection - (fromSource (lib.sources.cleanSource ./.)) - (unions [ - ./Makefile - ./src - ]); - */ - fromSource = source: - let - # This function uses `._isLibCleanSourceWith`, `.origSrc` and `.filter`, - # which are technically internal to lib.sources, - # but we'll allow this since both libraries are in the same code base - # and this function is a bridge between them. - isFiltered = source ? _isLibCleanSourceWith; - path = if isFiltered then source.origSrc else source; - in - # We can only support sources created from paths - if ! isPath path then - if isStringLike path then - throw '' - lib.fileset.fromSource: The source origin of the argument is a string-like value ("${toString path}"), but it should be a path instead. - Sources created from paths in strings cannot be turned into file sets, use `lib.sources` or derivations instead.'' - else - throw '' - lib.fileset.fromSource: The source origin of the argument is of type ${typeOf path}, but it should be a path instead.'' - else if ! pathExists path then - throw '' - lib.fileset.fromSource: The source origin (${toString path}) of the argument does not exist.'' - else if isFiltered then - _fromSourceFilter path source.filter - else - # If there's no filter, no need to run the expensive conversion, all subpaths will be included - _singleton path; - - /* The file set containing all files that are in either of two given file sets. This is the same as [`unions`](#function-library-lib.fileset.unions), but takes just two file sets instead of a list. @@ -357,66 +475,6 @@ in { ]; /* - Filter a file set to only contain files matching some predicate. - - Type: - fileFilter :: - ({ - name :: String, - type :: String, - ... - } -> Bool) - -> Path - -> FileSet - - Example: - # Include all regular `default.nix` files in the current directory - fileFilter (file: file.name == "default.nix") ./. - - # Include all non-Nix files from the current directory - fileFilter (file: ! hasSuffix ".nix" file.name) ./. - - # Include all files that start with a "." in the current directory - fileFilter (file: hasPrefix "." file.name) ./. - - # Include all regular files (not symlinks or others) in the current directory - fileFilter (file: file.type == "regular") ./. - */ - fileFilter = - /* - The predicate function to call on all files contained in given file set. - A file is included in the resulting file set if this function returns true for it. - - This function is called with an attribute set containing these attributes: - - - `name` (String): The name of the file - - - `type` (String, one of `"regular"`, `"symlink"` or `"unknown"`): The type of the file. - This matches result of calling [`builtins.readFileType`](https://nixos.org/manual/nix/stable/language/builtins.html#builtins-readFileType) on the file's path. - - Other attributes may be added in the future. - */ - predicate: - # The path whose files to filter - path: - if ! isFunction predicate then - throw '' - lib.fileset.fileFilter: First argument is of type ${typeOf predicate}, but it should be a function instead.'' - else if ! isPath path then - if path._type or "" == "fileset" then - throw '' - lib.fileset.fileFilter: Second argument is a file set, but it should be a path instead. - If you need to filter files in a file set, use `intersection fileset (fileFilter pred ./.)` instead.'' - else - throw '' - lib.fileset.fileFilter: Second argument is of type ${typeOf path}, but it should be a path instead.'' - else if ! pathExists path then - throw '' - lib.fileset.fileFilter: Second argument (${toString path}) is a path that does not exist.'' - else - _fileFilter predicate path; - - /* The file set containing all files that are in both of two given file sets. See also [Intersection (set theory)](https://en.wikipedia.org/wiki/Intersection_(set_theory)). @@ -508,92 +566,238 @@ in { (elemAt filesets 1); /* - Incrementally evaluate and trace a file set in a pretty way. - This function is only intended for debugging purposes. - The exact tracing format is unspecified and may change. + Filter a file set to only contain files matching some predicate. - This function takes a final argument to return. - In comparison, [`traceVal`](#function-library-lib.fileset.traceVal) returns - the given file set argument. + Type: + fileFilter :: + ({ + name :: String, + type :: String, + ... + } -> Bool) + -> Path + -> FileSet - This variant is useful for tracing file sets in the Nix repl. + Example: + # Include all regular `default.nix` files in the current directory + fileFilter (file: file.name == "default.nix") ./. + + # Include all non-Nix files from the current directory + fileFilter (file: ! hasSuffix ".nix" file.name) ./. + + # Include all files that start with a "." in the current directory + fileFilter (file: hasPrefix "." file.name) ./. + + # Include all regular files (not symlinks or others) in the current directory + fileFilter (file: file.type == "regular") ./. + */ + fileFilter = + /* + The predicate function to call on all files contained in given file set. + A file is included in the resulting file set if this function returns true for it. + + This function is called with an attribute set containing these attributes: + + - `name` (String): The name of the file + + - `type` (String, one of `"regular"`, `"symlink"` or `"unknown"`): The type of the file. + This matches result of calling [`builtins.readFileType`](https://nixos.org/manual/nix/stable/language/builtins.html#builtins-readFileType) on the file's path. + + Other attributes may be added in the future. + */ + predicate: + # The path whose files to filter + path: + if ! isFunction predicate then + throw '' + lib.fileset.fileFilter: First argument is of type ${typeOf predicate}, but it should be a function instead.'' + else if ! isPath path then + if path._type or "" == "fileset" then + throw '' + lib.fileset.fileFilter: Second argument is a file set, but it should be a path instead. + If you need to filter files in a file set, use `intersection fileset (fileFilter pred ./.)` instead.'' + else + throw '' + lib.fileset.fileFilter: Second argument is of type ${typeOf path}, but it should be a path instead.'' + else if ! pathExists path then + throw '' + lib.fileset.fileFilter: Second argument (${toString path}) is a path that does not exist.'' + else + _fileFilter predicate path; + + /* + Create a file set with the same files as a `lib.sources`-based value. + This does not import any of the files into the store. + + This can be used to gradually migrate from `lib.sources`-based filtering to `lib.fileset`. + + A file set can be turned back into a source using [`toSource`](#function-library-lib.fileset.toSource). + + :::{.note} + File sets cannot represent empty directories. + Turning the result of this function back into a source using `toSource` will therefore not preserve empty directories. + ::: + + Type: + fromSource :: SourceLike -> FileSet + + Example: + # There's no cleanSource-like function for file sets yet, + # but we can just convert cleanSource to a file set and use it that way + toSource { + root = ./.; + fileset = fromSource (lib.sources.cleanSource ./.); + } + + # Keeping a previous sourceByRegex (which could be migrated to `lib.fileset.unions`), + # but removing a subdirectory using file set functions + difference + (fromSource (lib.sources.sourceByRegex ./. [ + "^README\.md$" + # This regex includes everything in ./doc + "^doc(/.*)?$" + ]) + ./doc/generated + + # Use cleanSource, but limit it to only include ./Makefile and files under ./src + intersection + (fromSource (lib.sources.cleanSource ./.)) + (unions [ + ./Makefile + ./src + ]); + */ + fromSource = source: + let + # This function uses `._isLibCleanSourceWith`, `.origSrc` and `.filter`, + # which are technically internal to lib.sources, + # but we'll allow this since both libraries are in the same code base + # and this function is a bridge between them. + isFiltered = source ? _isLibCleanSourceWith; + path = if isFiltered then source.origSrc else source; + in + # We can only support sources created from paths + if ! isPath path then + if isStringLike path then + throw '' + lib.fileset.fromSource: The source origin of the argument is a string-like value ("${toString path}"), but it should be a path instead. + Sources created from paths in strings cannot be turned into file sets, use `lib.sources` or derivations instead.'' + else + throw '' + lib.fileset.fromSource: The source origin of the argument is of type ${typeOf path}, but it should be a path instead.'' + else if ! pathExists path then + throw '' + lib.fileset.fromSource: The source origin (${toString path}) of the argument is a path that does not exist.'' + else if isFiltered then + _fromSourceFilter path source.filter + else + # If there's no filter, no need to run the expensive conversion, all subpaths will be included + _singleton path; + + /* + Create a file set containing all [Git-tracked files](https://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository) in a repository. + + This function behaves like [`gitTrackedWith { }`](#function-library-lib.fileset.gitTrackedWith) - using the defaults. Type: - trace :: FileSet -> Any -> Any + gitTracked :: Path -> FileSet Example: - trace (unions [ ./Makefile ./src ./tests/run.sh ]) null - => - trace: /home/user/src/myProject - trace: - Makefile (regular) - trace: - src (all files in directory) - trace: - tests - trace: - run.sh (regular) - null + # Include all files tracked by the Git repository in the current directory + gitTracked ./. + + # Include only files tracked by the Git repository in the parent directory + # that are also in the current directory + intersection ./. (gitTracked ../.) */ - trace = + gitTracked = /* - The file set to trace. - - This argument can also be a path, - which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). + The [path](https://nixos.org/manual/nix/stable/language/values#type-path) to the working directory of a local Git repository. + This directory must contain a `.git` file or subdirectory. */ - fileset: + path: + # See the gitTrackedWith implementation for more explanatory comments let - # "fileset" would be a better name, but that would clash with the argument name, - # and we cannot change that because of https://github.com/nix-community/nixdoc/issues/76 - actualFileset = _coerce "lib.fileset.trace: Argument" fileset; + fetchResult = builtins.fetchGit path; in - seq - (_printFileset actualFileset) - (x: x); + if inPureEvalMode then + throw "lib.fileset.gitTracked: This function is currently not supported in pure evaluation mode, since it currently relies on `builtins.fetchGit`. See https://github.com/NixOS/nix/issues/9292." + else if ! isPath path then + throw "lib.fileset.gitTracked: Expected the argument to be a path, but it's a ${typeOf path} instead." + else if ! pathExists (path + "/.git") then + throw "lib.fileset.gitTracked: Expected the argument (${toString path}) to point to a local working tree of a Git repository, but it's not." + else + _mirrorStorePath path fetchResult.outPath; /* - Incrementally evaluate and trace a file set in a pretty way. - This function is only intended for debugging purposes. - The exact tracing format is unspecified and may change. + Create a file set containing all [Git-tracked files](https://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository) in a repository. + The first argument allows configuration with an attribute set, + while the second argument is the path to the Git working tree. + If you don't need the configuration, + you can use [`gitTracked`](#function-library-lib.fileset.gitTracked) instead. - This function returns the given file set. - In comparison, [`trace`](#function-library-lib.fileset.trace) takes another argument to return. + This is equivalent to the result of [`unions`](#function-library-lib.fileset.unions) on all files returned by [`git ls-files`](https://git-scm.com/docs/git-ls-files) + (which uses [`--cached`](https://git-scm.com/docs/git-ls-files#Documentation/git-ls-files.txt--c) by default). - This variant is useful for tracing file sets passed as arguments to other functions. + :::{.warning} + Currently this function is based on [`builtins.fetchGit`](https://nixos.org/manual/nix/stable/language/builtins.html#builtins-fetchGit) + As such, this function causes all Git-tracked files to be unnecessarily added to the Nix store, + without being re-usable by [`toSource`](#function-library-lib.fileset.toSource). + + This may change in the future. + ::: Type: - traceVal :: FileSet -> FileSet + gitTrackedWith :: { recurseSubmodules :: Bool ? false } -> Path -> FileSet Example: - toSource { - root = ./.; - fileset = traceVal (unions [ - ./Makefile - ./src - ./tests/run.sh - ]); - } - => - trace: /home/user/src/myProject - trace: - Makefile (regular) - trace: - src (all files in directory) - trace: - tests - trace: - run.sh (regular) - "/nix/store/...-source" + # Include all files tracked by the Git repository in the current directory + # and any submodules under it + gitTracked { recurseSubmodules = true; } ./. */ - traceVal = + gitTrackedWith = + { + /* + (optional, default: `false`) Whether to recurse into [Git submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) to also include their tracked files. + + If `true`, this is equivalent to passing the [--recurse-submodules](https://git-scm.com/docs/git-ls-files#Documentation/git-ls-files.txt---recurse-submodules) flag to `git ls-files`. + */ + recurseSubmodules ? false, + }: /* - The file set to trace and return. - - This argument can also be a path, - which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). + The [path](https://nixos.org/manual/nix/stable/language/values#type-path) to the working directory of a local Git repository. + This directory must contain a `.git` file or subdirectory. */ - fileset: + path: let - # "fileset" would be a better name, but that would clash with the argument name, - # and we cannot change that because of https://github.com/nix-community/nixdoc/issues/76 - actualFileset = _coerce "lib.fileset.traceVal: Argument" fileset; + # This imports the files unnecessarily, which currently can't be avoided + # because `builtins.fetchGit` is the only function exposing which files are tracked by Git. + # With the [lazy trees PR](https://github.com/NixOS/nix/pull/6530), + # the unnecessarily import could be avoided. + # However a simpler alternative still would be [a builtins.gitLsFiles](https://github.com/NixOS/nix/issues/2944). + fetchResult = builtins.fetchGit { + url = path; + + # This is the only `fetchGit` parameter that makes sense in this context. + # We can't just pass `submodules = recurseSubmodules` here because + # this would fail for Nix versions that don't support `submodules`. + ${if recurseSubmodules then "submodules" else null} = true; + }; in - seq - (_printFileset actualFileset) - # We could also return the original fileset argument here, - # but that would then duplicate work for consumers of the fileset, because then they have to coerce it again - actualFileset; + if inPureEvalMode then + throw "lib.fileset.gitTrackedWith: This function is currently not supported in pure evaluation mode, since it currently relies on `builtins.fetchGit`. See https://github.com/NixOS/nix/issues/9292." + else if ! isBool recurseSubmodules then + throw "lib.fileset.gitTrackedWith: Expected the attribute `recurseSubmodules` of the first argument to be a boolean, but it's a ${typeOf recurseSubmodules} instead." + else if recurseSubmodules && versionOlder nixVersion _fetchGitSubmodulesMinver then + throw "lib.fileset.gitTrackedWith: Setting the attribute `recurseSubmodules` to `true` is only supported for Nix version ${_fetchGitSubmodulesMinver} and after, but Nix version ${nixVersion} is used." + else if ! isPath path then + throw "lib.fileset.gitTrackedWith: Expected the second argument to be a path, but it's a ${typeOf path} instead." + # We can identify local working directories by checking for .git, + # see https://git-scm.com/docs/gitrepository-layout#_description. + # Note that `builtins.fetchGit` _does_ work for bare repositories (where there's no `.git`), + # even though `git ls-files` wouldn't return any files in that case. + else if ! pathExists (path + "/.git") then + throw "lib.fileset.gitTrackedWith: Expected the second argument (${toString path}) to point to a local working tree of a Git repository, but it's not." + else + _mirrorStorePath path fetchResult.outPath; } |