diff options
author | Travis A. Everett <travis.a.everett@gmail.com> | 2021-01-05 10:56:59 -0600 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-01-05 11:56:59 -0500 |
commit | 6fd9283bbac720946396f1d3598f92b5ea87e11d (patch) | |
tree | a699e18014df5f6987924753657f382d5d042e8d /pkgs/development/misc | |
parent | 645f39f33ed785912cd452325fd9cb3a28e3d4ac (diff) |
resholve: init at 0.4.0 (#85827)
resholve: init at 0.4.0 resholve attempts to resolve executables in shell scripts. Includes Nix builder for resolving dependencies in Nix-built shell projects.
Diffstat (limited to 'pkgs/development/misc')
-rw-r--r-- | pkgs/development/misc/resholve/README.md | 138 | ||||
-rw-r--r-- | pkgs/development/misc/resholve/default.nix | 9 | ||||
-rw-r--r-- | pkgs/development/misc/resholve/deps.nix | 120 | ||||
-rw-r--r-- | pkgs/development/misc/resholve/resholve-package.nix | 97 | ||||
-rw-r--r-- | pkgs/development/misc/resholve/resholve.nix | 74 |
5 files changed, 438 insertions, 0 deletions
diff --git a/pkgs/development/misc/resholve/README.md b/pkgs/development/misc/resholve/README.md new file mode 100644 index 0000000000000..ddba7fc149340 --- /dev/null +++ b/pkgs/development/misc/resholve/README.md @@ -0,0 +1,138 @@ +# Using resholve's Nix API + +resholve converts bare executable references in shell scripts to absolute +paths. This will hopefully make its way into the Nixpkgs manual soon, but +until then I'll outline how to use the `resholvePackage` function. + +> Fair warning: resholve does *not* aspire to resolving all valid Shell +> scripts. It depends on the OSH/Oil parser, which aims to support most (but +> not all) Bash, and aims to be a ~90% sort of solution. + +Let's start with a simple example from one of my own projects: + +```nix +{ stdenv, lib, resholvePackage, fetchFromGitHub, bashup-events44, bashInteractive_5, doCheck ? true, shellcheck }: + +resholvePackage rec { + pname = "shellswain"; + version = "unreleased"; + + src = fetchFromGitHub { + # ... + }; + + solutions = { + profile = { + # the only *required* arguments + scripts = [ "bin/shellswain.bash" ]; + interpreter = "none"; + inputs = [ bashup-events44 ]; + }; + }; + + makeFlags = [ "prefix=${placeholder "out"}" ]; + + inherit doCheck; + checkInputs = [ shellcheck ]; + + # ... +} +``` + +I'll focus on the `solutions` attribute, since this is the only part +that differs from other derivations. + +Each "solution" (k=v pair) +describes one resholve invocation. For most shell packages, one +invocation will probably be enough. resholve will make you be very +explicit about your script's dependencies, and it may also need your +help sorting out some references or problems that it can't safely +handle on its own. + +If you have more than one script, and your scripts need conflicting +directives, you can specify more than one solution to resolve the +scripts separately, but still produce a single package. + +Let's take a closer look: + +```nix + solutions = { + # each solution has a short name; this is what you'd use to + # override the settings of this solution, and it may also show up + # in (some) error messages. + profile = { + # specify one or more $out-relative script paths (unlike many + # builders, resholve will modify the output files during fixup + # to correctly resolve scripts that source within the package) + scripts = [ "bin/shellswain.bash" ]; + # "none" for no shebang, "${bash}/bin/bash" for bash, etc. + interpreter = "none"; + # packages resholve should resolve executables from + inputs = [ bashup-events44 ]; + }; + }; +``` + +resholve has a (growing) number of options for handling more complex +scripts. I won't cover these in excruciating detail here. You can find +more information about these in `man resholve` via `nixpkgs.resholve`. + +Instead, we'll look at the general form of the solutions attrset: + +```nix +solutions = { + shortname = { + # required + # $out-relative paths to try resolving + scripts = [ "bin/shunit2" ]; + # packages to resolve executables from + inputs = [ coreutils gnused gnugrep findutils ]; + # path for shebang, or 'none' to omit shebang + interpreter = "${bash}/bin/bash"; + + # optional + fake = { fake directives }; + fix = { fix directives }; + keep = { keep directives }; + # file to inject before first code-line of script + prologue = file; + # file to inject after last code-line of script + epilogue = file; + # extra command-line flags passed to resholve; generally this API + # should align with what resholve supports, but flags may help if + # you need to override the version of resholve. + flags = [ ]; + }; +}; +``` + +The main way you'll adjust how resholve handles your scripts are the +fake, fix, and keep directives. The manpage covers their purpose and +how to format them on the command-line, so I'll focus on how you'll +need to translate them into Nix types. + +```nix +# --fake 'f:setUp;tearDown builtin:setopt source:/etc/bashrc' +fake = { + function = [ "setUp" "tearDown" ]; + builtin = [ "setopt" ]; + source = [ "/etc/bashrc" ]; +}; + +# --fix 'aliases xargs:ls $GIT:gix' +fix = { + # all single-word directives use `true` as value + aliases = true; + xargs = [ "ls" ]; + "$GIT" = [ "gix" ]; +}; + +# --keep 'which:git;ls .:$HOME $LS:exa /etc/bashrc ~/.bashrc' +keep = { + which = [ "git" "ls" ]; + "." = [ "$HOME" ]; + "$LS" = [ "exa" ]; + "/etc/bashrc" = true; + "~/.bashrc" = true; +}; +``` diff --git a/pkgs/development/misc/resholve/default.nix b/pkgs/development/misc/resholve/default.nix new file mode 100644 index 0000000000000..7b5a79dd221a1 --- /dev/null +++ b/pkgs/development/misc/resholve/default.nix @@ -0,0 +1,9 @@ +{ callPackage +, doCheck ? true +}: + +rec { + resholve = callPackage ./resholve.nix { inherit doCheck; }; + resholvePackage = + callPackage ./resholve-package.nix { inherit resholve; }; +} diff --git a/pkgs/development/misc/resholve/deps.nix b/pkgs/development/misc/resholve/deps.nix new file mode 100644 index 0000000000000..9be283e493356 --- /dev/null +++ b/pkgs/development/misc/resholve/deps.nix @@ -0,0 +1,120 @@ +{ stdenv +, python27Packages +, fetchFromGitHub +, makeWrapper +, # re2c deps + autoreconfHook +, # py-yajl deps + git +, # oil deps + readline +, cmark +, file +, glibcLocales +, oilPatches ? [ ] +}: + +/* +Notes on specific dependencies: +- if/when python2.7 is removed from nixpkgs, this may need to figure + out how to build oil's vendored python2 +- I'm not sure if glibcLocales is worth the addition here. It's to fix + a libc test oil runs. My oil fork just disabled the libc tests, but + I haven't quite decided if that's the right long-term call, so I + didn't add a patch for it here yet. +*/ + +rec { + # had to add this as well; 1.3 causes a break here; sticking + # to oil's official 1.0.3 dep for now. + re2c = stdenv.mkDerivation rec { + pname = "re2c"; + version = "1.0.3"; + sourceRoot = "${src.name}/re2c"; + src = fetchFromGitHub { + owner = "skvadrik"; + repo = "re2c"; + rev = version; + sha256 = "0grx7nl9fwcn880v5ssjljhcb9c5p2a6xpwil7zxpmv0rwnr3yqi"; + }; + nativeBuildInputs = [ autoreconfHook ]; + preCheck = '' + patchShebangs run_tests.sh + ''; + }; + + py-yajl = python27Packages.buildPythonPackage rec { + pname = "oil-pyyajl-unstable"; + version = "2019-12-05"; + src = fetchFromGitHub { + owner = "oilshell"; + repo = "py-yajl"; + rev = "eb561e9aea6e88095d66abcc3990f2ee1f5339df"; + sha256 = "17hcgb7r7cy8r1pwbdh8di0nvykdswlqj73c85k6z8m0filj3hbh"; + fetchSubmodules = true; + }; + # just for submodule IIRC + nativeBuildInputs = [ git ]; + }; + + # resholve's primary dependency is this developer build of the oil shell. + oildev = python27Packages.buildPythonPackage rec { + pname = "oildev-unstable"; + version = "2020-03-31"; + + src = fetchFromGitHub { + owner = "oilshell"; + repo = "oil"; + rev = "ea80cdad7ae1152a25bd2a30b87fe3c2ad32394a"; + sha256 = "0pxn0f8qbdman4gppx93zwml7s5byqfw560n079v68qjgzh2brq2"; + + /* + It's not critical to drop most of these; the primary target is + the vendored fork of Python-2.7.13, which is ~ 55M and over 3200 + files, dozens of which get interpreter script patches in fixup. + */ + extraPostFetch = '' + rm -rf Python-2.7.13 benchmarks metrics py-yajl rfc gold web testdata services demo devtools cpp + ''; + }; + + # TODO: not sure why I'm having to set this for nix-build... + # can anyone tell if I'm doing something wrong? + SOURCE_DATE_EPOCH = 315532800; + + # These aren't, strictly speaking, nix/nixpkgs specific, but I've + # had hell upstreaming them. Pulling from resholve source and + # passing in from resholve.nix + patches = oilPatches; + + buildInputs = [ readline cmark py-yajl ]; + + nativeBuildInputs = [ re2c file makeWrapper ]; + + propagatedBuildInputs = with python27Packages; [ six typing ]; + + doCheck = true; + + preBuild = '' + build/dev.sh all + ''; + + postPatch = '' + patchShebangs asdl build core doctools frontend native oil_lang + ''; + + _NIX_SHELL_LIBCMARK = "${cmark}/lib/libcmark${stdenv.hostPlatform.extensions.sharedLibrary}"; + + # See earlier note on glibcLocales + LOCALE_ARCHIVE = stdenv.lib.optionalString (stdenv.buildPlatform.libc == "glibc") "${glibcLocales}/lib/locale/locale-archive"; + + meta = { + description = "A new unix shell"; + homepage = "https://www.oilshell.org/"; + license = with stdenv.lib.licenses; [ + psfl # Includes a portion of the python interpreter and standard library + asl20 # Licence for Oil itself + ]; + }; + }; +} diff --git a/pkgs/development/misc/resholve/resholve-package.nix b/pkgs/development/misc/resholve/resholve-package.nix new file mode 100644 index 0000000000000..cc971196a4f86 --- /dev/null +++ b/pkgs/development/misc/resholve/resholve-package.nix @@ -0,0 +1,97 @@ +{ stdenv, lib, resholve }: + +{ pname +, src +, version +, passthru ? { } +, solutions +, ... +}@attrs: +let + inherit stdenv; + /* These functions break up the work of partially validating the + * 'solutions' attrset and massaging it into env/cli args. + * + * Note: some of the left-most args do not *have* to be passed as + * deep as they are, but I've done so to provide more error context + */ + + # for brevity / line length + spaces = l: builtins.concatStringsSep " " l; + semicolons = l: builtins.concatStringsSep ";" l; + + /* Throw a fit with dotted attr path context */ + nope = path: msg: + throw "${builtins.concatStringsSep "." path}: ${msg}"; + + /* Special-case directive value representations by type */ + makeDirective = solution: env: name: val: + if builtins.isInt val then builtins.toString val + else if builtins.isString val then name + else if true == val then name + else if false == val then "" # omit! + else if null == val then "" # omit! + else if builtins.isList val then "${name}:${semicolons val}" + else nope [ solution env name ] "unexpected type: ${builtins.typeOf val}"; + + /* Build fake/fix/keep directives from Nix types */ + makeDirectives = solution: env: val: + lib.mapAttrsToList (makeDirective solution env) val; + + /* Special-case value representation by type/name */ + makeEnvVal = solution: env: val: + if env == "inputs" then lib.makeBinPath val + else if builtins.isString val then val + else if builtins.isList val then spaces val + else if builtins.isAttrs val then spaces (makeDirectives solution env val) + else nope [ solution env ] "unexpected type: ${builtins.typeOf val}"; + + /* Shell-format each env value */ + shellEnv = solution: env: value: + lib.escapeShellArg (makeEnvVal solution env value); + + /* Build a single ENV=val pair */ + makeEnv = solution: env: value: + "RESHOLVE_${lib.toUpper env}=${shellEnv solution env value}"; + + /* Discard attrs claimed by makeArgs */ + removeCliArgs = value: + removeAttrs value [ "scripts" "flags" ]; + + /* Verify required arguments are present */ + validateSolution = { scripts, inputs, interpreter, ... }: true; + + /* Pull out specific solution keys to build ENV=val pairs */ + makeEnvs = solution: value: + spaces (lib.mapAttrsToList (makeEnv solution) (removeCliArgs value)); + + /* Pull out specific solution keys to build CLI argstring */ + makeArgs = { flags ? [ ], scripts, ... }: + spaces (flags ++ scripts); + + /* Build a single resholve invocation */ + makeInvocation = solution: value: + if validateSolution value then + "${makeEnvs solution value} resholve --overwrite ${makeArgs value}" + else throw "invalid solution"; # shouldn't trigger for now + + /* Build resholve invocation for each solution. */ + makeCommands = solutions: + lib.mapAttrsToList makeInvocation solutions; + + self = (stdenv.mkDerivation ((removeAttrs attrs [ "solutions" ]) + // { + inherit pname version src; + buildInputs = [ resholve ]; + + # enable below for verbose debug info if needed + # supports default python.logging levels + # LOGLEVEL="INFO"; + preFixup = '' + pushd "$out" + ${builtins.concatStringsSep "\n" (makeCommands solutions)} + popd + ''; + })); +in +lib.extendDerivation true passthru self diff --git a/pkgs/development/misc/resholve/resholve.nix b/pkgs/development/misc/resholve/resholve.nix new file mode 100644 index 0000000000000..2d1880f704cfd --- /dev/null +++ b/pkgs/development/misc/resholve/resholve.nix @@ -0,0 +1,74 @@ +{ stdenv +, callPackage +, python27Packages +, installShellFiles +, fetchFromGitHub +, file +, findutils +, gettext +, bats +, bash +, doCheck ? true +}: +let + version = "0.4.0"; + rSrc = fetchFromGitHub { + owner = "abathur"; + repo = "resholve"; + rev = "v${version}"; + hash = "sha256-wfxcX3wMZqoi5bWjXYRa21UDDJmTDfE+21p4mL2IJog="; + }; + deps = callPackage ./deps.nix { + /* + resholve needs to patch Oil, but trying to avoid adding + them all *to* nixpkgs, since they aren't specific to + nix/nixpkgs. + */ + oilPatches = [ + "${rSrc}/0001-add_setup_py.patch" + "${rSrc}/0002-add_MANIFEST_in.patch" + "${rSrc}/0003-fix_codegen_shebang.patch" + "${rSrc}/0004-disable-internal-py-yajl-for-nix-built.patch" + ]; + }; +in +python27Packages.buildPythonApplication { + pname = "resholve"; + inherit version; + src = rSrc; + format = "other"; + + nativeBuildInputs = [ installShellFiles ]; + + propagatedBuildInputs = [ deps.oildev python27Packages.ConfigArgParse ]; + + patchPhase = '' + for file in resholve; do + substituteInPlace $file --subst-var-by version ${version} + done + ''; + + installPhase = '' + install -Dm755 resholve $out/bin/resholve + installManPage resholve.1 + ''; + + inherit doCheck; + checkInputs = [ bats ]; + RESHOLVE_PATH = "${stdenv.lib.makeBinPath [ file findutils gettext ]}"; + + checkPhase = '' + # explicit interpreter for test suite + export INTERP="${bash}/bin/bash" PATH="$out/bin:$PATH" + patchShebangs . + ./test.sh + ''; + + meta = with stdenv.lib; { + description = "Resolve external shell-script dependencies"; + homepage = "https://github.com/abathur/resholve"; + license = with licenses; [ mit ]; + maintainers = with maintainers; [ abathur ]; + platforms = platforms.all; + }; +} |