diff options
| author | Matthew Bauer <mjbauer95@gmail.com> | 2020-07-06 14:40:49 -0400 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-07-06 14:40:49 -0400 |
| commit | 669eb09bd5b4f539e8d93383697a4ddfdacac239 (patch) | |
| tree | 0902a9e1d78b51c1ced2f133834f0da0af02e474 /pkgs/build-support | |
| parent | 0046802ab6d3389b45f3e9fce6f5e2746e2e7f80 (diff) | |
| parent | 2af99b8ba1a9b81df88819f7cd84bec18befbabc (diff) | |
Merge branch 'master' into fetchurl-no-hash
Diffstat (limited to 'pkgs/build-support')
34 files changed, 910 insertions, 447 deletions
diff --git a/pkgs/build-support/agda/default.nix b/pkgs/build-support/agda/default.nix index 205aff5557307..3c973e8cc0acb 100644 --- a/pkgs/build-support/agda/default.nix +++ b/pkgs/build-support/agda/default.nix @@ -30,6 +30,16 @@ let withPackages = arg: if builtins.isAttrs arg then withPackages' arg else withPackages' { pkgs = arg; }; + extensions = [ + "agda" + "agda-lib" + "agdai" + "lagda" + "lagda.md" + "lagda.org" + "lagda.rst" + "lagda.tex" + ]; defaults = { pname @@ -39,6 +49,7 @@ let , libraryFile ? "${libraryName}.agda-lib" , buildPhase ? null , installPhase ? null + , extraExtensions ? [] , ... }: let agdaWithArgs = withPackages (builtins.filter (p: p ? isAgdaDerivation) buildInputs); @@ -59,7 +70,7 @@ let installPhase = if installPhase != null then installPhase else '' runHook preInstall mkdir -p $out - find \( -name '*.agda' -or -name '*.agdai' -or -name '*.agda-lib' \) -exec cp -p --parents -t "$out" {} + + find \( ${concatMapStringsSep " -or " (p: "-name '*.${p}'") (extensions ++ extraExtensions)} \) -exec cp -p --parents -t "$out" {} + runHook postInstall ''; }; diff --git a/pkgs/build-support/bintools-wrapper/add-flags.sh b/pkgs/build-support/bintools-wrapper/add-flags.sh index e5c0556556c7e..e99beb381586e 100644 --- a/pkgs/build-support/bintools-wrapper/add-flags.sh +++ b/pkgs/build-support/bintools-wrapper/add-flags.sh @@ -1,32 +1,32 @@ # See cc-wrapper for comments. var_templates_list=( - NIX+IGNORE_LD_THROUGH_GCC - NIX+LDFLAGS - NIX+LDFLAGS_BEFORE - NIX+LDFLAGS_AFTER - NIX+LDFLAGS_HARDEN - NIX+HARDENING_ENABLE + NIX_IGNORE_LD_THROUGH_GCC + NIX_LDFLAGS + NIX_LDFLAGS_BEFORE + NIX_LDFLAGS_AFTER + NIX_LDFLAGS_HARDEN + NIX_HARDENING_ENABLE ) var_templates_bool=( - NIX+SET_BUILD_ID - NIX+DONT_SET_RPATH + NIX_SET_BUILD_ID + NIX_DONT_SET_RPATH ) accumulateRoles for var in "${var_templates_list[@]}"; do - mangleVarList "$var" ${role_infixes[@]+"${role_infixes[@]}"} + mangleVarList "$var" ${role_suffixes[@]+"${role_suffixes[@]}"} done for var in "${var_templates_bool[@]}"; do - mangleVarBool "$var" ${role_infixes[@]+"${role_infixes[@]}"} + mangleVarBool "$var" ${role_suffixes[@]+"${role_suffixes[@]}"} done if [ -e @out@/nix-support/libc-ldflags ]; then - NIX_@infixSalt@_LDFLAGS+=" $(< @out@/nix-support/libc-ldflags)" + NIX_LDFLAGS_@suffixSalt@+=" $(< @out@/nix-support/libc-ldflags)" fi if [ -e @out@/nix-support/libc-ldflags-before ]; then - NIX_@infixSalt@_LDFLAGS_BEFORE="$(< @out@/nix-support/libc-ldflags-before) $NIX_@infixSalt@_LDFLAGS_BEFORE" + NIX_LDFLAGS_BEFORE_@suffixSalt@="$(< @out@/nix-support/libc-ldflags-before) $NIX_LDFLAGS_BEFORE_@suffixSalt@" fi -export NIX_BINTOOLS_WRAPPER_@infixSalt@_FLAGS_SET=1 +export NIX_BINTOOLS_WRAPPER_FLAGS_SET_@suffixSalt@=1 diff --git a/pkgs/build-support/bintools-wrapper/add-hardening.sh b/pkgs/build-support/bintools-wrapper/add-hardening.sh index b7180870860a0..5e49b7bd90536 100644 --- a/pkgs/build-support/bintools-wrapper/add-hardening.sh +++ b/pkgs/build-support/bintools-wrapper/add-hardening.sh @@ -5,7 +5,7 @@ declare -A hardeningEnableMap=() # Intentionally word-split in case 'NIX_HARDENING_ENABLE' is defined in Nix. The # array expansion also prevents undefined variables from causing trouble with # `set -u`. -for flag in ${NIX_@infixSalt@_HARDENING_ENABLE-}; do +for flag in ${NIX_HARDENING_ENABLE_@suffixSalt@-}; do hardeningEnableMap["$flag"]=1 done diff --git a/pkgs/build-support/bintools-wrapper/default.nix b/pkgs/build-support/bintools-wrapper/default.nix index 5cc99c6412a7a..9e31ca6a8a215 100644 --- a/pkgs/build-support/bintools-wrapper/default.nix +++ b/pkgs/build-support/bintools-wrapper/default.nix @@ -45,7 +45,7 @@ let coreutils_bin = if nativeTools then "" else getBin coreutils; # See description in cc-wrapper. - infixSalt = replaceStrings ["-" "."] ["_" "_"] targetPlatform.config; + suffixSalt = replaceStrings ["-" "."] ["_" "_"] targetPlatform.config; # The dynamic linker has different names on different platforms. This is a # shell glob that ought to match it. @@ -83,7 +83,7 @@ stdenv.mkDerivation { shell = getBin shell + shell.shellPath or ""; gnugrep_bin = if nativeTools then "" else gnugrep; - inherit targetPrefix infixSalt; + inherit targetPrefix suffixSalt; outputs = [ "out" ] ++ optionals propagateDoc ([ "man" ] ++ optional (bintools ? info) "info"); @@ -95,9 +95,9 @@ stdenv.mkDerivation { (mapc (lambda (arg) (when (file-directory-p (concat arg "/lib")) - (setenv "NIX_${infixSalt}_LDFLAGS" (concat (getenv "NIX_${infixSalt}_LDFLAGS") " -L" arg "/lib"))) + (setenv "NIX_LDFLAGS_${suffixSalt}" (concat (getenv "NIX_LDFLAGS_${suffixSalt}") " -L" arg "/lib"))) (when (file-directory-p (concat arg "/lib64")) - (setenv "NIX_${infixSalt}_LDFLAGS" (concat (getenv "NIX_${infixSalt}_LDFLAGS") " -L" arg "/lib64")))) + (setenv "NIX_LDFLAGS_${suffixSalt}" (concat (getenv "NIX_LDFLAGS_${suffixSalt}") " -L" arg "/lib64")))) '(${concatStringsSep " " (map (pkg: "\"${pkg}\"") pkgs)})) ''; }; @@ -248,6 +248,11 @@ stdenv.mkDerivation { printWords "''${ldflagsBefore[@]}" > $out/nix-support/libc-ldflags-before '') + + optionalString stdenv.targetPlatform.isMacOS '' + # Ensure consistent LC_VERSION_MIN_MACOSX and remove LC_UUID. + echo "-macosx_version_min 10.12 -sdk_version 10.12 -no_uuid" >> $out/nix-support/libc-ldflags-before + '' + + optionalString (!nativeTools) '' ## ## User env support diff --git a/pkgs/build-support/bintools-wrapper/ld-wrapper.sh b/pkgs/build-support/bintools-wrapper/ld-wrapper.sh index ed2f00a8974ee..81b5a90edd5c2 100644 --- a/pkgs/build-support/bintools-wrapper/ld-wrapper.sh +++ b/pkgs/build-support/bintools-wrapper/ld-wrapper.sh @@ -16,7 +16,7 @@ fi source @out@/nix-support/utils.bash -if [ -z "${NIX_BINTOOLS_WRAPPER_@infixSalt@_FLAGS_SET:-}" ]; then +if [ -z "${NIX_BINTOOLS_WRAPPER_FLAGS_SET_@suffixSalt@:-}" ]; then source @out@/nix-support/add-flags.sh fi @@ -24,7 +24,7 @@ fi # Optionally filter out paths not refering to the store. expandResponseParams "$@" if [[ "${NIX_ENFORCE_PURITY:-}" = 1 && -n "${NIX_STORE:-}" - && ( -z "$NIX_@infixSalt@_IGNORE_LD_THROUGH_GCC" || -z "${NIX_@infixSalt@_LDFLAGS_SET:-}" ) ]]; then + && ( -z "$NIX_IGNORE_LD_THROUGH_GCC_@suffixSalt@" || -z "${NIX_LDFLAGS_SET_@suffixSalt@:-}" ) ]]; then rest=() nParams=${#params[@]} declare -i n=0 @@ -60,12 +60,12 @@ source @out@/nix-support/add-hardening.sh extraAfter=() extraBefore=(${hardeningLDFlags[@]+"${hardeningLDFlags[@]}"}) -if [ -z "${NIX_@infixSalt@_LDFLAGS_SET:-}" ]; then - extraAfter+=($NIX_@infixSalt@_LDFLAGS) - extraBefore+=($NIX_@infixSalt@_LDFLAGS_BEFORE) +if [ -z "${NIX_LDFLAGS_SET_@suffixSalt@:-}" ]; then + extraAfter+=($NIX_LDFLAGS_@suffixSalt@) + extraBefore+=($NIX_LDFLAGS_BEFORE_@suffixSalt@) fi -extraAfter+=($NIX_@infixSalt@_LDFLAGS_AFTER) +extraAfter+=($NIX_LDFLAGS_AFTER_@suffixSalt@) # Specify the target emulation if nothing is passed in ("-m" overrides this # environment variable). Ensures we never blindly fallback on targeting the host @@ -84,8 +84,8 @@ declare -A libs declare -i relocatable=0 link32=0 if - [ "$NIX_@infixSalt@_DONT_SET_RPATH" != 1 ] \ - || [ "$NIX_@infixSalt@_SET_BUILD_ID" = 1 ] \ + [ "$NIX_DONT_SET_RPATH_@suffixSalt@" != 1 ] \ + || [ "$NIX_SET_BUILD_ID_@suffixSalt@" = 1 ] \ || [ -e @out@/nix-support/dynamic-linker-m32 ] then prev= @@ -144,7 +144,7 @@ if [ -e "@out@/nix-support/dynamic-linker-m32" ] && (( "$link32" )); then fi # Add all used dynamic libraries to the rpath. -if [ "$NIX_@infixSalt@_DONT_SET_RPATH" != 1 ]; then +if [ "$NIX_DONT_SET_RPATH_@suffixSalt@" != 1 ]; then # For each directory in the library search path (-L...), # see if it contains a dynamic library used by a -l... flag. If # so, add the directory to the rpath. @@ -186,7 +186,7 @@ fi # Only add --build-id if this is a final link. FIXME: should build gcc # with --enable-linker-build-id instead? -if [ "$NIX_@infixSalt@_SET_BUILD_ID" = 1 ] && ! (( "$relocatable" )); then +if [ "$NIX_SET_BUILD_ID_@suffixSalt@" = 1 ] && ! (( "$relocatable" )); then extraAfter+=(--build-id) fi diff --git a/pkgs/build-support/bintools-wrapper/setup-hook.sh b/pkgs/build-support/bintools-wrapper/setup-hook.sh index a714bd151c90c..7e9547b96c259 100644 --- a/pkgs/build-support/bintools-wrapper/setup-hook.sh +++ b/pkgs/build-support/bintools-wrapper/setup-hook.sh @@ -10,11 +10,11 @@ bintoolsWrapper_addLDVars () { # See ../setup-hooks/role.bash - local role_post role_pre + local role_post getHostRoleEnvHook if [[ -d "$1/lib64" && ! -L "$1/lib64" ]]; then - export NIX_${role_pre}LDFLAGS+=" -L$1/lib64" + export NIX_LDFLAGS${role_post}+=" -L$1/lib64" fi if [[ -d "$1/lib" ]]; then @@ -24,7 +24,7 @@ bintoolsWrapper_addLDVars () { # directories and bloats the size of the environment variable space. local -a glob=( $1/lib/lib* ) if [ "${#glob[*]}" -gt 0 ]; then - export NIX_${role_pre}LDFLAGS+=" -L$1/lib" + export NIX_LDFLAGS${role_post}+=" -L$1/lib" fi fi } @@ -52,7 +52,7 @@ fi # Export tool environment variables so various build systems use the right ones. -export NIX_${role_pre}BINTOOLS=@out@ +export NIX_BINTOOLS${role_post}=@out@ for cmd in \ ar as ld nm objcopy objdump readelf ranlib strip strings size windres @@ -60,7 +60,6 @@ do if PATH=$_PATH type -p "@targetPrefix@${cmd}" > /dev/null then - export "${role_pre}${cmd^^}=@targetPrefix@${cmd}"; export "${cmd^^}${role_post}=@targetPrefix@${cmd}"; fi done @@ -70,4 +69,4 @@ done export NIX_HARDENING_ENABLE # No local scope in sourced file -unset -v role_pre role_post cmd upper_case +unset -v role_post cmd upper_case diff --git a/pkgs/build-support/build-fhs-userenv/env.nix b/pkgs/build-support/build-fhs-userenv/env.nix index 8de43d5a9195d..083e7617b5029 100644 --- a/pkgs/build-support/build-fhs-userenv/env.nix +++ b/pkgs/build-support/build-fhs-userenv/env.nix @@ -58,7 +58,7 @@ let # Force compilers and other tools to look in default search paths unset NIX_ENFORCE_PURITY - export NIX_CC_WRAPPER_${stdenv.cc.infixSalt}_TARGET_HOST=1 + export NIX_CC_WRAPPER_TARGET_HOST_${stdenv.cc.suffixSalt}=1 export NIX_CFLAGS_COMPILE='-idirafter /usr/include' export NIX_CFLAGS_LINK='-L/usr/lib -L/usr/lib32' export NIX_LDFLAGS='-L/usr/lib -L/usr/lib32' diff --git a/pkgs/build-support/buildenv/builder.pl b/pkgs/build-support/buildenv/builder.pl index fc6ffce735cc3..411b147cc58ef 100755 --- a/pkgs/build-support/buildenv/builder.pl +++ b/pkgs/build-support/buildenv/builder.pl @@ -129,6 +129,15 @@ sub findFiles { return; } + # If target already exists and both targets resolves to the same path, skip + if (defined $oldTarget && $oldTarget ne "" && abs_path($target) eq abs_path($oldTarget)) { + # Prefer the target that is not a symlink, if any + if (-l $oldTarget && ! -l $target) { + $symlinks{$relName} = [$target, $priority]; + } + return; + } + # If target already exists as a symlink to a file (not a # directory) in a higher-priority package, skip. if (defined $oldTarget && $priority > $oldPriority && $oldTarget ne "" && ! -d $oldTarget) { diff --git a/pkgs/build-support/cc-wrapper/add-flags.sh b/pkgs/build-support/cc-wrapper/add-flags.sh index 323ea5bfd7722..3398f11e8c215 100644 --- a/pkgs/build-support/cc-wrapper/add-flags.sh +++ b/pkgs/build-support/cc-wrapper/add-flags.sh @@ -5,15 +5,15 @@ # wrapped binary just inherit the work of the forker's wrapper script. var_templates_list=( - NIX+CFLAGS_COMPILE - NIX+CFLAGS_COMPILE_BEFORE - NIX+CFLAGS_LINK - NIX+CXXSTDLIB_COMPILE - NIX+CXXSTDLIB_LINK - NIX+GNATFLAGS_COMPILE + NIX_CFLAGS_COMPILE + NIX_CFLAGS_COMPILE_BEFORE + NIX_CFLAGS_LINK + NIX_CXXSTDLIB_COMPILE + NIX_CXXSTDLIB_LINK + NIX_GNATFLAGS_COMPILE ) var_templates_bool=( - NIX+ENFORCE_NO_NATIVE + NIX_ENFORCE_NO_NATIVE ) accumulateRoles @@ -21,37 +21,37 @@ accumulateRoles # We need to mangle names for hygiene, but also take parameters/overrides # from the environment. for var in "${var_templates_list[@]}"; do - mangleVarList "$var" ${role_infixes[@]+"${role_infixes[@]}"} + mangleVarList "$var" ${role_suffixes[@]+"${role_suffixes[@]}"} done for var in "${var_templates_bool[@]}"; do - mangleVarBool "$var" ${role_infixes[@]+"${role_infixes[@]}"} + mangleVarBool "$var" ${role_suffixes[@]+"${role_suffixes[@]}"} done # `-B@out@/bin' forces cc to use ld-wrapper.sh when calling ld. -NIX_@infixSalt@_CFLAGS_COMPILE="-B@out@/bin/ $NIX_@infixSalt@_CFLAGS_COMPILE" +NIX_CFLAGS_COMPILE_@suffixSalt@="-B@out@/bin/ $NIX_CFLAGS_COMPILE_@suffixSalt@" # Export and assign separately in order that a failing $(..) will fail # the script. if [ -e @out@/nix-support/libc-cflags ]; then - NIX_@infixSalt@_CFLAGS_COMPILE="$(< @out@/nix-support/libc-cflags) $NIX_@infixSalt@_CFLAGS_COMPILE" + NIX_CFLAGS_COMPILE_@suffixSalt@="$(< @out@/nix-support/libc-cflags) $NIX_CFLAGS_COMPILE_@suffixSalt@" fi if [ -e @out@/nix-support/cc-cflags ]; then - NIX_@infixSalt@_CFLAGS_COMPILE="$(< @out@/nix-support/cc-cflags) $NIX_@infixSalt@_CFLAGS_COMPILE" + NIX_CFLAGS_COMPILE_@suffixSalt@="$(< @out@/nix-support/cc-cflags) $NIX_CFLAGS_COMPILE_@suffixSalt@" fi if [ -e @out@/nix-support/gnat-cflags ]; then - NIX_@infixSalt@_GNATFLAGS_COMPILE="$(< @out@/nix-support/gnat-cflags) $NIX_@infixSalt@_GNATFLAGS_COMPILE" + NIX_GNATFLAGS_COMPILE_@suffixSalt@="$(< @out@/nix-support/gnat-cflags) $NIX_GNATFLAGS_COMPILE_@suffixSalt@" fi if [ -e @out@/nix-support/cc-ldflags ]; then - NIX_@infixSalt@_LDFLAGS+=" $(< @out@/nix-support/cc-ldflags)" + NIX_LDFLAGS_@suffixSalt@+=" $(< @out@/nix-support/cc-ldflags)" fi if [ -e @out@/nix-support/cc-cflags-before ]; then - NIX_@infixSalt@_CFLAGS_COMPILE_BEFORE="$(< @out@/nix-support/cc-cflags-before) $NIX_@infixSalt@_CFLAGS_COMPILE_BEFORE" + NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@="$(< @out@/nix-support/cc-cflags-before) $NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@" fi # That way forked processes will not extend these environment variables again. -export NIX_CC_WRAPPER_@infixSalt@_FLAGS_SET=1 +export NIX_CC_WRAPPER_FLAGS_SET_@suffixSalt@=1 diff --git a/pkgs/build-support/cc-wrapper/add-hardening.sh b/pkgs/build-support/cc-wrapper/add-hardening.sh index fc40fe7408b60..8e2fe6c407ea7 100644 --- a/pkgs/build-support/cc-wrapper/add-hardening.sh +++ b/pkgs/build-support/cc-wrapper/add-hardening.sh @@ -5,7 +5,7 @@ declare -A hardeningEnableMap=() # Intentionally word-split in case 'NIX_HARDENING_ENABLE' is defined in Nix. The # array expansion also prevents undefined variables from causing trouble with # `set -u`. -for flag in ${NIX_@infixSalt@_HARDENING_ENABLE-}; do +for flag in ${NIX_HARDENING_ENABLE_@suffixSalt@-}; do hardeningEnableMap["$flag"]=1 done diff --git a/pkgs/build-support/cc-wrapper/cc-wrapper.sh b/pkgs/build-support/cc-wrapper/cc-wrapper.sh index ba3dfc96f5c4a..cf00202221e6b 100644 --- a/pkgs/build-support/cc-wrapper/cc-wrapper.sh +++ b/pkgs/build-support/cc-wrapper/cc-wrapper.sh @@ -18,12 +18,12 @@ fi source @out@/nix-support/utils.bash # Flirting with a layer violation here. -if [ -z "${NIX_BINTOOLS_WRAPPER_@infixSalt@_FLAGS_SET:-}" ]; then +if [ -z "${NIX_BINTOOLS_WRAPPER_FLAGS_SET_@suffixSalt@:-}" ]; then source @bintools@/nix-support/add-flags.sh fi # Put this one second so libc ldflags take priority. -if [ -z "${NIX_CC_WRAPPER_@infixSalt@_FLAGS_SET:-}" ]; then +if [ -z "${NIX_CC_WRAPPER_FLAGS_SET_@suffixSalt@:-}" ]; then source @out@/nix-support/add-flags.sh fi @@ -113,7 +113,7 @@ fi # Clear march/mtune=native -- they bring impurity. -if [ "$NIX_@infixSalt@_ENFORCE_NO_NATIVE" = 1 ]; then +if [ "$NIX_ENFORCE_NO_NATIVE_@suffixSalt@" = 1 ]; then rest=() # Old bash empty array hack for p in ${params+"${params[@]}"}; do @@ -129,36 +129,36 @@ fi if [[ "$isCpp" = 1 ]]; then if [[ "$cppInclude" = 1 ]]; then - NIX_@infixSalt@_CFLAGS_COMPILE+=" ${NIX_@infixSalt@_CXXSTDLIB_COMPILE:-@default_cxx_stdlib_compile@}" + NIX_CFLAGS_COMPILE_@suffixSalt@+=" ${NIX_CXXSTDLIB_COMPILE_@suffixSalt@:-@default_cxx_stdlib_compile@}" fi - NIX_@infixSalt@_CFLAGS_LINK+=" $NIX_@infixSalt@_CXXSTDLIB_LINK" + NIX_CFLAGS_LINK_@suffixSalt@+=" $NIX_CXXSTDLIB_LINK_@suffixSalt@" fi source @out@/nix-support/add-hardening.sh # Add the flags for the C compiler proper. -extraAfter=($NIX_@infixSalt@_CFLAGS_COMPILE) -extraBefore=(${hardeningCFlags[@]+"${hardeningCFlags[@]}"} $NIX_@infixSalt@_CFLAGS_COMPILE_BEFORE) +extraAfter=($NIX_CFLAGS_COMPILE_@suffixSalt@) +extraBefore=(${hardeningCFlags[@]+"${hardeningCFlags[@]}"} $NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@) if [ "$dontLink" != 1 ]; then # Add the flags that should only be passed to the compiler when # linking. - extraAfter+=($NIX_@infixSalt@_CFLAGS_LINK) + extraAfter+=($NIX_CFLAGS_LINK_@suffixSalt@) # Add the flags that should be passed to the linker (and prevent - # `ld-wrapper' from adding NIX_@infixSalt@_LDFLAGS again). - for i in $NIX_@infixSalt@_LDFLAGS_BEFORE; do + # `ld-wrapper' from adding NIX_LDFLAGS_@suffixSalt@ again). + for i in $NIX_LDFLAGS_BEFORE_@suffixSalt@; do extraBefore+=("-Wl,$i") done - for i in $NIX_@infixSalt@_LDFLAGS; do + for i in $NIX_LDFLAGS_@suffixSalt@; do if [ "${i:0:3}" = -L/ ]; then extraAfter+=("$i") else extraAfter+=("-Wl,$i") fi done - export NIX_@infixSalt@_LDFLAGS_SET=1 + export NIX_LDFLAGS_SET_@suffixSalt@=1 fi # As a very special hack, if the arguments are just `-v', then don't diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix index 36177ec2e3586..da16a23f9dff3 100644 --- a/pkgs/build-support/cc-wrapper/default.nix +++ b/pkgs/build-support/cc-wrapper/default.nix @@ -54,12 +54,12 @@ let "-isystem ${libcxx}/include/c++/v1" else ""; - # The "infix salt" is a arbitrary string added in the middle of env vars + # The "suffix salt" is a arbitrary string added in the end of env vars # defined by cc-wrapper's hooks so that multiple cc-wrappers can be used # without interfering. For the moment, it is defined as the target triple, # adjusted to be a valid bash identifier. This should be considered an # unstable implementation detail, however. - infixSalt = replaceStrings ["-" "."] ["_" "_"] targetPlatform.config; + suffixSalt = replaceStrings ["-" "."] ["_" "_"] targetPlatform.config; expand-response-params = if buildPackages.stdenv.hasCC && buildPackages.stdenv.cc != "/dev/null" @@ -106,7 +106,7 @@ stdenv.mkDerivation { shell = getBin shell + shell.shellPath or ""; gnugrep_bin = if nativeTools then "" else gnugrep; - inherit targetPrefix infixSalt; + inherit targetPrefix suffixSalt; outputs = [ "out" ] ++ optionals propagateDoc [ "man" "info" ]; @@ -123,7 +123,7 @@ stdenv.mkDerivation { (mapc (lambda (arg) (when (file-directory-p (concat arg "/include")) - (setenv "NIX_${infixSalt}_CFLAGS_COMPILE" (concat (getenv "NIX_${infixSalt}_CFLAGS_COMPILE") " -isystem " arg "/include")))) + (setenv "NIX_CFLAGS_COMPILE_${suffixSalt}" (concat (getenv "NIX_CFLAGS_COMPILE_${suffixSalt}") " -isystem " arg "/include")))) '(${concatStringsSep " " (map (pkg: "\"${pkg}\"") pkgs)})) ''; }; diff --git a/pkgs/build-support/cc-wrapper/gnat-wrapper.sh b/pkgs/build-support/cc-wrapper/gnat-wrapper.sh index 15b53d76c6307..d3f7d382c19a7 100644 --- a/pkgs/build-support/cc-wrapper/gnat-wrapper.sh +++ b/pkgs/build-support/cc-wrapper/gnat-wrapper.sh @@ -18,12 +18,12 @@ fi source @out@/nix-support/utils.bash # Flirting with a layer violation here. -if [ -z "${NIX_BINTOOLS_WRAPPER_@infixSalt@_FLAGS_SET:-}" ]; then +if [ -z "${NIX_BINTOOLS_WRAPPER_FLAGS_SET_@suffixSalt@:-}" ]; then source @bintools@/nix-support/add-flags.sh fi # Put this one second so libc ldflags take priority. -if [ -z "${NIX_CC_WRAPPER_@infixSalt@_FLAGS_SET:-}" ]; then +if [ -z "${NIX_CC_WRAPPER_FLAGS_SET_@suffixSalt@:-}" ]; then source @out@/nix-support/add-flags.sh fi @@ -108,7 +108,7 @@ fi # Clear march/mtune=native -- they bring impurity. -if [ "$NIX_@infixSalt@_ENFORCE_NO_NATIVE" = 1 ]; then +if [ "$NIX_ENFORCE_NO_NATIVE_@suffixSalt@" = 1 ]; then rest=() # Old bash empty array hack for p in ${params+"${params[@]}"}; do @@ -124,12 +124,12 @@ fi if [ "$(basename $0)x" = "gnatmakex" ]; then extraBefore=("--GNATBIND=@out@/bin/gnatbind" "--GNATLINK=@out@/bin/gnatlink") - extraAfter=($NIX_@infixSalt@_GNATFLAGS_COMPILE) + extraAfter=($NIX_GNATFLAGS_COMPILE_@suffixSalt@) fi if [ "$(basename $0)x" = "gnatbindx" ]; then extraBefore=() - extraAfter=($NIX_@infixSalt@_GNATFLAGS_COMPILE) + extraAfter=($NIX_GNATFLAGS_COMPILE_@suffixSalt@) fi if [ "$(basename $0)x" = "gnatlinkx" ]; then diff --git a/pkgs/build-support/cc-wrapper/setup-hook.sh b/pkgs/build-support/cc-wrapper/setup-hook.sh index 5b13f26830978..6a913cc4eac72 100644 --- a/pkgs/build-support/cc-wrapper/setup-hook.sh +++ b/pkgs/build-support/cc-wrapper/setup-hook.sh @@ -6,9 +6,9 @@ # tools. For example: # # # Flags for compiling (whether or not linking) C code for the... -# NIX_BUILD_CFLAGS_COMPILE # ...build platform -# NIX_CFLAGS_COMPILE # ...host platform -# NIX_TARGET_CFLAGS_COMPILE # ...target platform +# NIX_CFLAGS_COMPILE_FOR_BUILD # ...build platform +# NIX_CFLAGS_COMPILE # ...host platform +# NIX_CFLAGS_COMPILE_FOR_TARGET # ...target platform # # Notice that these platforms are the 3 *relative* to the package using # cc-wrapper, not absolute like `x86_64-pc-linux-gnu`. @@ -33,12 +33,12 @@ # The basic strategy is: # # - Everyone exclusively *adds information* to relative-platform-specific -# environment variables, like `NIX_TARGET_CFLAGS_COMPILE`, to communicate +# environment variables, like `NIX_CFLAGS_COMPILE_FOR_TARGET`, to communicate # with the wrapped binaries. # # - The wrapped binaries will exclusively *read* cc-wrapper-derivation-specific -# environment variables distinguished with with `infixSalt`, like -# `NIX_@infixSalt@_CFLAGS_COMPILE`. +# environment variables distinguished with with `suffixSalt`, like +# `NIX_CFLAGS_COMPILE_@suffixSalt@`. # # - `add-flags`, beyond its old task of reading extra flags stuck inside the # cc-wrapper derivation, will convert the relative-platform-specific @@ -65,15 +65,15 @@ # function is guaranteed to be exactly the same. ccWrapper_addCVars () { # See ../setup-hooks/role.bash - local role_post role_pre + local role_post getHostRoleEnvHook if [ -d "$1/include" ]; then - export NIX_${role_pre}CFLAGS_COMPILE+=" -isystem $1/include" + export NIX_CFLAGS_COMPILE${role_post}+=" -isystem $1/include" fi if [ -d "$1/Library/Frameworks" ]; then - export NIX_${role_pre}CFLAGS_COMPILE+=" -iframework $1/Library/Frameworks" + export NIX_CFLAGS_COMPILE${role_post}+=" -iframework $1/Library/Frameworks" fi } @@ -105,10 +105,10 @@ fi # Export tool environment variables so various build systems use the right ones. -export NIX_${role_pre}CC=@out@ +export NIX_CC${role_post}=@out@ -export ${role_pre}CC=@named_cc@ -export ${role_pre}CXX=@named_cxx@ +export CC${role_post}=@named_cc@ +export CXX${role_post}=@named_cxx@ export CC${role_post}=@named_cc@ export CXX${role_post}=@named_cxx@ @@ -117,4 +117,4 @@ export CXX${role_post}=@named_cxx@ export NIX_HARDENING_ENABLE # No local scope in sourced file -unset -v role_pre role_post +unset -v role_post diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 83f4a9e0c01b5..7ff325382a6b5 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -11,6 +11,7 @@ jq, jshon, lib, + makeWrapper, moreutils, nix, pigz, @@ -29,6 +30,7 @@ writeReferencesToFile, writeScript, writeText, + writePython3, }: # WARNING: this API is unstable and may be subject to backwards-incompatible changes in the future. @@ -204,24 +206,17 @@ rec { mkdir image tar -C image -xpf "$fromImage" - # If the image name isn't set, read it from the image repository json. - if [[ -z "$fromImageName" ]]; then - fromImageName=$(jshon -k < image/repositories | head -n 1) - echo "From-image name wasn't set. Read $fromImageName." - fi - - # If the tag isn't set, use the name as an index into the json - # and read the first key found. - if [[ -z "$fromImageTag" ]]; then - fromImageTag=$(jshon -e $fromImageName -k < image/repositories \ - | head -n1) - echo "From-image tag wasn't set. Read $fromImageTag." + if [[ -n "$fromImageName" ]] && [[ -n "$fromImageTag" ]]; then + parentID="$( + cat "image/manifest.json" | + jq -r '.[] | select(.RepoTags | contains([$desiredTag])) | rtrimstr(".json")' \ + --arg desiredTag "$fromImageName:$fromImageTag" + )" + else + echo "From-image name or tag wasn't set. Reading the first ID." + parentID="$(cat "image/manifest.json" | jq -r '.[0].Config | rtrimstr(".json")')" fi - # Use the name and tag to get the parent ID field. - parentID=$(jshon -e $fromImageName -e $fromImageTag -u \ - < image/repositories) - cat ./image/manifest.json | jq -r '.[0].Layers | .[]' > layer-list else touch layer-list @@ -305,106 +300,6 @@ rec { ${text} ''; - # Create $maxLayers worth of Docker Layers, one layer per store path - # unless there are more paths than $maxLayers. In that case, create - # $maxLayers-1 for the most popular layers, and smush the remainaing - # store paths in to one final layer. - # - # NOTE: the `closures` parameter is a list of closures to include. - # The TOP LEVEL store paths themselves will never be present in the - # resulting image. At this time (2019-12-16) none of these layers - # are appropriate to include, as they are all created as - # implementation details of dockerTools. - mkManyPureLayers = { - name, - # Files to add to the layer. - closures, - configJson, - # Docker has a 125-layer maximum, we pick 100 to ensure there is - # plenty of room for extension. - # https://github.com/moby/moby/blob/b3e9f7b13b0f0c414fa6253e1f17a86b2cff68b5/layer/layer_store.go#L23-L26 - maxLayers ? 100 - }: - let - storePathToLayer = substituteAll - { shell = runtimeShell; - isExecutable = true; - src = ./store-path-to-layer.sh; - }; - - overallClosure = writeText "closure" (lib.concatStringsSep " " closures); - in - runCommand "${name}-granular-docker-layers" { - inherit maxLayers; - paths = referencesByPopularity overallClosure; - nativeBuildInputs = [ jshon rsync tarsum moreutils ]; - enableParallelBuilding = true; - } - '' - mkdir layers - - # Delete impurities for store path layers, so they don't get - # shared and taint other projects. - cat ${configJson} \ - | jshon -d config \ - | jshon -s "1970-01-01T00:00:01Z" -i created > generic.json - - # WARNING! - # The following code is fiddly w.r.t. ensuring every layer is - # created, and that no paths are missed. If you change the - # following head and tail call lines, double-check that your - # code behaves properly when the number of layers equals: - # maxLayers-1, maxLayers, and maxLayers+1, 0 - paths() { - cat $paths ${lib.concatMapStringsSep " " (path: "| (grep -v ${path} || true)") (closures ++ [ overallClosure ])} - } - - paths | head -n $((maxLayers - 1)) | cat -n | xargs -r -P$NIX_BUILD_CORES -n2 ${storePathToLayer} - if [ $(paths | wc -l) -ge $maxLayers ]; then - paths | tail -n+$maxLayers | xargs ${storePathToLayer} $maxLayers - fi - - echo "Finished building layer '$name'" - - mv ./layers $out - ''; - - # Create a "Customisation" layer which adds symlinks at the root of - # the image to the root paths of the closure. Also add the config - # data like what command to run and the environment to run it in. - mkCustomisationLayer = { - name, - # Files to add to the layer. - contents, - baseJson, - extraCommands, - uid ? 0, gid ? 0, - }: - runCommand "${name}-customisation-layer" { - nativeBuildInputs = [ jshon rsync tarsum ]; - inherit extraCommands; - } - '' - cp -r ${contents}/ ./layer - - if [[ -n $extraCommands ]]; then - chmod ug+w layer - (cd layer; eval "$extraCommands") - fi - - # Tar up the layer and throw it into 'layer.tar', while calculating its checksum. - echo "Packing layer..." - mkdir $out - tarhash=$(tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf - . | tee $out/layer.tar | tarsum) - - # Add a 'checksum' field to the JSON, with the value set to the - # checksum of the tarball. - cat ${baseJson} | jshon -s "$tarhash" -i checksum > $out/json - - # Indicate to docker that we're using schema version 1.0. - echo -n "1.0" > $out/VERSION - ''; - # Create a "layer" (set of files). mkPureLayer = { # Name of the layer @@ -438,7 +333,7 @@ rec { chmod ug+w layer - if [[ -n $extraCommands ]]; then + if [[ -n "$extraCommands" ]]; then (cd layer; eval "$extraCommands") fi @@ -541,131 +436,14 @@ rec { ''; }; - buildLayeredImage = { - # Image Name - name, - # Image tag, the Nix's output hash will be used if null - tag ? null, - # Files to put on the image (a nix store path or list of paths). - contents ? [], - # Docker config; e.g. what command to run on the container. - config ? {}, - # Time of creation of the image. Passing "now" will make the - # created date be the time of building. - created ? "1970-01-01T00:00:01Z", - # Optional bash script to run on the files prior to fixturizing the layer. - extraCommands ? "", uid ? 0, gid ? 0, - # We pick 100 to ensure there is plenty of room for extension. I - # believe the actual maximum is 128. - maxLayers ? 100 - }: - assert - (lib.assertMsg (maxLayers > 1) - "the maxLayers argument of dockerTools.buildLayeredImage function must be greather than 1 (current value: ${toString maxLayers})"); + buildLayeredImage = {name, ...}@args: let - baseName = baseNameOf name; - contentsEnv = symlinkJoin { - name = "bulk-layers"; - paths = if builtins.isList contents - then contents - else [ contents ]; - }; - - configJson = let - pure = writeText "${baseName}-config.json" (builtins.toJSON { - inherit created config; - architecture = buildPackages.go.GOARCH; - os = "linux"; - }); - impure = runCommand "${baseName}-standard-dynamic-date.json" - { nativeBuildInputs = [ jq ]; } - '' - jq ".created = \"$(TZ=utc date --iso-8601="seconds")\"" ${pure} > $out - ''; - in if created == "now" then impure else pure; - - bulkLayers = mkManyPureLayers { - name = baseName; - closures = [ contentsEnv configJson ]; - # One layer will be taken up by the customisationLayer, so - # take up one less. - maxLayers = maxLayers - 1; - inherit configJson; - }; - customisationLayer = mkCustomisationLayer { - name = baseName; - contents = contentsEnv; - baseJson = configJson; - inherit uid gid extraCommands; - }; - result = runCommand "docker-image-${baseName}.tar.gz" { - nativeBuildInputs = [ jshon pigz coreutils findutils jq ]; - # Image name and tag must be lowercase - imageName = lib.toLower name; - baseJson = configJson; - passthru.imageTag = - if tag == null - then lib.head (lib.splitString "-" (lib.last (lib.splitString "/" result))) - else lib.toLower tag; - # Docker can't be made to run darwin binaries - meta.badPlatforms = lib.platforms.darwin; - } '' - ${if (tag == null) then '' - outName="$(basename "$out")" - outHash=$(echo "$outName" | cut -d - -f 1) - - imageTag=$outHash - '' else '' - imageTag="${tag}" - ''} - - find ${bulkLayers} -mindepth 1 -maxdepth 1 | sort -t/ -k5 -n > layer-list - echo ${customisationLayer} >> layer-list - - mkdir image - imageJson=$(cat ${configJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}") - manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]") - for layer in $(cat layer-list); do - layerChecksum=$(sha256sum $layer/layer.tar | cut -d ' ' -f1) - layerID=$(sha256sum "$layer/json" | cut -d ' ' -f 1) - ln -s "$layer" "./image/$layerID" - - manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= . + [\"$layerID/layer.tar\"]") - imageJson=$(echo "$imageJson" | jq ".history |= . + [{\"created\": \"$(jq -r .created ${configJson})\"}]") - imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= . + [\"sha256:$layerChecksum\"]") - done - imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1) - echo "$imageJson" > "image/$imageJsonChecksum.json" - manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"$imageJsonChecksum.json\"") - echo "$manifestJson" > image/manifest.json - - jshon -n object \ - -n object -s "$layerID" -i "$imageTag" \ - -i "$imageName" > image/repositories - - echo "Cooking the image..." - # tar exits with an exit code of 1 if files changed while it was - # reading them. It considers a change in the number of hard links - # to be a "change", which can cause this to fail if images are being - # built concurrently and the auto-optimise-store nix option is turned on. - # Since the contents of these files will not change, we can reasonably - # ignore this exit code. - set +e - tar -C image --dereference --hard-dereference --sort=name \ - --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 \ - --mode=a-w --xform s:'^./':: --use-compress-program='pigz -nT' \ - --warning=no-file-changed -cf $out . - RET=$? - if [ $RET -ne 0 ] && [ $RET -ne 1 ]; then - exit $RET - fi - set -e - - echo "Finished." - ''; - + stream = streamLayeredImage args; in - result; + runCommand "${name}.tar.gz" { + inherit (stream) imageName; + buildInputs = [ pigz ]; + } "${stream} | pigz -nT > $out"; # 1. extract the base image # 2. create the layer @@ -774,20 +552,22 @@ rec { configName="$(cat ./image/manifest.json | jq -r '.[0].Config')" baseEnvs="$(cat "./image/$configName" | jq '.config.Env // []')" + # Extract the parentID from the manifest + if [[ -n "$fromImageName" ]] && [[ -n "$fromImageTag" ]]; then + parentID="$( + cat "image/manifest.json" | + jq -r '.[] | select(.RepoTags | contains([$desiredTag])) | rtrimstr(".json")' \ + --arg desiredTag "$fromImageName:$fromImageTag" + )" + else + echo "From-image name or tag wasn't set. Reading the first ID." + parentID="$(cat "image/manifest.json" | jq -r '.[0].Config | rtrimstr(".json")')" + fi + # Otherwise do not import the base image configuration and manifest chmod a+w image image/*.json rm -f image/*.json - if [[ -z "$fromImageName" ]]; then - fromImageName=$(jshon -k < image/repositories|head -n1) - fi - if [[ -z "$fromImageTag" ]]; then - fromImageTag=$(jshon -e $fromImageName -k \ - < image/repositories|head -n1) - fi - parentID=$(jshon -e $fromImageName -e $fromImageTag -u \ - < image/repositories) - for l in image/*/layer.tar; do ls_tar $l >> baseFiles done @@ -904,4 +684,117 @@ rec { }) ); + streamLayeredImage = { + # Image Name + name, + # Image tag, the Nix's output hash will be used if null + tag ? null, + # Files to put on the image (a nix store path or list of paths). + contents ? [], + # Docker config; e.g. what command to run on the container. + config ? {}, + # Time of creation of the image. Passing "now" will make the + # created date be the time of building. + created ? "1970-01-01T00:00:01Z", + # Optional bash script to run on the files prior to fixturizing the layer. + extraCommands ? "", + # We pick 100 to ensure there is plenty of room for extension. I + # believe the actual maximum is 128. + maxLayers ? 100 + }: + assert + (lib.assertMsg (maxLayers > 1) + "the maxLayers argument of dockerTools.buildLayeredImage function must be greather than 1 (current value: ${toString maxLayers})"); + let + streamScript = writePython3 "stream" {} ./stream_layered_image.py; + baseJson = writeText "${name}-base.json" (builtins.toJSON { + inherit config; + architecture = buildPackages.go.GOARCH; + os = "linux"; + }); + customisationLayer = runCommand "${name}-customisation-layer" { inherit extraCommands; } '' + cp -r ${contentsEnv}/ $out + + if [[ -n $extraCommands ]]; then + chmod u+w $out + (cd $out; eval "$extraCommands") + fi + ''; + contentsEnv = symlinkJoin { + name = "${name}-bulk-layers"; + paths = if builtins.isList contents + then contents + else [ contents ]; + }; + + # NOTE: the `closures` parameter is a list of closures to include. + # The TOP LEVEL store paths themselves will never be present in the + # resulting image. At this time (2020-06-18) none of these layers + # are appropriate to include, as they are all created as + # implementation details of dockerTools. + closures = [ baseJson contentsEnv ]; + overallClosure = writeText "closure" (lib.concatStringsSep " " closures); + conf = runCommand "${name}-conf.json" { + inherit maxLayers created; + imageName = lib.toLower name; + paths = referencesByPopularity overallClosure; + buildInputs = [ jq ]; + } '' + paths() { + cat $paths ${lib.concatMapStringsSep " " (path: "| (grep -v ${path} || true)") (closures ++ [ overallClosure ])} + } + ${if (tag == null) then '' + outName="$(basename "$out")" + outHash=$(echo "$outName" | cut -d - -f 1) + + imageTag=$outHash + '' else '' + imageTag="${tag}" + ''} + + # convert "created" to iso format + if [[ "$created" != "now" ]]; then + created="$(date -Iseconds -d "$created")" + fi + + # Create $maxLayers worth of Docker Layers, one layer per store path + # unless there are more paths than $maxLayers. In that case, create + # $maxLayers-1 for the most popular layers, and smush the remainaing + # store paths in to one final layer. + # + # The following code is fiddly w.r.t. ensuring every layer is + # created, and that no paths are missed. If you change the + # following lines, double-check that your code behaves properly + # when the number of layers equals: + # maxLayers-1, maxLayers, and maxLayers+1, 0 + store_layers="$( + paths | + jq -sR ' + rtrimstr("\n") | split("\n") + | (.[:$maxLayers-1] | map([.])) + [ .[$maxLayers-1:] ] + | map(select(length > 0)) + ' \ + --argjson maxLayers "$(( maxLayers - 1 ))" # one layer will be taken up by the customisation layer + )" + + cat ${baseJson} | jq ' + . + { + "store_layers": $store_layers, + "customisation_layer", $customisation_layer, + "repo_tag": $repo_tag, + "created": $created + } + ' --argjson store_layers "$store_layers" \ + --arg customisation_layer ${customisationLayer} \ + --arg repo_tag "$imageName:$imageTag" \ + --arg created "$created" | + tee $out + ''; + result = runCommand "stream-${name}" { + inherit (conf) imageName; + buildInputs = [ makeWrapper ]; + } '' + makeWrapper ${streamScript} $out --add-flags ${conf} + ''; + in result; } diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix index d533e3abd03bd..068daa8df722e 100644 --- a/pkgs/build-support/docker/examples.nix +++ b/pkgs/build-support/docker/examples.nix @@ -121,6 +121,7 @@ rec { # the image env variable NIX_PAGER. pkgs.coreutils pkgs.nix + pkgs.bash ]; config = { Env = [ @@ -313,4 +314,44 @@ rec { ) ]; }; + + nixLayered = pkgs.dockerTools.buildLayeredImageWithNixDb { + name = "nix-layered"; + tag = "latest"; + contents = [ + # nix-store uses cat program to display results as specified by + # the image env variable NIX_PAGER. + pkgs.coreutils + pkgs.nix + pkgs.bash + ]; + config = { + Env = [ + "NIX_PAGER=cat" + # A user is required by nix + # https://github.com/NixOS/nix/blob/9348f9291e5d9e4ba3c4347ea1b235640f54fd79/src/libutil/util.cc#L478 + "USER=nobody" + ]; + }; + }; + + # 19. Support files in the store on buildLayeredImage + # See: https://github.com/NixOS/nixpkgs/pull/91084#issuecomment-653496223 + filesInStore = pkgs.dockerTools.buildLayeredImageWithNixDb { + name = "file-in-store"; + tag = "latest"; + contents = [ + pkgs.coreutils + pkgs.nix + (pkgs.writeScriptBin "myscript" '' + #!${pkgs.runtimeShell} + cat ${pkgs.writeText "somefile" "some data"} + '') + ]; + config = { + Cmd = [ "myscript" ]; + # For some reason 'nix-store --verify' requires this environment variable + Env = [ "USER=root" ]; + }; + }; } diff --git a/pkgs/build-support/docker/store-path-to-layer.sh b/pkgs/build-support/docker/store-path-to-layer.sh deleted file mode 100755 index 7437da51cc4a4..0000000000000 --- a/pkgs/build-support/docker/store-path-to-layer.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!@shell@ - -set -eu - -layerNumber=$1 -shift - -layerPath="./layers/$layerNumber" -echo "Creating layer #$layerNumber for $@" - -mkdir -p "$layerPath" - -# Make sure /nix and /nix/store appear first in the archive. -# -# We create the directories here and use them because -# when there are other things being added to the -# nix store, tar could fail, saying, -# "tar: /nix/store: file changed as we read it" -mkdir -p nix/store - -# Then we change into the /nix/store in order to -# avoid a similar "file changed as we read it" error -# as above. Namely, if we use the absolute path of -# /nix/store/123-pkg and something new is added to the nix -# store while tar is running, it will detect a change to -# /nix/store and fail. Instead, if we cd into the nix store -# and copy the relative nix store path, tar will ignore -# changes to /nix/store. In order to create the correct -# structure in the tar file, we transform the relative nix -# store path to the absolute store path. -tarhash=$( - basename -a "$@" | - tar --create --preserve-permissions --absolute-names nix \ - --directory /nix/store --verbatim-files-from --files-from - \ - --hard-dereference --sort=name \ - --mtime="@$SOURCE_DATE_EPOCH" \ - --owner=0 --group=0 \ - --transform 's,^nix$,/\0,' \ - --transform 's,^nix/store$,/\0,' \ - --transform 's,^[^/],/nix/store/\0,rS' | - tee "$layerPath/layer.tar" | - tarsum -) - -# Add a 'checksum' field to the JSON, with the value set to the -# checksum of the tarball. -cat ./generic.json | jshon -s "$tarhash" -i checksum > $layerPath/json - -# Indicate to docker that we're using schema version 1.0. -echo -n "1.0" > $layerPath/VERSION diff --git a/pkgs/build-support/docker/stream_layered_image.py b/pkgs/build-support/docker/stream_layered_image.py new file mode 100644 index 0000000000000..9a13878a783df --- /dev/null +++ b/pkgs/build-support/docker/stream_layered_image.py @@ -0,0 +1,309 @@ +""" +This script generates a Docker image from a set of store paths. Uses +Docker Image Specification v1.2 as reference [1]. + +It expects a JSON file with the following properties and writes the +image as an uncompressed tarball to stdout: + +* "architecture", "config", "os", "created", "repo_tag" correspond to + the fields with the same name on the image spec [2]. +* "created" can be "now". +* "created" is also used as mtime for files added to the image. +* "store_layers" is a list of layers in ascending order, where each + layer is the list of store paths to include in that layer. + +The main challenge for this script to create the final image in a +streaming fashion, without dumping any intermediate data to disk +for performance. + +A docker image has each layer contents archived as separate tarballs, +and they later all get enveloped into a single big tarball in a +content addressed fashion. However, because how "tar" format works, +we have to know about the name (which includes the checksum in our +case) and the size of the tarball before we can start adding it to the +outer tarball. We achieve that by creating the layer tarballs twice; +on the first iteration we calculate the file size and the checksum, +and on the second one we actually stream the contents. 'add_layer_dir' +function does all this. + +[1]: https://github.com/moby/moby/blob/master/image/spec/v1.2.md +[2]: https://github.com/moby/moby/blob/4fb59c20a4fb54f944fe170d0ff1d00eb4a24d6f/image/spec/v1.2.md#image-json-field-descriptions +""" # noqa: E501 + + +import io +import os +import re +import sys +import json +import hashlib +import pathlib +import tarfile +import itertools +import threading +from datetime import datetime +from collections import namedtuple + + +def archive_paths_to(obj, paths, mtime, add_nix, filter=None): + """ + Writes the given store paths as a tar file to the given stream. + + obj: Stream to write to. Should have a 'write' method. + paths: List of store paths. + add_nix: Whether /nix and /nix/store directories should be + prepended to the archive. + filter: An optional transformation to be applied to TarInfo + objects. Should take a single TarInfo object and return + another one. Defaults to identity. + """ + + filter = filter if filter else lambda i: i + + # gettarinfo makes the paths relative, this makes them + # absolute again + def append_root(ti): + ti.name = "/" + ti.name + return ti + + def apply_filters(ti): + ti.mtime = mtime + ti.uid = 0 + ti.gid = 0 + ti.uname = "root" + ti.gname = "root" + return filter(ti) + + def dir(path): + ti = tarfile.TarInfo(path) + ti.type = tarfile.DIRTYPE + return ti + + with tarfile.open(fileobj=obj, mode="w|") as tar: + # To be consistent with the docker utilities, we need to have + # these directories first when building layer tarballs. But + # we don't need them on the customisation layer. + if add_nix: + tar.addfile(apply_filters(dir("/nix"))) + tar.addfile(apply_filters(dir("/nix/store"))) + + for path in paths: + path = pathlib.Path(path) + files = itertools.chain([path], path.rglob("*")) + for filename in sorted(files): + ti = append_root(tar.gettarinfo(filename)) + + # copy hardlinks as regular files + if ti.islnk(): + ti.type = tarfile.REGTYPE + ti.linkname = "" + ti.size = filename.stat().st_size + + ti = apply_filters(ti) + if ti.isfile(): + with open(filename, "rb") as f: + tar.addfile(ti, f) + else: + tar.addfile(ti) + + +class ExtractChecksum: + """ + A writable stream which only calculates the final file size and + sha256sum, while discarding the actual contents. + """ + + def __init__(self): + self._digest = hashlib.sha256() + self._size = 0 + + def write(self, data): + self._digest.update(data) + self._size += len(data) + + def extract(self): + """ + Returns: Hex-encoded sha256sum and size as a tuple. + """ + return (self._digest.hexdigest(), self._size) + + +# Some metadata for a layer +LayerInfo = namedtuple("LayerInfo", ["size", "checksum", "path", "paths"]) + + +def add_layer_dir(tar, paths, mtime, add_nix=True, filter=None): + """ + Appends given store paths to a TarFile object as a new layer. + + tar: 'tarfile.TarFile' object for the new layer to be added to. + paths: List of store paths. + mtime: 'mtime' of the added files and the layer tarball. + Should be an integer representing a POSIX time. + add_nix: Whether /nix and /nix/store directories should be + added to a layer. + filter: An optional transformation to be applied to TarInfo + objects inside the layer. Should take a single TarInfo + object and return another one. Defaults to identity. + + Returns: A 'LayerInfo' object containing some metadata of + the layer added. + """ + + invalid_paths = [i for i in paths if not i.startswith("/nix/store/")] + assert len(invalid_paths) == 0, \ + "Expecting absolute store paths, but got: {invalid_paths}" + + # First, calculate the tarball checksum and the size. + extract_checksum = ExtractChecksum() + archive_paths_to( + extract_checksum, + paths, + mtime=mtime, + add_nix=add_nix, + filter=filter + ) + (checksum, size) = extract_checksum.extract() + + path = f"{checksum}/layer.tar" + layer_tarinfo = tarfile.TarInfo(path) + layer_tarinfo.size = size + layer_tarinfo.mtime = mtime + + # Then actually stream the contents to the outer tarball. + read_fd, write_fd = os.pipe() + with open(read_fd, "rb") as read, open(write_fd, "wb") as write: + def producer(): + archive_paths_to( + write, + paths, + mtime=mtime, + add_nix=add_nix, + filter=filter + ) + write.close() + + # Closing the write end of the fifo also closes the read end, + # so we don't need to wait until this thread is finished. + # + # Any exception from the thread will get printed by the default + # exception handler, and the 'addfile' call will fail since it + # won't be able to read required amount of bytes. + threading.Thread(target=producer).start() + tar.addfile(layer_tarinfo, read) + + return LayerInfo(size=size, checksum=checksum, path=path, paths=paths) + + +def add_customisation_layer(tar, path, mtime): + """ + Adds the contents of the store path as a new layer. This is different + than the 'add_layer_dir' function defaults in the sense that the contents + of a single store path will be added to the root of the layer. eg (without + the /nix/store prefix). + + tar: 'tarfile.TarFile' object for the new layer to be added to. + path: A store path. + mtime: 'mtime' of the added files and the layer tarball. Should be an + integer representing a POSIX time. + """ + + def filter(ti): + ti.name = re.sub("^/nix/store/[^/]*", "", ti.name) + return ti + return add_layer_dir( + tar, + [path], + mtime=mtime, + add_nix=False, + filter=filter + ) + + +def add_bytes(tar, path, content, mtime): + """ + Adds a file to the tarball with given path and contents. + + tar: 'tarfile.TarFile' object. + path: Path of the file as a string. + content: Contents of the file. + mtime: 'mtime' of the file. Should be an integer representing a POSIX time. + """ + assert type(content) is bytes + + ti = tarfile.TarInfo(path) + ti.size = len(content) + ti.mtime = mtime + tar.addfile(ti, io.BytesIO(content)) + + +def main(): + with open(sys.argv[1], "r") as f: + conf = json.load(f) + + created = ( + datetime.now(tz=datetime.timezone.utc) + if conf["created"] == "now" + else datetime.fromisoformat(conf["created"]) + ) + mtime = int(created.timestamp()) + + with tarfile.open(mode="w|", fileobj=sys.stdout.buffer) as tar: + layers = [] + for num, store_layer in enumerate(conf["store_layers"]): + print( + "Creating layer", num, + "from paths:", store_layer, + file=sys.stderr) + info = add_layer_dir(tar, store_layer, mtime=mtime) + layers.append(info) + + print("Creating the customisation layer...", file=sys.stderr) + layers.append( + add_customisation_layer( + tar, + conf["customisation_layer"], + mtime=mtime + ) + ) + + print("Adding manifests...", file=sys.stderr) + + image_json = { + "created": datetime.isoformat(created), + "architecture": conf["architecture"], + "os": "linux", + "config": conf["config"], + "rootfs": { + "diff_ids": [f"sha256:{layer.checksum}" for layer in layers], + "type": "layers", + }, + "history": [ + { + "created": conf["created"], + "comment": f"store paths: {layer.paths}" + } + for layer in layers + ], + } + + image_json = json.dumps(image_json, indent=4).encode("utf-8") + image_json_checksum = hashlib.sha256(image_json).hexdigest() + image_json_path = f"{image_json_checksum}.json" + add_bytes(tar, image_json_path, image_json, mtime=mtime) + + manifest_json = [ + { + "Config": image_json_path, + "RepoTags": [conf["repo_tag"]], + "Layers": [layer.path for layer in layers], + } + ] + manifest_json = json.dumps(manifest_json, indent=4).encode("utf-8") + add_bytes(tar, "manifest.json", manifest_json, mtime=mtime) + + print("Done.", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/pkgs/build-support/dotnetbuildhelpers/default.nix b/pkgs/build-support/dotnetbuildhelpers/default.nix index 0edfd0b467ab5..809619ed55d93 100644 --- a/pkgs/build-support/dotnetbuildhelpers/default.nix +++ b/pkgs/build-support/dotnetbuildhelpers/default.nix @@ -12,7 +12,7 @@ cp -v "$script" "$target"/"$scriptName" chmod 755 "$target"/"$scriptName" patchShebangs "$target"/"$scriptName" - substituteInPlace "$target"/"$scriptName" --replace pkg-config ${pkgconfig}/bin/pkg-config + substituteInPlace "$target"/"$scriptName" --replace pkg-config ${pkgconfig}/bin/${pkgconfig.targetPrefix}pkg-config substituteInPlace "$target"/"$scriptName" --replace monodis ${mono}/bin/monodis done '' diff --git a/pkgs/build-support/emacs/wrapper.nix b/pkgs/build-support/emacs/wrapper.nix index 25f068cd3fa08..1f2fbd8068e7b 100644 --- a/pkgs/build-support/emacs/wrapper.nix +++ b/pkgs/build-support/emacs/wrapper.nix @@ -174,7 +174,7 @@ runCommand mkdir -p $out/share # Link icons and desktop files into place - for dir in applications icons info man; do + for dir in applications icons info man emacs; do ln -s $emacs/share/$dir $out/share/$dir done '' diff --git a/pkgs/build-support/fetchmavenartifact/default.nix b/pkgs/build-support/fetchmavenartifact/default.nix index d7ad406943be5..583a9ea396cfa 100644 --- a/pkgs/build-support/fetchmavenartifact/default.nix +++ b/pkgs/build-support/fetchmavenartifact/default.nix @@ -3,7 +3,7 @@ { fetchurl, stdenv }: let defaultRepos = [ - "http://central.maven.org/maven2" + "http://repo1.maven.org/maven2" "http://oss.sonatype.org/content/repositories/releases" "http://oss.sonatype.org/content/repositories/public" "http://repo.typesafe.com/typesafe/releases" diff --git a/pkgs/build-support/fetchurl/mirrors.nix b/pkgs/build-support/fetchurl/mirrors.nix index 4e8b0aa99a8f1..0eba8816b6387 100644 --- a/pkgs/build-support/fetchurl/mirrors.nix +++ b/pkgs/build-support/fetchurl/mirrors.nix @@ -426,7 +426,6 @@ # Maven Central maven = [ "https://repo1.maven.org/maven2/" - "https://central.maven.org/maven2/" ]; # Alsa Project diff --git a/pkgs/build-support/ocaml/oasis.nix b/pkgs/build-support/ocaml/oasis.nix index 74977486f2fb1..c1d1d699765bd 100644 --- a/pkgs/build-support/ocaml/oasis.nix +++ b/pkgs/build-support/ocaml/oasis.nix @@ -23,7 +23,7 @@ stdenv.mkDerivation (args // { buildPhase = '' runHook preBuild oasis setup - ocaml setup.ml -configure + ocaml setup.ml -configure --prefix $OCAMLFIND_DESTDIR --exec-prefix $out ocaml setup.ml -build runHook postBuild ''; @@ -37,9 +37,7 @@ stdenv.mkDerivation (args // { installPhase = '' runHook preInstall mkdir -p $out - sed -i s+/usr/local+$out+g setup.ml - sed -i s+/usr/local+$out+g setup.data - prefix=$OCAMLFIND_DESTDIR ocaml setup.ml -install + ocaml setup.ml -install runHook postInstall ''; diff --git a/pkgs/build-support/pkg-config-wrapper/add-flags.sh b/pkgs/build-support/pkg-config-wrapper/add-flags.sh new file mode 100644 index 0000000000000..35ecf62ca230f --- /dev/null +++ b/pkgs/build-support/pkg-config-wrapper/add-flags.sh @@ -0,0 +1,12 @@ +# See cc-wrapper for comments. +var_templates_list=( + PKG_CONFIG_PATH +) + +accumulateRoles + +for var in "${var_templates_list[@]}"; do + mangleVarList "$var" ${role_suffixes[@]+"${role_suffixes[@]}"} +done + +export NIX_PKG_CONFIG_WRAPPER_FLAGS_SET_@suffixSalt@=1 diff --git a/pkgs/build-support/pkg-config-wrapper/default.nix b/pkgs/build-support/pkg-config-wrapper/default.nix new file mode 100644 index 0000000000000..e01df107dd17f --- /dev/null +++ b/pkgs/build-support/pkg-config-wrapper/default.nix @@ -0,0 +1,126 @@ +# The wrapper script ensures variables like PKG_CONFIG_PATH and +# PKG_CONFIG_PATH_FOR_BUILD work properly. + +{ stdenvNoCC +, buildPackages +, pkg-config +, baseBinName ? "pkg-config" +, propagateDoc ? pkg-config != null && pkg-config ? man +, extraPackages ? [], extraBuildCommands ? "" +}: + +with stdenvNoCC.lib; + +let + stdenv = stdenvNoCC; + inherit (stdenv) hostPlatform targetPlatform; + + # Prefix for binaries. Customarily ends with a dash separator. + # + # TODO(@Ericson2314) Make unconditional, or optional but always true by + # default. + targetPrefix = stdenv.lib.optionalString (targetPlatform != hostPlatform) + (targetPlatform.config + "-"); + + # See description in cc-wrapper. + suffixSalt = replaceStrings ["-" "."] ["_" "_"] targetPlatform.config; + +in + +stdenv.mkDerivation { + pname = targetPrefix + pkg-config.pname + "-wrapper"; + inherit (pkg-config) version; + + preferLocalBuild = true; + + shell = getBin stdenvNoCC.shell + stdenvNoCC.shell.shellPath or ""; + + inherit targetPrefix suffixSalt baseBinName; + + outputs = [ "out" ] ++ optionals propagateDoc ([ "man" ] ++ optional (pkg-config ? doc) "doc"); + + passthru = { + inherit pkg-config; + }; + + dontBuild = true; + dontConfigure = true; + + unpackPhase = '' + src=$PWD + ''; + + installPhase = + '' + mkdir -p $out/bin $out/nix-support + + wrap() { + local dst="$1" + local wrapper="$2" + export prog="$3" + substituteAll "$wrapper" "$out/bin/$dst" + chmod +x "$out/bin/$dst" + } + + echo $pkg-config > $out/nix-support/orig-pkg-config + + wrap ${targetPrefix}${baseBinName} ${./pkg-config-wrapper.sh} "${getBin pkg-config}/bin/${baseBinName}" + '' + # symlink in share for autoconf to find macros + + # TODO(@Ericson2314): in the future just make the unwrapped pkg-config a + # propagated dep once we can rely on downstream deps comming first in + # search paths. (https://github.com/NixOS/nixpkgs/pull/31414 took a crack + # at this.) + + '' + ln -s ${pkg-config}/share $out/share + ''; + + strictDeps = true; + + wrapperName = "PKG_CONFIG_WRAPPER"; + + setupHooks = [ + ../setup-hooks/role.bash + ./setup-hook.sh + ]; + + postFixup = + ## + ## User env support + ## + + # Propagate the underling unwrapped pkg-config so that if you + # install the wrapper, you get anything else it might provide. + '' + printWords ${pkg-config} > $out/nix-support/propagated-user-env-packages + '' + + ## + ## Man page and doc support + ## + + optionalString propagateDoc ('' + ln -s ${pkg-config.man} $man + '' + optionalString (pkg-config ? doc) '' + ln -s ${pkg-config.doc} $doc + '') + + + '' + substituteAll ${./add-flags.sh} $out/nix-support/add-flags.sh + substituteAll ${../wrapper-common/utils.bash} $out/nix-support/utils.bash + '' + + ## + ## Extra custom steps + ## + + extraBuildCommands; + + meta = + let pkg-config_ = if pkg-config != null then pkg-config else {}; in + (if pkg-config_ ? meta then removeAttrs pkg-config.meta ["priority"] else {}) // + { description = + stdenv.lib.attrByPath ["meta" "description"] "pkg-config" pkg-config_ + + " (wrapper script)"; + priority = 10; + }; +} diff --git a/pkgs/build-support/pkg-config-wrapper/pkg-config-wrapper.sh b/pkgs/build-support/pkg-config-wrapper/pkg-config-wrapper.sh new file mode 100644 index 0000000000000..f7c7429eb0b3e --- /dev/null +++ b/pkgs/build-support/pkg-config-wrapper/pkg-config-wrapper.sh @@ -0,0 +1,21 @@ +#! @shell@ +set -eu -o pipefail +o posix +shopt -s nullglob + +if (( "${NIX_DEBUG:-0}" >= 7 )); then + set -x +fi + +source @out@/nix-support/utils.bash + +if [ -z "${NIX_PKG_CONFIG_WRAPPER_FLAGS_SET_@suffixSalt@:-}" ]; then + source @out@/nix-support/add-flags.sh +fi + +if (( ${#role_suffixes[@]} > 0 )); then + # replace env var with nix-modified one + PKG_CONFIG_PATH=$PKG_CONFIG_PATH_@suffixSalt@ exec @prog@ "$@" +else + # pkg-config isn't a bonafied dependency so ignore setup hook entirely + exec @prog@ "$@" +fi diff --git a/pkgs/build-support/pkg-config-wrapper/setup-hook.sh b/pkgs/build-support/pkg-config-wrapper/setup-hook.sh new file mode 100644 index 0000000000000..34f1a999a82ee --- /dev/null +++ b/pkgs/build-support/pkg-config-wrapper/setup-hook.sh @@ -0,0 +1,29 @@ +# pkg-config Wrapper hygiene +# +# See comments in cc-wrapper's setup hook. This works exactly the same way. + +# Skip setup hook if we're neither a build-time dep, nor, temporarily, doing a +# native compile. +# +# TODO(@Ericson2314): No native exception +[[ -z ${strictDeps-} ]] || (( "$hostOffset" < 0 )) || return 0 + +pkgConfigWrapper_addPkgConfigPath () { + # See ../setup-hooks/role.bash + local role_post + getHostRoleEnvHook + + addToSearchPath "PKG_CONFIG_PATH${role_post}" "$1/lib/pkgconfig" + addToSearchPath "PKG_CONFIG_PATH${role_post}" "$1/share/pkgconfig" +} + +# See ../setup-hooks/role.bash +getTargetRole +getTargetRoleWrapper + +addEnvHooks "$targetOffset" pkgConfigWrapper_addPkgConfigPath + +export PKG_CONFIG${role_post}=@targetPrefix@@baseBinName@ + +# No local scope in sourced file +unset -v role_post diff --git a/pkgs/build-support/rust/build-rust-crate/build-crate.nix b/pkgs/build-support/rust/build-rust-crate/build-crate.nix index d6d1cebb2e91b..f82effdbca7ca 100644 --- a/pkgs/build-support/rust/build-rust-crate/build-crate.nix +++ b/pkgs/build-support/rust/build-rust-crate/build-crate.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, mkRustcDepArgs, rust }: +{ lib, stdenv, mkRustcDepArgs, mkRustcFeatureArgs, rust }: { crateName, dependencies, crateFeatures, crateRenames, libName, release, libPath, @@ -13,7 +13,7 @@ ++ ["-C codegen-units=$NIX_BUILD_CORES"] ++ ["--remap-path-prefix=$NIX_BUILD_TOP=/" ] ++ [(mkRustcDepArgs dependencies crateRenames)] - ++ [crateFeatures] + ++ [(mkRustcFeatureArgs crateFeatures)] ++ extraRustcOpts ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "--target ${rust.toRustTarget stdenv.hostPlatform} -C linker=${stdenv.hostPlatform.config}-gcc" # since rustc 1.42 the "proc_macro" crate is part of the default crate prelude diff --git a/pkgs/build-support/rust/build-rust-crate/configure-crate.nix b/pkgs/build-support/rust/build-rust-crate/configure-crate.nix index c47bc00938c19..8e2f5f7f35e03 100644 --- a/pkgs/build-support/rust/build-rust-crate/configure-crate.nix +++ b/pkgs/build-support/rust/build-rust-crate/configure-crate.nix @@ -1,5 +1,6 @@ -{ lib, stdenv, echo_colored, noisily, mkRustcDepArgs }: -{ build +{ lib, stdenv, echo_colored, noisily, mkRustcDepArgs, mkRustcFeatureArgs }: +{ + build , buildDependencies , colors , completeBuildDeps @@ -30,6 +31,9 @@ let version_ = lib.splitString "-" crateVersion; optLevel = if release then 3 else 0; completeDepsDir = lib.concatStringsSep " " completeDeps; completeBuildDepsDir = lib.concatStringsSep " " completeBuildDeps; + envFeatures = lib.concatStringsSep " " ( + map (f: lib.replaceChars ["-"] ["_"] (lib.toUpper f)) crateFeatures + ); in '' ${echo_colored colors} ${noisily colors verbose} @@ -161,14 +165,24 @@ in '' EXTRA_BUILD_FLAGS="$EXTRA_BUILD_FLAGS $(tr '\n' ' ' < target/link.build)" fi noisily rustc --crate-name build_script_build $BUILD --crate-type bin ${rustcOpts} \ - ${crateFeatures} --out-dir target/build/${crateName} --emit=dep-info,link \ + ${mkRustcFeatureArgs crateFeatures} --out-dir target/build/${crateName} --emit=dep-info,link \ -L dependency=target/buildDeps ${buildDeps} --cap-lints allow $EXTRA_BUILD_FLAGS --color ${colors} mkdir -p target/build/${crateName}.out export RUST_BACKTRACE=1 BUILD_OUT_DIR="-L $OUT_DIR" mkdir -p $OUT_DIR - target/build/${crateName}/build_script_build > target/build/${crateName}.opt + + ( + # Features should be set as environment variable for build scripts: + # https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts + for feature in ${envFeatures}; do + export CARGO_FEATURE_$feature=1 + done + + target/build/${crateName}/build_script_build > target/build/${crateName}.opt + ) + set +e EXTRA_BUILD=$(sed -n "s/^cargo:rustc-flags=\(.*\)/\1/p" target/build/${crateName}.opt | tr '\n' ' ' | sort -u) EXTRA_FEATURES=$(sed -n "s/^cargo:rustc-cfg=\(.*\)/--cfg \1/p" target/build/${crateName}.opt | tr '\n' ' ') diff --git a/pkgs/build-support/rust/build-rust-crate/default.nix b/pkgs/build-support/rust/build-rust-crate/default.nix index d9ed26f1d94a7..d559aba161658 100644 --- a/pkgs/build-support/rust/build-rust-crate/default.nix +++ b/pkgs/build-support/rust/build-rust-crate/default.nix @@ -45,14 +45,17 @@ let " --extern ${name}=${dep.lib}/lib/lib${extern}-${dep.metadata}${stdenv.hostPlatform.extensions.sharedLibrary}") ) dependencies; + # Create feature arguments for rustc. + mkRustcFeatureArgs = lib.concatMapStringsSep " " (f: ''--cfg feature=\"${f}\"''); + inherit (import ./log.nix { inherit lib; }) noisily echo_colored; configureCrate = import ./configure-crate.nix { - inherit lib stdenv echo_colored noisily mkRustcDepArgs; + inherit lib stdenv echo_colored noisily mkRustcDepArgs mkRustcFeatureArgs; }; buildCrate = import ./build-crate.nix { - inherit lib stdenv mkRustcDepArgs rust; + inherit lib stdenv mkRustcDepArgs mkRustcFeatureArgs rust; }; installCrate = import ./install-crate.nix { inherit stdenv; }; @@ -233,8 +236,11 @@ stdenv.mkDerivation (rec { ++ lib.concatMap (dep: dep.completeBuildDeps ++ dep.completeDeps) buildDependencies ); - crateFeatures = lib.optionalString (crate ? features) - (lib.concatMapStringsSep " " (f: ''--cfg feature=\"${f}\"'') (crate.features ++ features)); + # Create a list of features that are enabled by the crate itself and + # through the features argument of buildRustCrate. Exclude features + # with a forward slash, since they are passed through to dependencies. + crateFeatures = lib.optionals (crate ? features) + (builtins.filter (f: !lib.hasInfix "/" f) (crate.features ++ features)); libName = if crate ? libName then crate.libName else crate.crateName; libPath = if crate ? libPath then crate.libPath else ""; @@ -244,7 +250,8 @@ stdenv.mkDerivation (rec { metadata = let depsMetadata = lib.foldl' (str: dep: str + dep.metadata) "" (dependencies ++ buildDependencies); hashedMetadata = builtins.hashString "sha256" - (crateName + "-" + crateVersion + "___" + toString crateFeatures + "___" + depsMetadata); + (crateName + "-" + crateVersion + "___" + toString (mkRustcFeatureArgs crateFeatures) + + "___" + depsMetadata); in lib.substring 0 10 hashedMetadata; build = crate.build or ""; diff --git a/pkgs/build-support/rust/build-rust-crate/test/default.nix b/pkgs/build-support/rust/build-rust-crate/test/default.nix index bdd6c86d5f07e..24ddc11459ec4 100644 --- a/pkgs/build-support/rust/build-rust-crate/test/default.nix +++ b/pkgs/build-support/rust/build-rust-crate/test/default.nix @@ -344,6 +344,32 @@ let buildTests = true; expectedTestOutputs = [ "test baz_false ... ok" ]; }; + buildScriptFeatureEnv = { + crateName = "build-script-feature-env"; + features = [ "some-feature" "crate/another_feature" ]; + src = symlinkJoin { + name = "build-script-feature-env"; + paths = [ + (mkFile "src/main.rs" '' + #[cfg(test)] + #[test] + fn feature_not_visible() { + assert!(std::env::var("CARGO_FEATURE_SOME_FEATURE").is_err()); + assert!(option_env!("CARGO_FEATURE_SOME_FEATURE").is_none()); + } + fn main() {} + '') + (mkFile "build.rs" '' + fn main() { + assert!(std::env::var("CARGO_FEATURE_SOME_FEATURE").is_ok()); + assert!(option_env!("CARGO_FEATURE_SOME_FEATURE").is_none()); + } + '') + ]; + }; + buildTests = true; + expectedTestOutputs = [ "test feature_not_visible ... ok" ]; + }; # Regression test for https://github.com/NixOS/nixpkgs/pull/88054 # Build script output should be rewritten as valid env vars. buildScriptIncludeDirDeps = let diff --git a/pkgs/build-support/rust/default.nix b/pkgs/build-support/rust/default.nix index 770a6d1042e7e..8d3a7ba6929c1 100644 --- a/pkgs/build-support/rust/default.nix +++ b/pkgs/build-support/rust/default.nix @@ -28,6 +28,13 @@ , meta ? {} , target ? null , cargoVendorDir ? null +, checkType ? buildType + +# Needed to `pushd`/`popd` into a subdir of a tarball if this subdir +# contains a Cargo.toml, but isn't part of a workspace (which is e.g. the +# case for `rustfmt`/etc from the `rust-sources). +# Otherwise, everything from the tarball would've been built/tested. +, buildAndTestSubdir ? null , ... } @ args: assert cargoVendorDir == null -> cargoSha256 != "unset"; @@ -163,6 +170,7 @@ stdenv.mkDerivation (args // { ''; buildPhase = with builtins; args.buildPhase or '' + ${stdenv.lib.optionalString (buildAndTestSubdir != null) "pushd ${buildAndTestSubdir}"} runHook preBuild ( @@ -178,22 +186,29 @@ stdenv.mkDerivation (args // { --frozen ${concatStringsSep " " cargoBuildFlags} ) - # rename the output dir to a architecture independent one - mapfile -t targets < <(find "$NIX_BUILD_TOP" -type d | grep '${releaseDir}$') - for target in "''${targets[@]}"; do - rm -rf "$target/../../${buildType}" - ln -srf "$target" "$target/../../" - done - runHook postBuild + + ${stdenv.lib.optionalString (buildAndTestSubdir != null) "popd"} + + # This needs to be done after postBuild: packages like `cargo` do a pushd/popd in + # the pre/postBuild-hooks that need to be taken into account before gathering + # all binaries to install. + bins=$(find $releaseDir \ + -maxdepth 1 \ + -type f \ + -executable ! \( -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \)) ''; - checkPhase = args.checkPhase or '' + checkPhase = args.checkPhase or (let + argstr = "${stdenv.lib.optionalString (checkType == "release") "--release"} --target ${rustTarget} --frozen"; + in '' + ${stdenv.lib.optionalString (buildAndTestSubdir != null) "pushd ${buildAndTestSubdir}"} runHook preCheck - echo "Running cargo cargo test -- ''${checkFlags} ''${checkFlagsArray+''${checkFlagsArray[@]}}" - cargo test -- ''${checkFlags} ''${checkFlagsArray+"''${checkFlagsArray[@]}"} + echo "Running cargo test ${argstr} -- ''${checkFlags} ''${checkFlagsArray+''${checkFlagsArray[@]}}" + cargo test ${argstr} -- ''${checkFlags} ''${checkFlagsArray+"''${checkFlagsArray[@]}"} runHook postCheck - ''; + ${stdenv.lib.optionalString (buildAndTestSubdir != null) "popd"} + ''); doCheck = args.doCheck or true; @@ -203,13 +218,16 @@ stdenv.mkDerivation (args // { installPhase = args.installPhase or '' runHook preInstall + + # rename the output dir to a architecture independent one + mapfile -t targets < <(find "$NIX_BUILD_TOP" -type d | grep '${releaseDir}$') + for target in "''${targets[@]}"; do + rm -rf "$target/../../${buildType}" + ln -srf "$target" "$target/../../" + done mkdir -p $out/bin $out/lib - find $releaseDir \ - -maxdepth 1 \ - -type f \ - -executable ! \( -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \) \ - -print0 | xargs -r -0 cp -t $out/bin + xargs -r cp -t $out/bin <<< $bins find $releaseDir \ -maxdepth 1 \ -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \ diff --git a/pkgs/build-support/setup-hooks/role.bash b/pkgs/build-support/setup-hooks/role.bash index 6f1c36f5c050d..cf69e732e7c3d 100644 --- a/pkgs/build-support/setup-hooks/role.bash +++ b/pkgs/build-support/setup-hooks/role.bash @@ -3,21 +3,17 @@ # derivation) in which the derivation is used. # # The role is intened to be use as part of other variables names like -# - $NIX_${role_pre}_SOMETHING -# - $NIX_SOMETHING_${role_post} +# - $NIX_SOMETHING${role_post} function getRole() { case $1 in -1) - role_pre='BUILD_' role_post='_FOR_BUILD' ;; 0) - role_pre='' role_post='' ;; 1) - role_pre='TARGET_' role_post='_FOR_TARGET' ;; *) @@ -54,18 +50,18 @@ function getTargetRoleEnvHook() { } # This variant is inteneded specifically for code-prodocing tool wrapper scripts -# `NIX_@wrapperName@_@infixSalt@_TARGET_*` tracks this (needs to be an exported +# `NIX_@wrapperName@_TARGET_*_@suffixSalt@` tracks this (needs to be an exported # env var so can't use fancier data structures). function getTargetRoleWrapper() { case $targetOffset in -1) - export NIX_@wrapperName@_@infixSalt@_TARGET_BUILD=1 + export NIX_@wrapperName@_TARGET_BUILD_@suffixSalt@=1 ;; 0) - export NIX_@wrapperName@_@infixSalt@_TARGET_HOST=1 + export NIX_@wrapperName@_TARGET_HOST_@suffixSalt@=1 ;; 1) - export NIX_@wrapperName@_@infixSalt@_TARGET_TARGET=1 + export NIX_@wrapperName@_TARGET_TARGET_@suffixSalt@=1 ;; *) echo "@name@: used as improper sort of dependency" >2 diff --git a/pkgs/build-support/wrapper-common/utils.bash b/pkgs/build-support/wrapper-common/utils.bash index 4fd57162072e4..8c4680a8e4460 100644 --- a/pkgs/build-support/wrapper-common/utils.bash +++ b/pkgs/build-support/wrapper-common/utils.bash @@ -1,29 +1,29 @@ -# Accumulate infixes for taking in the right input parameters with the `mangle*` +# Accumulate suffixes for taking in the right input parameters with the `mangle*` # functions below. See setup-hook for details. accumulateRoles() { - declare -ga role_infixes=() - if [ "${NIX_@wrapperName@_@infixSalt@_TARGET_BUILD:-}" ]; then - role_infixes+=(_BUILD_) + declare -ga role_suffixes=() + if [ "${NIX_@wrapperName@_TARGET_BUILD_@suffixSalt@:-}" ]; then + role_suffixes+=('_FOR_BUILD') fi - if [ "${NIX_@wrapperName@_@infixSalt@_TARGET_HOST:-}" ]; then - role_infixes+=(_) + if [ "${NIX_@wrapperName@_TARGET_HOST_@suffixSalt@:-}" ]; then + role_suffixes+=('') fi - if [ "${NIX_@wrapperName@_@infixSalt@_TARGET_TARGET:-}" ]; then - role_infixes+=(_TARGET_) + if [ "${NIX_@wrapperName@_TARGET_TARGET_@suffixSalt@:-}" ]; then + role_suffixes+=('_FOR_TARGET') fi } mangleVarList() { local var="$1" shift - local -a role_infixes=("$@") + local -a role_suffixes=("$@") - local outputVar="${var/+/_@infixSalt@_}" + local outputVar="${var}_@suffixSalt@" declare -gx ${outputVar}+='' # For each role we serve, we accumulate the input parameters into our own # cc-wrapper-derivation-specific environment variables. - for infix in "${role_infixes[@]}"; do - local inputVar="${var/+/${infix}}" + for suffix in "${role_suffixes[@]}"; do + local inputVar="${var}${suffix}" if [ -v "$inputVar" ]; then export ${outputVar}+="${!outputVar:+ }${!inputVar}" fi @@ -33,12 +33,12 @@ mangleVarList() { mangleVarBool() { local var="$1" shift - local -a role_infixes=("$@") + local -a role_suffixes=("$@") - local outputVar="${var/+/_@infixSalt@_}" + local outputVar="${var}_@suffixSalt@" declare -gxi ${outputVar}+=0 - for infix in "${role_infixes[@]}"; do - local inputVar="${var/+/${infix}}" + for suffix in "${role_suffixes[@]}"; do + local inputVar="${var}${suffix}" if [ -v "$inputVar" ]; then # "1" in the end makes `let` return success error code when # expression itself evaluates to zero. |