diff options
author | Franz Pletz <fpletz@fnordicwalking.de> | 2018-03-04 17:10:27 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-03-04 17:10:27 +0000 |
commit | 0f78afdf253e3f53e76ec3379518d208a18c568c (patch) | |
tree | 7524e34c1ac817f8d5d507b7dbc07b6a87d8eac5 /pkgs/build-support | |
parent | 431d05cff73138babc64e0ef95ee0eaa8b9084d7 (diff) | |
parent | 58895aa1cb752198b8ab89aacfe51e6a8a69e326 (diff) |
Merge pull request #32248 from awakesecurity/parnell/fetchdocker
Support fetching docker images from V2 registries
Diffstat (limited to 'pkgs/build-support')
-rw-r--r-- | pkgs/build-support/fetchdocker/credentials.nix | 38 | ||||
-rw-r--r-- | pkgs/build-support/fetchdocker/default.nix | 61 | ||||
-rw-r--r-- | pkgs/build-support/fetchdocker/fetchDockerConfig.nix | 13 | ||||
-rw-r--r-- | pkgs/build-support/fetchdocker/fetchDockerLayer.nix | 13 | ||||
-rw-r--r-- | pkgs/build-support/fetchdocker/fetchdocker-builder.sh | 28 | ||||
-rw-r--r-- | pkgs/build-support/fetchdocker/generic-fetcher.nix | 97 |
6 files changed, 250 insertions, 0 deletions
diff --git a/pkgs/build-support/fetchdocker/credentials.nix b/pkgs/build-support/fetchdocker/credentials.nix new file mode 100644 index 0000000000000..c01288dbf53a0 --- /dev/null +++ b/pkgs/build-support/fetchdocker/credentials.nix @@ -0,0 +1,38 @@ +# We provide three paths to get the credentials into the builder's +# environment: +# +# 1. Via impureEnvVars. This method is difficult for multi-user Nix +# installations (but works very well for single-user Nix +# installations!) because it requires setting the environment +# variables on the nix-daemon which is either complicated or unsafe +# (i.e: configuring via Nix means the secrets will be persisted +# into the store) +# +# 2. If the DOCKER_CREDENTIALS key with a path to a credentials file +# is added to the NIX_PATH (usually via the '-I ' argument to most +# Nix tools) then an attempt will be made to read credentials from +# it. The semantics are simple, the file should contain two lines +# for the username and password based authentication: +# +# $ cat ./credentials-file.txt +# DOCKER_USER=myusername +# DOCKER_PASS=mypassword +# +# ... and a single line for the token based authentication: +# +# $ cat ./credentials-file.txt +# DOCKER_TOKEN=mytoken +# +# 3. A credential file at /etc/nix-docker-credentials.txt with the +# same format as the file described in #2 can also be used to +# communicate credentials to the builder. This is necessary for +# situations (like Hydra) where you cannot customize the NIX_PATH +# given to the nix-build invocation to provide it with the +# DOCKER_CREDENTIALS path +let + pathParts = + (builtins.filter + ({path, prefix}: "DOCKER_CREDENTIALS" == prefix) + builtins.nixPath); +in + if (pathParts != []) then (builtins.head pathParts).path else "" diff --git a/pkgs/build-support/fetchdocker/default.nix b/pkgs/build-support/fetchdocker/default.nix new file mode 100644 index 0000000000000..ae3ae4185e053 --- /dev/null +++ b/pkgs/build-support/fetchdocker/default.nix @@ -0,0 +1,61 @@ +{ stdenv, lib, coreutils, bash, gnutar, jq, writeText }: +let + stripScheme = + builtins.replaceStrings [ "https://" "http://" ] [ "" "" ]; + stripNixStore = + s: lib.removePrefix "/nix/store/" s; +in +{ name +, registry ? "https://registry-1.docker.io/v2/" +, repository ? "library" +, imageName +, tag +, imageLayers +, imageConfig +, image ? "${stripScheme registry}/${repository}/${imageName}:${tag}" +}: + +# Make sure there are *no* slashes in the repository or container +# names since we use these to make the output derivation name for the +# nix-store path. +assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters repository); +assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters imageName); + +let + # Abuse `builtins.toPath` to collapse possible double slashes + repoTag0 = builtins.toString (builtins.toPath "/${stripScheme registry}/${repository}/${imageName}"); + repoTag1 = lib.removePrefix "/" repoTag0; + + layers = builtins.map stripNixStore imageLayers; + + manifest = + writeText "manifest.json" (builtins.toJSON [ + { Config = stripNixStore imageConfig; + Layers = layers; + RepoTags = [ "${repoTag1}:${tag}" ]; + }]); + + repositories = + writeText "repositories" (builtins.toJSON { + "${repoTag1}" = { + "${tag}" = lib.last layers; + }; + }); + + imageFileStorePaths = + writeText "imageFileStorePaths.txt" + (lib.concatStringsSep "\n" ((lib.unique imageLayers) ++ [imageConfig])); +in +stdenv.mkDerivation { + builder = ./fetchdocker-builder.sh; + buildInputs = [ coreutils ]; + preferLocalBuild = true; + + inherit name imageName repository tag; + inherit bash gnutar manifest repositories; + inherit imageFileStorePaths; + + passthru = { + inherit image; + }; +} diff --git a/pkgs/build-support/fetchdocker/fetchDockerConfig.nix b/pkgs/build-support/fetchdocker/fetchDockerConfig.nix new file mode 100644 index 0000000000000..9fd813bfa575a --- /dev/null +++ b/pkgs/build-support/fetchdocker/fetchDockerConfig.nix @@ -0,0 +1,13 @@ +pkgargs@{ stdenv, lib, haskellPackages, writeText, gawk }: +let + generic-fetcher = + import ./generic-fetcher.nix pkgargs; +in + +args@{ repository ? "library", imageName, tag, ... }: + +generic-fetcher ({ + fetcher = "hocker-config"; + name = "${repository}_${imageName}_${tag}-config.json"; + tag = "unused"; +} // args) diff --git a/pkgs/build-support/fetchdocker/fetchDockerLayer.nix b/pkgs/build-support/fetchdocker/fetchDockerLayer.nix new file mode 100644 index 0000000000000..869ba637429cd --- /dev/null +++ b/pkgs/build-support/fetchdocker/fetchDockerLayer.nix @@ -0,0 +1,13 @@ +pkgargs@{ stdenv, lib, haskellPackages, writeText, gawk }: +let + generic-fetcher = + import ./generic-fetcher.nix pkgargs; +in + +args@{ layerDigest, ... }: + +generic-fetcher ({ + fetcher = "hocker-layer"; + name = "docker-layer-${layerDigest}.tar.gz"; + tag = "unused"; +} // args) diff --git a/pkgs/build-support/fetchdocker/fetchdocker-builder.sh b/pkgs/build-support/fetchdocker/fetchdocker-builder.sh new file mode 100644 index 0000000000000..7443591e6569b --- /dev/null +++ b/pkgs/build-support/fetchdocker/fetchdocker-builder.sh @@ -0,0 +1,28 @@ +source "${stdenv}/setup" +header "exporting ${repository}/${imageName} (tag: ${tag}) into ${out}" +mkdir -p "${out}" + +cat <<EOF > "${out}/compositeImage.sh" +#! ${bash}/bin/bash +# +# Create a tar archive of a docker image's layers, docker image config +# json, manifest.json, and repositories json; this streams directly to +# stdout and is intended to be used in concert with docker load, i.e: +# +# ${out}/compositeImage.sh | docker load + +# The first character follow the 's' command for sed becomes the +# delimiter sed will use; this makes the transformation regex easy to +# read. We feed tar a file listing the files we want in the archive, +# because the paths are absolute and docker load wants them flattened in +# the archive, we need to transform all of the paths going in by +# stripping everything *including* the last solidus so that we end up +# with the basename of the path. +${gnutar}/bin/tar \ + --transform='s=.*/==' \ + --transform="s=.*-manifest.json=manifest.json=" \ + --transform="s=.*-repositories=repositories=" \ + -c "${manifest}" "${repositories}" -T "${imageFileStorePaths}" +EOF +chmod +x "${out}/compositeImage.sh" +stopNest diff --git a/pkgs/build-support/fetchdocker/generic-fetcher.nix b/pkgs/build-support/fetchdocker/generic-fetcher.nix new file mode 100644 index 0000000000000..e051cee08432e --- /dev/null +++ b/pkgs/build-support/fetchdocker/generic-fetcher.nix @@ -0,0 +1,97 @@ +{ stdenv, lib, haskellPackages, writeText, gawk }: +let + awk = "${gawk}/bin/awk"; + dockerCredentialsFile = import ./credentials.nix; + stripScheme = + builtins.replaceStrings [ "https://" "http://" ] [ "" "" ]; +in +{ fetcher +, name + , registry ? "https://registry-1.docker.io/v2/" + , repository ? "library" + , imageName + , sha256 + , tag ? "" + , layerDigest ? "" +}: + +# There must be no slashes in the repository or container names since +# we use these to make the output derivation name for the nix store +# path +assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters repository); +assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters imageName); + +# Only allow hocker-config and hocker-layer as fetchers for now +assert (builtins.elem fetcher ["hocker-config" "hocker-layer"]); + +# If layerDigest is non-empty then it must not have a 'sha256:' prefix! +assert + (if layerDigest != "" + then !lib.hasPrefix "sha256:" layerDigest + else true); + +let + layerDigestFlag = + lib.optionalString (layerDigest != "") "--layer ${layerDigest}"; +in +stdenv.mkDerivation { + inherit name; + builder = writeText "${fetcher}-builder.sh" '' + source "$stdenv/setup" + header "${fetcher} exporting to $out" + + declare -A creds + + # This is a hack for Hydra since we have no way of adding values + # to the NIX_PATH for Hydra jobsets!! + staticCredentialsFile="/etc/nix-docker-credentials.txt" + if [ ! -f "$dockerCredentialsFile" -a -f "$staticCredentialsFile" ]; then + echo "credentials file not set, falling back on static credentials file at: $staticCredentialsFile" + dockerCredentialsFile=$staticCredentialsFile + fi + + if [ -f "$dockerCredentialsFile" ]; then + header "using credentials from $dockerCredentialsFile" + + CREDSFILE=$(cat "$dockerCredentialsFile") + creds[token]=$(${awk} -F'=' '/DOCKER_TOKEN/ {print $2}' <<< "$CREDSFILE" | head -n1) + + # Prefer DOCKER_TOKEN over the username and password + # authentication method + if [ -z "''${creds[token]}" ]; then + creds[user]=$(${awk} -F'=' '/DOCKER_USER/ {print $2}' <<< "$CREDSFILE" | head -n1) + creds[pass]=$(${awk} -F'=' '/DOCKER_PASS/ {print $2}' <<< "$CREDSFILE" | head -n1) + fi + fi + + # These variables will be filled in first by the impureEnvVars, if + # those variables are empty then they will default to the + # credentials that may have been read in from the 'DOCKER_CREDENTIALS' + DOCKER_USER="''${DOCKER_USER:-''${creds[user]}}" + DOCKER_PASS="''${DOCKER_PASS:-''${creds[pass]}}" + DOCKER_TOKEN="''${DOCKER_TOKEN:-''${creds[token]}}" + + ${fetcher} --out="$out" \ + ''${registry:+--registry "$registry"} \ + ''${DOCKER_USER:+--username "$DOCKER_USER"} \ + ''${DOCKER_PASS:+--password "$DOCKER_PASS"} \ + ''${DOCKER_TOKEN:+--token "$DOCKER_TOKEN"} \ + ${layerDigestFlag} \ + "${repository}/${imageName}" \ + "${tag}" + + stopNest + ''; + + buildInputs = [ haskellPackages.hocker ]; + + outputHashAlgo = "sha256"; + outputHashMode = "flat"; + outputHash = sha256; + + preferLocalBuild = true; + + impureEnvVars = [ "DOCKER_USER" "DOCKER_PASS" "DOCKER_TOKEN" ]; + + inherit registry dockerCredentialsFile; +} |