diff options
author | Graham Christensen <graham@grahamc.com> | 2017-07-25 22:17:41 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-07-25 22:17:41 -0400 |
commit | 3ffcbdd0f0fcb227c87c2d182b4dab75bd66b85a (patch) | |
tree | f09d987f1bd0ac61a52928a577b4a01d3a84130d /maintainers | |
parent | 318189f26febf90a6fa8e506e93ddbf37331a09f (diff) | |
parent | 90acbe5449268dc4e1b344b420cbb3b820e4c37a (diff) |
Merge branch 'master' into patch-3
Diffstat (limited to 'maintainers')
-rw-r--r-- | maintainers/docker/.dockerignore | 14 | ||||
-rw-r--r-- | maintainers/docker/Dockerfile | 12 | ||||
-rw-r--r-- | maintainers/scripts/all-tarballs.nix | 2 | ||||
-rwxr-xr-x | maintainers/scripts/gnome.sh | 2 | ||||
-rwxr-xr-x | maintainers/scripts/hydra-eval-failures.py | 8 | ||||
-rwxr-xr-x | maintainers/scripts/nix-diff.sh | 277 | ||||
-rwxr-xr-x | maintainers/scripts/rebuild-amount.sh | 345 | ||||
-rwxr-xr-x | maintainers/scripts/update-python-libraries | 243 |
8 files changed, 626 insertions, 277 deletions
diff --git a/maintainers/docker/.dockerignore b/maintainers/docker/.dockerignore deleted file mode 100644 index eb4668233e6a3..0000000000000 --- a/maintainers/docker/.dockerignore +++ /dev/null @@ -1,14 +0,0 @@ -*~ -,* -.*.swp -.*.swo -result -result-* -/doc/NEWS.html -/doc/NEWS.txt -/doc/manual.html -/doc/manual.pdf -.version-suffix - -.DS_Store -.git diff --git a/maintainers/docker/Dockerfile b/maintainers/docker/Dockerfile deleted file mode 100644 index f02a9d653fd55..0000000000000 --- a/maintainers/docker/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM busybox - -RUN dir=`mktemp -d` && trap 'rm -rf "$dir"' EXIT && \ - wget -O- https://nixos.org/releases/nix/nix-1.7/nix-1.7-x86_64-linux.tar.bz2 | bzcat | tar x -C $dir && \ - mkdir -m 0755 /nix && USER=root sh $dir/*/install && \ - echo ". /root/.nix-profile/etc/profile.d/nix.sh" >> /etc/profile - -ADD . /root/nix/nixpkgs -ONBUILD ENV NIX_PATH nixpkgs=/root/nix/nixpkgs:nixos=/root/nix/nixpkgs/nixos -ONBUILD ENV PATH /root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin -ONBUILD ENV ENV /etc/profile -ENV ENV /etc/profile diff --git a/maintainers/scripts/all-tarballs.nix b/maintainers/scripts/all-tarballs.nix index 552f88022961e..d981a1fa7dbc0 100644 --- a/maintainers/scripts/all-tarballs.nix +++ b/maintainers/scripts/all-tarballs.nix @@ -14,5 +14,5 @@ removeAttrs (import ../../pkgs/top-level/release.nix supportedSystems = [ "x86_64-linux" ]; }) [ # Remove jobs whose evaluation depends on a writable Nix store. - "tarball" "unstable" + "tarball" "unstable" "darwin-tested" ] diff --git a/maintainers/scripts/gnome.sh b/maintainers/scripts/gnome.sh index b2d55d7fbed26..e5a8d606f1a3b 100755 --- a/maintainers/scripts/gnome.sh +++ b/maintainers/scripts/gnome.sh @@ -6,7 +6,7 @@ GNOME_FTP=ftp.gnome.org/pub/GNOME/sources # projects that don't follow the GNOME major versioning, or that we don't want to # programmatically update -NO_GNOME_MAJOR='gtkhtml gdm' +NO_GNOME_MAJOR="ghex gtkhtml gdm" usage() { echo "Usage: $0 gnome_dir <show project>|<update project>|<update-all> [major.minor]" >&2 diff --git a/maintainers/scripts/hydra-eval-failures.py b/maintainers/scripts/hydra-eval-failures.py index b339f296056cd..6bbc0a45e44d8 100755 --- a/maintainers/scripts/hydra-eval-failures.py +++ b/maintainers/scripts/hydra-eval-failures.py @@ -48,8 +48,8 @@ def get_maintainers(attr_name): @click.command() @click.option( '--jobset', - default="nixos/release-16.09", - help='Hydra project like nixos/release-16.09') + default="nixos/release-17.03", + help='Hydra project like nixos/release-17.03') def cli(jobset): """ Given a Hydra project, inspect latest evaluation @@ -74,13 +74,13 @@ def cli(jobset): # TODO: dependency failed without propagated builds for tr in d('img[alt="Failed"]').parents('tr'): a = pq(tr)('a')[1] - print "- [ ] [{}]({})".format(a.text, a.get('href')) + print("- [ ] [{}]({})".format(a.text, a.get('href'))) sys.stdout.flush() maintainers = get_maintainers(a.text) if maintainers: - print " - maintainers: {}".format(", ".join(map(lambda u: '@' + u, maintainers))) + print(" - maintainers: {}".format(", ".join(map(lambda u: '@' + u, maintainers)))) # TODO: print last three persons that touched this file # TODO: pinpoint the diff that broke this build, or maybe it's transient or maybe it never worked? diff --git a/maintainers/scripts/nix-diff.sh b/maintainers/scripts/nix-diff.sh new file mode 100755 index 0000000000000..0c65e29cf4351 --- /dev/null +++ b/maintainers/scripts/nix-diff.sh @@ -0,0 +1,277 @@ +#!/usr/bin/env nix-shell +#! nix-shell -i bash -p coreutils gnugrep gnused + +################################################################################ +# nix-diff.sh # +################################################################################ +# This script "diffs" Nix profile generations. # +# # +# Example: # +################################################################################ +# > nix-diff.sh 90 92 # +# + gnumake-4.2.1 # +# + gnumake-4.2.1-doc # +# - htmldoc-1.8.29 # +################################################################################ +# The example shows that as of generation 92 and since generation 90, # +# gnumake-4.2.1 and gnumake-4.2.1-doc have been installed, while # +# htmldoc-1.8.29 has been removed. # +# # +# The example above shows the default, minimal output mode of this script. # +# For more features, run `nix-diff.sh -h` for usage instructions. # +################################################################################ + +usage() { + cat <<EOF +usage: nix-diff.sh [-h | [-p profile | -s] [-q] [-l] [range]] +-h: print this message before exiting +-q: list the derivations installed in the parent generation +-l: diff every available intermediate generation between parent and + child +-p profile: specify the Nix profile to use + * defaults to ~/.nix-profile +-s: use the system profile + * equivalent to: -p /nix/var/nix/profiles/system +profile: * should be something like /nix/var/nix/profiles/default, not a + generation link like /nix/var/nix/profiles/default-2-link +range: the range of generations to diff + * the following patterns are allowed, where A, B, and N are positive + integers, and G is the currently active generation: + A..B => diffs from generation A to generation B + ~N => diffs from the Nth newest generation (older than G) to G + A => diffs from generation A to G + * defaults to ~1 +EOF +} + +usage_tip() { + echo 'run `nix-diff.sh -h` for usage instructions' >&2 + exit 1 +} + +while getopts :hqlp:s opt; do + case $opt in + h) + usage + exit + ;; + q) + opt_query=1 + ;; + l) + opt_log=1 + ;; + p) + opt_profile=$OPTARG + ;; + s) + opt_profile=/nix/var/nix/profiles/system + ;; + \?) + echo "error: invalid option -$OPTARG" >&2 + usage_tip + ;; + esac +done +shift $((OPTIND-1)) + +if [ -n "$opt_profile" ]; then + if ! [ -L "$opt_profile" ]; then + echo "error: expecting \`$opt_profile\` to be a symbolic link" >&2 + usage_tip + fi +else + opt_profile=$(readlink ~/.nix-profile) + if (( $? != 0 )); then + echo 'error: unable to dereference `~/.nix-profile`' >&2 + echo 'specify the profile manually with the `-p` flag' >&2 + usage_tip + fi +fi + +list_gens() { + nix-env -p "$opt_profile" --list-generations \ + | sed -r 's:^\s*::' \ + | cut -d' ' -f1 +} + +current_gen() { + nix-env -p "$opt_profile" --list-generations \ + | grep -E '\(current\)\s*$' \ + | sed -r 's:^\s*::' \ + | cut -d' ' -f1 +} + +neg_gen() { + local i=0 from=$1 n=$2 tmp + for gen in $(list_gens | sort -rn); do + if ((gen < from)); then + tmp=$gen + ((i++)) + ((i == n)) && break + fi + done + if ((i < n)); then + echo -n "error: there aren't $n generation(s) older than" >&2 + echo " generation $from" >&2 + return 1 + fi + echo $tmp +} + +match() { + argv=("$@") + for i in $(seq $(($#-1))); do + if grep -E "^${argv[$i]}\$" <(echo "$1") >/dev/null; then + echo $i + return + fi + done + echo 0 +} + +case $(match "$1" '' '[0-9]+' '[0-9]+\.\.[0-9]+' '~[0-9]+') in + 1) + diffTo=$(current_gen) + diffFrom=$(neg_gen $diffTo 1) + (($? == 1)) && usage_tip + ;; + 2) + diffFrom=$1 + diffTo=$(current_gen) + ;; + 3) + diffFrom=${1%%.*} + diffTo=${1##*.} + ;; + 4) + diffTo=$(current_gen) + diffFrom=$(neg_gen $diffTo ${1#*~}) + (($? == 1)) && usage_tip + ;; + 0) + echo 'error: invalid invocation' >&2 + usage_tip + ;; +esac + +dirA="${opt_profile}-${diffFrom}-link" +dirB="${opt_profile}-${diffTo}-link" + +declare -a temp_files +temp_length() { + echo -n ${#temp_files[@]} +} +temp_make() { + temp_files[$(temp_length)]=$(mktemp) +} +temp_clean() { + rm -f ${temp_files[@]} +} +temp_name() { + echo -n "${temp_files[$(($(temp_length)-1))]}" +} +trap 'temp_clean' EXIT + +temp_make +versA=$(temp_name) +refs=$(nix-store -q --references "$dirA") +(( $? != 0 )) && exit 1 +echo "$refs" \ + | grep -v env-manifest.nix \ + | sort \ + > "$versA" + +print_tag() { + local gen=$1 + nix-env -p "$opt_profile" --list-generations \ + | grep -E "^\s*${gen}" \ + | sed -r 's:^\s*::' \ + | sed -r 's:\s*$::' +} + +if [ -n "$opt_query" ]; then + print_tag $diffFrom + cat "$versA" \ + | sed -r 's:^[^-]+-(.*)$: \1:' + + print_line=1 +fi + +if [ -n "$opt_log" ]; then + gens=$(for gen in $(list_gens); do + ((diffFrom < gen && gen < diffTo)) && echo $gen + done) + # Force the $diffTo generation to be included in this list, instead of using + # `gen <= diffTo` in the preceding loop, so we encounter an error upon the + # event of its nonexistence. + gens=$(echo "$gens" + echo $diffTo) +else + gens=$diffTo +fi + +temp_make +add=$(temp_name) +temp_make +rem=$(temp_name) +temp_make +out=$(temp_name) + +for gen in $gens; do + + [ -n "$print_line" ] && echo + + temp_make + versB=$(temp_name) + + dirB="${opt_profile}-${gen}-link" + refs=$(nix-store -q --references "$dirB") + (( $? != 0 )) && exit 1 + echo "$refs" \ + | grep -v env-manifest.nix \ + | sort \ + > "$versB" + + in=$(comm -3 -1 "$versA" "$versB") + sed -r 's:^[^-]*-(.*)$:\1+:' <(echo "$in") \ + | sort -f \ + > "$add" + + un=$(comm -3 -2 "$versA" "$versB") + sed -r 's:^[^-]*-(.*)$:\1-:' <(echo "$un") \ + | sort -f \ + > "$rem" + + cat "$rem" "$add" \ + | sort -f \ + | sed -r 's:(.*)-$:- \1:' \ + | sed -r 's:(.*)\+$:\+ \1:' \ + | grep -v '^$' \ + > "$out" + + if [ -n "$opt_query" -o -n "$opt_log" ]; then + + lines=$(wc -l "$out" | cut -d' ' -f1) + tag=$(print_tag "$gen") + (( $? != 0 )) && exit 1 + if [ $lines -eq 0 ]; then + echo "$tag (no change)" + else + echo "$tag" + fi + cat "$out" \ + | sed 's:^: :' + + print_line=1 + + else + echo "diffing from generation $diffFrom to $diffTo" + cat "$out" + fi + + versA=$versB + +done + +exit 0 diff --git a/maintainers/scripts/rebuild-amount.sh b/maintainers/scripts/rebuild-amount.sh index ebc5dc3b87ec7..098a8c88cb7e2 100755 --- a/maintainers/scripts/rebuild-amount.sh +++ b/maintainers/scripts/rebuild-amount.sh @@ -1,260 +1,115 @@ -#!/bin/sh - -usage () { - echo 1>&2 " -usage: - $0 - [--git commit..commit | --git commit] - [--svn rev:rev | --svn rev] - [--path path[:path]*] - [--help] - -This program is used to investigate how any changes inside your nixpkgs -repository may hurt. With these kind of information you may choose wisely -where you should commit your changes. - -This program adapts it-self to your versionning system to avoid too much -effort on your Internet bandwidth. If you need to check more than one -commits / revisions, you may use the following commands: - - --git remotes/trunk..master - --svn 17670:17677 - - Check the differences between each commit separating the first and the - last commit. - - --path /etc/nixos/nixpkgs:/tmp/nixpkgs_1:/tmp/nixpkgs_2 - - Check the differences between multiple directories containing different - versions of nixpkgs. - -All these options exist with one commit / revision argument. Such options -are used to compare your \$NIXPKGS path with the specified version. - -If you omit to mention any other commit / revision, then your \$NIXPKGS path -is compared with its last update. This command is useful to test code from -a dirty repository. - -" - - exit 1; -} - -##################### -# Process Arguments # -##################### - -: ${NIXPKGS=/etc/nixos/nixpkgs/} - -vcs="" -gitCommits="" -svnRevisions="" -pathLocations="" -verbose=false - -argfun="" -for arg; do - if test -z "$argfun"; then - case $arg in - --git) vcs="git"; argfun="set_gitCommits";; - --svn) vcs="svn"; argfun="set_svnRevisions";; - --path) vcs="path"; argfun="set_pathLocations";; - --verbose) verbose=true;; - --help) usage;; - *) usage;; - esac - else - case $argfun in - set_*) - var=$(echo $argfun | sed 's,^set_,,') - eval $var=$arg - ;; - esac - argfun="" - fi -done - -if $verbose; then - set -x -else - set +x +#!/usr/bin/env bash +set -e + +if [ "$#" != 1 ] && [ "$#" != 2 ]; then + cat <<-EOF + Usage: $0 commit-spec [commit-spec] + You need to be in a git-controlled nixpkgs tree. + The current state of the tree will be used if the second commit is missing. + EOF + exit 1 fi -############################ -# Find the repository type # -############################ +# A slightly hacky way to get the config. +parallel="$(echo 'config.rebuild-amount.parallel or false' | nix-repl . 2>/dev/null \ + | grep -v '^\(nix-repl.*\)\?$' | tail -n 1 || true)" -if test -z "$vcs"; then - if test -x "$NIXPKGS/.git"; then - if git --git-dir="$NIXPKGS/.git" branch > /dev/null 2>&1; then - vcs="git" - gitCommits=$(git --git-dir="$NIXPKGS/.git" log -n 1 --pretty=format:%H 2> /dev/null) - fi - elif test -x "$NIXPKGS/.svn"; then - cd "$NIXPKGS" - if svn info > /dev/null 2>&1; then - vcs="svn"; - svnRevisions=$(svn info | sed -n 's,Revision: ,,p') - fi - cd - - else - usage - fi -fi +echo "Estimating rebuild amount by counting changed Hydra jobs." -############################### -# Define a storage directory. # -############################### +toRemove=() -pkgListDir="" -exitCode=1 -cleanup(){ - test -e "$pkgListDir" && rm -rf "$pkgListDir" - exit $exitCode; +cleanup() { + rm -rf "${toRemove[@]}" } - trap cleanup EXIT SIGINT SIGQUIT ERR -pkgListDir=$(mktemp --tmpdir -d rebuild-amount-XXXXXXXX) -vcsDir="$pkgListDir/.vcs" - -########################### -# Versionning for Dummies # -########################### - -path_init() { - if test "${pathLocations#*:}" = "$pathLocations"; then - pathLocations="$NIXPKGS:$pathLocations" - fi - pathLocations="${pathLocations}:" +MKTEMP='mktemp --tmpdir nix-rebuild-amount-XXXXXXXX' + +nixexpr() { + cat <<-EONIX + let + lib = import $1/lib; + hydraJobs = import $1/pkgs/top-level/release.nix + # Compromise: accuracy vs. resources needed for evaluation. + { supportedSystems = cfg.systems or [ "x86_64-linux" "x86_64-darwin" ]; }; + cfg = (import $1 {}).config.rebuild-amount or {}; + + recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; }; + + # hydraJobs leaves recurseForDerivations as empty attrmaps; + # that would break nix-env and we also need to recurse everywhere. + tweak = lib.mapAttrs + (name: val: + if name == "recurseForDerivations" then true + else if lib.isAttrs val && val.type or null != "derivation" + then recurseIntoAttrs (tweak val) + else val + ); + + # Some of these contain explicit references to platform(s) we want to avoid; + # some even (transitively) depend on ~/.nixpkgs/config.nix (!) + blacklist = [ + "tarball" "metrics" "manual" + "darwin-tested" "unstable" "stdenvBootstrapTools" + "moduleSystem" "lib-tests" # these just confuse the output + ]; + + in + tweak (builtins.removeAttrs hydraJobs blacklist) + EONIX } -path_getNext() { - pathLoc="${pathLocations%%:*}" - pathLocations="${pathLocations#*:}" +# Output packages in tree $2 that weren't in $1. +# Changing the output hash or name is taken as a change. +# Extra nix-env parameters can be in $3 +newPkgs() { + # We use files instead of pipes, as running multiple nix-env processes + # could eat too much memory for a standard 4GiB machine. + local -a list + for i in 1 2; do + local l="$($MKTEMP)" + list[$i]="$l" + toRemove+=("$l") + + local expr="$($MKTEMP)" + toRemove+=("$expr") + nixexpr "${!i}" > "$expr" + + nix-env -f "$expr" -qaP --no-name --out-path --show-trace $3 \ + | sort > "${list[$i]}" & + + if [ "$parallel" != "true" ]; then + wait + fi + done + + wait + comm -13 "${list[@]}" } -path_setPath() { - path="$pathLoc" -} - -path_setName() { - name=$(echo "$pathLoc" | tr '/' '_') -} - -################ -# Git Commands # -################ - -git_init() { - git clone "$NIXPKGS/.git" "$vcsDir" > /dev/null 2>&1 - if echo "gitCommits" | grep -c "\.\." > /dev/null 2>&1; then - gitCommits=$(git --git-dir="$vcsDir/.git" log --reverse --pretty=format:%H $gitCommits 2> /dev/null) - else - pathLocations="$vcsDir:$NIXPKGS" - vcs="path" - path_init - fi -} - -git_getNext() { - git --git-dir="$vcsDir/.git" checkout $(echo "$gitCommits" | head -n 1) > /dev/null 2>&1 - gitCommits=$(echo "$gitCommits" | sed '1 d') -} - -git_setPath() { - path="$vcsDir" -} - -git_setName() { - name=$(git --git-dir="$vcsDir/.git" log -n 1 --pretty=format:%H 2> /dev/null) -} - -####################### -# Subversion Commands # -####################### - -svn_init() { - cp -r "$NIXPKGS" "$vcsDir" > /dev/null 2>&1 - if echo "svnRevisions" | grep -c ":" > /dev/null 2>&1; then - svnRevisions=$(seq ${svnRevisions%:*} ${svnRevisions#*:}) - else - pathLocations="$vcsDir:$NIXPKGS" - vcs="path" - path_init - fi -} - -svn_getNext() { - cd "$vcsDir" - svn checkout $(echo "$svnRevisions" | head -n 1) > /dev/null 2>&1 - cd - - svnRevisions=$(echo "$svnRevisions" | sed '1 d') -} - -svn_setPath() { - path="$vcsDir" -} - -svn_setName() { - name=$(svn info 2> /dev/null | sed -n 's,Revision: ,,p') -} - -#################### -# Logical Commands # -#################### - -init () { ${vcs}_init; } -getNext () { ${vcs}_getNext; } -setPath () { ${vcs}_setPath; } -setName () { ${vcs}_setName; } - - -##################### -# Check for Rebuild # -##################### - -# Generate the list of all derivations that could be build from a nixpkgs -# respository. This list of derivation hashes is compared with previous -# lists and a brief summary is produced on the output. - -compareNames () { - nb=$(diff -y --suppress-common-lines --speed-large-files "$pkgListDir/$1.drvs" "$pkgListDir/$2.drvs" 2> /dev/null | wc -l) - echo "$1 -> $2: $nb" -} - -echo "Please wait, this may take some minutes ..." - -init -first="" -oldPrev="" - -prev="" -curr="" - -while true; do - getNext - setPath # set path=... - setName # set name=... - curr="$name" - - test -z "$curr" && break || true - - nix-instantiate "$path" > "$pkgListDir/$curr.drvs" > /dev/null 2>&1 || true - - if test -n "$prev"; then - compareNames "$prev" "$curr" - else - echo "Number of package to rebuild:" - first="$curr" - fi - oldPrev="$prev" - prev="$curr" +# Prepare nixpkgs trees. +declare -a tree +for i in 1 2; do + if [ -n "${!i}" ]; then # use the given commit + dir="$($MKTEMP -d)" + tree[$i]="$dir" + toRemove+=("$dir") + + git clone --shared --no-checkout --quiet . "${tree[$i]}" + (cd "${tree[$i]}" && git checkout --quiet "${!i}") + else #use the current tree + tree[$i]="$(pwd)" + fi done -if test "$first" != "$oldPrev"; then - echo "Number of package to rebuild (first -> last):" - compareNames "$first" "$curr" -fi +newlist="$($MKTEMP)" +toRemove+=("$newlist") +# Notes: +# - the evaluation is done on x86_64-linux, like on Hydra. +# - using $newlist file so that newPkgs() isn't in a sub-shell (because of toRemove) +newPkgs "${tree[1]}" "${tree[2]}" '--argstr system "x86_64-linux"' > "$newlist" + +# Hacky: keep only the last word of each attribute path and sort. +sed -n 's/\([^. ]*\.\)*\([^. ]*\) .*$/\2/p' < "$newlist" \ + | sort | uniq -c -exitCode=0 diff --git a/maintainers/scripts/update-python-libraries b/maintainers/scripts/update-python-libraries new file mode 100755 index 0000000000000..7c73510c353a6 --- /dev/null +++ b/maintainers/scripts/update-python-libraries @@ -0,0 +1,243 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i python3 -p 'python3.withPackages(ps: with ps; [ requests toolz ])' + +""" +Update a Python package expression by passing in the `.nix` file, or the directory containing it. +You can pass in multiple files or paths. + +You'll likely want to use +`` + $ ./update-python-libraries ../../pkgs/development/python-modules/* +`` +to update all libraries in that folder. +""" + +import argparse +import logging +import os +import re +import requests +import toolz + +INDEX = "https://pypi.io/pypi" +"""url of PyPI""" + +EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip', '.whl'] +"""Permitted file extensions. These are evaluated from left to right and the first occurance is returned.""" + +import logging +logging.basicConfig(level=logging.INFO) + + +def _get_values(attribute, text): + """Match attribute in text and return all matches. + + :returns: List of matches. + """ + regex = '{}\s+=\s+"(.*)";'.format(attribute) + regex = re.compile(regex) + values = regex.findall(text) + return values + +def _get_unique_value(attribute, text): + """Match attribute in text and return unique match. + + :returns: Single match. + """ + values = _get_values(attribute, text) + n = len(values) + if n > 1: + raise ValueError("found too many values for {}".format(attribute)) + elif n == 1: + return values[0] + else: + raise ValueError("no value found for {}".format(attribute)) + +def _get_line_and_value(attribute, text): + """Match attribute in text. Return the line and the value of the attribute.""" + regex = '({}\s+=\s+"(.*)";)'.format(attribute) + regex = re.compile(regex) + value = regex.findall(text) + n = len(value) + if n > 1: + raise ValueError("found too many values for {}".format(attribute)) + elif n == 1: + return value[0] + else: + raise ValueError("no value found for {}".format(attribute)) + + +def _replace_value(attribute, value, text): + """Search and replace value of attribute in text.""" + old_line, old_value = _get_line_and_value(attribute, text) + new_line = old_line.replace(old_value, value) + new_text = text.replace(old_line, new_line) + return new_text + +def _fetch_page(url): + r = requests.get(url) + if r.status_code == requests.codes.ok: + return r.json() + else: + raise ValueError("request for {} failed".format(url)) + +def _get_latest_version_pypi(package, extension): + """Get latest version and hash from PyPI.""" + url = "{}/{}/json".format(INDEX, package) + json = _fetch_page(url) + + version = json['info']['version'] + for release in json['releases'][version]: + if release['filename'].endswith(extension): + # TODO: In case of wheel we need to do further checks! + sha256 = release['digests']['sha256'] + break + else: + sha256 = None + return version, sha256 + + +def _get_latest_version_github(package, extension): + raise ValueError("updating from GitHub is not yet supported.") + + +FETCHERS = { + 'fetchFromGitHub' : _get_latest_version_github, + 'fetchPypi' : _get_latest_version_pypi, + 'fetchurl' : _get_latest_version_pypi, +} + + +DEFAULT_SETUPTOOLS_EXTENSION = 'tar.gz' + + +FORMATS = { + 'setuptools' : DEFAULT_SETUPTOOLS_EXTENSION, + 'wheel' : 'whl' +} + +def _determine_fetcher(text): + # Count occurences of fetchers. + nfetchers = sum(text.count('src = {}'.format(fetcher)) for fetcher in FETCHERS.keys()) + if nfetchers == 0: + raise ValueError("no fetcher.") + elif nfetchers > 1: + raise ValueError("multiple fetchers.") + else: + # Then we check which fetcher to use. + for fetcher in FETCHERS.keys(): + if 'src = {}'.format(fetcher) in text: + return fetcher + + +def _determine_extension(text, fetcher): + """Determine what extension is used in the expression. + + If we use: + - fetchPypi, we check if format is specified. + - fetchurl, we determine the extension from the url. + - fetchFromGitHub we simply use `.tar.gz`. + """ + if fetcher == 'fetchPypi': + try: + format = _get_unique_value('format', text) + except ValueError as e: + format = None # format was not given + + try: + extension = _get_unique_value('extension', text) + except ValueError as e: + extension = None # extension was not given + + if extension is None: + if format is None: + format = 'setuptools' + extension = FORMATS[format] + + elif fetcher == 'fetchurl': + url = _get_unique_value('url', text) + extension = os.path.splitext(url)[1] + if 'pypi' not in url: + raise ValueError('url does not point to PyPI.') + + elif fetcher == 'fetchFromGitHub': + raise ValueError('updating from GitHub is not yet implemented.') + + return extension + + +def _update_package(path): + + + + # Read the expression + with open(path, 'r') as f: + text = f.read() + + # Determine pname. + pname = _get_unique_value('pname', text) + + # Determine version. + version = _get_unique_value('version', text) + + # First we check how many fetchers are mentioned. + fetcher = _determine_fetcher(text) + + extension = _determine_extension(text, fetcher) + + new_version, new_sha256 = _get_latest_version_pypi(pname, extension) + + if new_version == version: + logging.info("Path {}: no update available for {}.".format(path, pname)) + return False + if not new_sha256: + raise ValueError("no file available for {}.".format(pname)) + + text = _replace_value('version', new_version, text) + text = _replace_value('sha256', new_sha256, text) + + with open(path, 'w') as f: + f.write(text) + + logging.info("Path {}: updated {} from {} to {}".format(path, pname, version, new_version)) + + return True + + +def _update(path): + + # We need to read and modify a Nix expression. + if os.path.isdir(path): + path = os.path.join(path, 'default.nix') + + # If a default.nix does not exist, we quit. + if not os.path.isfile(path): + logging.info("Path {}: does not exist.".format(path)) + return False + + # If file is not a Nix expression, we quit. + if not path.endswith(".nix"): + logging.info("Path {}: does not end with `.nix`.".format(path)) + return False + + try: + return _update_package(path) + except ValueError as e: + logging.warning("Path {}: {}".format(path, e)) + return False + +def main(): + + parser = argparse.ArgumentParser() + parser.add_argument('package', type=str, nargs='+') + + args = parser.parse_args() + + packages = map(os.path.abspath, args.package) + + count = list(map(_update, packages)) + + logging.info("{} package(s) updated".format(sum(count))) + +if __name__ == '__main__': + main() \ No newline at end of file |