about summary refs log tree commit diff
path: root/lib/fileset/benchmark.sh
blob: 59ddb6d49af792e82644ca88fb6c018c07d55796 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
#!/usr/bin/env nix-shell
#!nix-shell -i bash -p sta jq bc nix -I nixpkgs=../..
# shellcheck disable=SC2016

# Benchmarks lib.fileset
# Run:
# [nixpkgs]$ lib/fileset/benchmark.sh HEAD

set -euo pipefail
shopt -s inherit_errexit dotglob

if (( $# == 0 )); then
    echo "Usage: $0 HEAD"
    echo "Benchmarks the current tree against the HEAD commit. Any git ref will work."
    exit 1
fi
compareTo=$1

SCRIPT_FILE=$(readlink -f "${BASH_SOURCE[0]}")
SCRIPT_DIR=$(dirname "$SCRIPT_FILE")

nixpkgs=$(cd "$SCRIPT_DIR/../.."; pwd)

tmp="$(mktemp -d)"
clean_up() {
    rm -rf "$tmp"
}
trap clean_up EXIT SIGINT SIGTERM
work="$tmp/work"
mkdir "$work"
cd "$work"

declare -a stats=(
    ".envs.elements"
    ".envs.number"
    ".gc.totalBytes"
    ".list.concats"
    ".list.elements"
    ".nrFunctionCalls"
    ".nrLookups"
    ".nrOpUpdates"
    ".nrPrimOpCalls"
    ".nrThunks"
    ".sets.elements"
    ".sets.number"
    ".symbols.number"
    ".values.number"
)

runs=10

run() {
    # Empty the file
    : > cpuTimes

    for i in $(seq 0 "$runs"); do
        NIX_PATH=nixpkgs=$1 NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=$tmp/stats.json \
            nix-instantiate --eval --strict --show-trace >/dev/null \
            --expr 'with import <nixpkgs/lib>; with fileset; '"$2"

        # Only measure the time after the first run, one is warmup
        if (( i > 0 )); then
            jq '.cpuTime' "$tmp/stats.json" >> cpuTimes
        fi
    done

    # Compute mean and standard deviation
    read -r mean sd < <(sta --mean --sd --brief <cpuTimes)

    jq --argjson mean "$mean" --argjson sd "$sd" \
        '.cpuTimeMean = $mean | .cpuTimeSd = $sd' \
        "$tmp/stats.json"
}

bench() {
    echo "Benchmarking expression $1" >&2
    #echo "Running benchmark on index" >&2
    run "$nixpkgs" "$1" > "$tmp/new.json"
    (
        #echo "Checking out $compareTo" >&2
        git -C "$nixpkgs" worktree add --quiet "$tmp/worktree" "$compareTo"
        trap 'git -C "$nixpkgs" worktree remove "$tmp/worktree"' EXIT
        #echo "Running benchmark on $compareTo" >&2
        run "$tmp/worktree" "$1" > "$tmp/old.json"
    )

    read -r oldMean oldSd newMean newSd percentageMean percentageSd < \
        <(jq -rn --slurpfile old "$tmp/old.json" --slurpfile new "$tmp/new.json" \
        ' $old[0].cpuTimeMean as $om
        | $old[0].cpuTimeSd as $os
        | $new[0].cpuTimeMean as $nm
        | $new[0].cpuTimeSd as $ns
        | (100 / $om * $nm) as $pm
        # Copied from https://github.com/sharkdp/hyperfine/blob/b38d550b89b1dab85139eada01c91a60798db9cc/src/benchmark/relative_speed.rs#L46-L53
        | ($pm * pow(pow($ns / $nm; 2) + pow($os / $om; 2); 0.5)) as $ps
        | [ $om, $os, $nm, $ns, $pm, $ps ]
        | @sh')

    echo -e "Mean CPU time $newMean (σ = $newSd) for $runs runs is \e[0;33m$percentageMean% (σ = $percentageSd%)\e[0m of the old value $oldMean (σ = $oldSd)" >&2

    different=0
    for stat in "${stats[@]}"; do
        oldValue=$(jq "$stat" "$tmp/old.json")
        newValue=$(jq "$stat" "$tmp/new.json")
        if (( oldValue != newValue )); then
            percent=$(bc <<< "scale=100; result = 100/$oldValue*$newValue; scale=4; result / 1")
            if (( oldValue < newValue )); then
                echo -e "Statistic $stat ($newValue) is \e[0;31m$percent% (+$(( newValue - oldValue )))\e[0m of the old value $oldValue" >&2
            else
                echo -e "Statistic $stat ($newValue) is \e[0;32m$percent% (-$(( oldValue - newValue )))\e[0m of the old value $oldValue" >&2
            fi
            (( different++ )) || true
        fi
    done
    echo "$different stats differ between the current tree and $compareTo"
    echo ""
}

# Create a fairly populated tree
touch f{0..5}
mkdir d{0..5}
mkdir e{0..5}
touch d{0..5}/f{0..5}
mkdir -p d{0..5}/d{0..5}
mkdir -p e{0..5}/e{0..5}
touch d{0..5}/d{0..5}/f{0..5}
mkdir -p d{0..5}/d{0..5}/d{0..5}
mkdir -p e{0..5}/e{0..5}/e{0..5}
touch d{0..5}/d{0..5}/d{0..5}/f{0..5}
mkdir -p d{0..5}/d{0..5}/d{0..5}/d{0..5}
mkdir -p e{0..5}/e{0..5}/e{0..5}/e{0..5}
touch d{0..5}/d{0..5}/d{0..5}/d{0..5}/f{0..5}

bench 'toSource { root = ./.; fileset = ./.; }'

rm -rf -- *

touch {0..1000}
bench 'toSource { root = ./.; fileset = unions (mapAttrsToList (name: value: ./. + "/${name}") (builtins.readDir ./.)); }'
rm -rf -- *