1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
|
{ stdenv, lib, callPackage, runCommand, writeReferencesToFile, writeText, vmTools, writeScript
, docker, shadow, utillinux, coreutils, jshon, e2fsprogs, goPackages, pigz }:
# WARNING: this API is unstable and may be subject to backwards-incompatible changes in the future.
rec {
pullImage = callPackage ./pull.nix {};
# We need to sum layer.tar, not a directory, hence tarsum instead of nix-hash.
# And we cannot untar it, because then we cannot preserve permissions ecc.
tarsum = runCommand "tarsum" {
buildInputs = [ goPackages.go ];
} ''
mkdir tarsum
cd tarsum
cp ${./tarsum.go} tarsum.go
export GOPATH=$(pwd)
mkdir src
ln -sT ${docker.src}/pkg/tarsum src/tarsum
go build
cp tarsum $out
'';
# buildEnv creates symlinks to dirs, which is hard to edit inside the overlay VM
mergeDrvs = { drvs, onlyDeps ? false }:
runCommand "merge-drvs" {
inherit drvs onlyDeps;
} ''
if [ -n "$onlyDeps" ]; then
echo $drvs > $out
exit 0
fi
mkdir $out
for drv in $drvs; do
echo Merging $drv
if [ -d "$drv" ]; then
cp -drf --preserve=mode -f $drv/* $out/
else
tar -C $out -xpf $drv || true
fi
done
'';
shellScript = text:
writeScript "script.sh" ''
#!${stdenv.shell}
set -e
export PATH=${coreutils}/bin:/bin
${text}
'';
shadowSetup = ''
export PATH=${shadow}/bin:$PATH
mkdir -p /etc/pam.d
if [ ! -f /etc/passwd ]; then
echo "root:x:0:0::/root:/bin/sh" > /etc/passwd
echo "root:!x:::::::" > /etc/shadow
fi
if [ ! -f /etc/group ]; then
echo "root:x:0:" > /etc/group
echo "root:x::" > /etc/gshadow
fi
if [ ! -f /etc/pam.d/other ]; then
cat > /etc/pam.d/other <<EOF
account sufficient pam_unix.so
auth sufficient pam_rootok.so
password requisite pam_unix.so nullok sha512
session required pam_unix.so
EOF
fi
if [ ! -f /etc/login.defs ]; then
touch /etc/login.defs
fi
'';
runWithOverlay = { name , fromImage ? null, fromImageName ? null, fromImageTag ? null
, diskSize ? 1024, preMount ? "", postMount ? "", postUmount ? "" }:
vmTools.runInLinuxVM (
runCommand name {
preVM = vmTools.createEmptyImage { size = diskSize; fullName = "docker-run-disk"; };
inherit fromImage fromImageName fromImageTag;
buildInputs = [ utillinux e2fsprogs jshon ];
} ''
rm -rf $out
mkdir disk
mkfs /dev/${vmTools.hd}
mount /dev/${vmTools.hd} disk
cd disk
if [ -n "$fromImage" ]; then
echo Unpacking base image
mkdir image
tar -C image -xpf "$fromImage"
if [ -z "$fromImageName" ]; then
fromImageName=$(jshon -k < image/repositories|head -n1)
fi
if [ -z "$fromImageTag" ]; then
fromImageTag=$(jshon -e $fromImageName -k < image/repositories|head -n1)
fi
parentID=$(jshon -e $fromImageName -e $fromImageTag -u < image/repositories)
fi
lowerdir=""
while [ -n "$parentID" ]; do
echo Unpacking layer $parentID
mkdir -p image/$parentID/layer
tar -C image/$parentID/layer -xpf image/$parentID/layer.tar
rm image/$parentID/layer.tar
find image/$parentID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
lowerdir=$lowerdir''${lowerdir:+:}image/$parentID/layer
parentID=$(cat image/$parentID/json|(jshon -e parent -u 2>/dev/null || true))
done
mkdir work
mkdir layer
mkdir mnt
${preMount}
if [ -n "$lowerdir" ]; then
mount -t overlay overlay -olowerdir=$lowerdir,workdir=work,upperdir=layer mnt
else
mount --bind layer mnt
fi
${postMount}
umount mnt
pushd layer
find . -type c -exec bash -c 'name="$(basename {})"; touch "$(dirname {})/.wh.$name"; rm "{}"' \;
popd
${postUmount}
'');
exportImage = { name ? fromImage.name, fromImage, fromImageName ? null, fromImageTag ? null, diskSize ? 1024 }:
runWithOverlay {
inherit name fromImage fromImageName fromImageTag diskSize;
postMount = ''
echo Packing raw image
tar -C mnt --mtime=0 -cf $out .
'';
};
mkPureLayer = { baseJson, contents ? null, extraCommands ? "" }:
runCommand "docker-layer" {
inherit baseJson contents extraCommands;
buildInputs = [ jshon ];
} ''
mkdir layer
if [ -n "$contents" ]; then
echo Adding contents
for c in $contents; do
cp -drf $c/* layer/
chmod -R ug+w layer/
done
fi
pushd layer
${extraCommands}
popd
echo Packing layer
mkdir $out
tar -C layer --mtime=0 -cf $out/layer.tar .
ts=$(${tarsum} < $out/layer.tar)
cat ${baseJson} | jshon -s "$ts" -i checksum > $out/json
echo -n "1.0" > $out/VERSION
'';
mkRootLayer = { runAsRoot, baseJson, fromImage ? null, fromImageName ? null, fromImageTag ? null
, diskSize ? 1024, contents ? null, extraCommands ? "" }:
let runAsRootScript = writeScript "run-as-root.sh" runAsRoot;
in runWithOverlay {
name = "docker-layer";
inherit fromImage fromImageName fromImageTag diskSize;
preMount = lib.optionalString (contents != null) ''
echo Adding contents
for c in ${builtins.toString contents}; do
cp -drf $c/* layer/
chmod -R ug+w layer/
done
'';
postMount = ''
mkdir -p mnt/{dev,proc,sys,nix/store}
mount --rbind /dev mnt/dev
mount --rbind /sys mnt/sys
mount --rbind /nix/store mnt/nix/store
unshare -imnpuf --mount-proc chroot mnt ${runAsRootScript}
umount -R mnt/dev mnt/sys mnt/nix/store
rmdir --ignore-fail-on-non-empty mnt/dev mnt/proc mnt/sys mnt/nix/store mnt/nix
'';
postUmount = ''
pushd layer
${extraCommands}
popd
echo Packing layer
mkdir $out
tar -C layer --mtime=0 -cf $out/layer.tar .
ts=$(${tarsum} < $out/layer.tar)
cat ${baseJson} | jshon -s "$ts" -i checksum > $out/json
echo -n "1.0" > $out/VERSION
'';
};
# 1. extract the base image
# 2. create the layer
# 3. add layer deps to the layer itself, diffing with the base image
# 4. compute the layer id
# 5. put the layer in the image
# 6. repack the image
buildImage = args@{ name, tag ? "latest"
, fromImage ? null, fromImageName ? null, fromImageTag ? null
, contents ? null, config ? null, runAsRoot ? null
, diskSize ? 1024, extraCommands ? "" }:
let
baseName = baseNameOf name;
baseJson = writeText "${baseName}-config.json" (builtins.toJSON {
created = "1970-01-01T00:00:01Z";
architecture = "amd64";
os = "linux";
config = config;
});
layer = (if runAsRoot == null
then mkPureLayer { inherit baseJson contents extraCommands; }
else mkRootLayer { inherit baseJson fromImage fromImageName fromImageTag contents runAsRoot diskSize extraCommands; });
result = runCommand "${baseName}.tar.gz" {
buildInputs = [ jshon pigz ];
imageName = name;
imageTag = tag;
inherit fromImage baseJson;
layerClosure = writeReferencesToFile layer;
passthru = {
buildArgs = args;
};
} ''
mkdir image
touch baseFiles
if [ -n "$fromImage" ]; then
echo Unpacking base image
tar -C image -xpf "$fromImage"
if [ -z "$fromImageName" ]; then
fromImageName=$(jshon -k < image/repositories|head -n1)
fi
if [ -z "$fromImageTag" ]; then
fromImageTag=$(jshon -e $fromImageName -k < image/repositories|head -n1)
fi
parentID=$(jshon -e $fromImageName -e $fromImageTag -u < image/repositories)
for l in image/*/layer.tar; do
tar -tf $l >> baseFiles
done
fi
chmod -R ug+rw image
mkdir temp
cp ${layer}/* temp/
chmod ug+w temp/*
# FIXME: might not be /nix/store
echo '/nix' >> layerFiles
echo '/nix/store' >> layerFiles
for dep in $(cat $layerClosure); do
find $dep >> layerFiles
done
echo Adding layer
tar -tf temp/layer.tar >> baseFiles
sed 's/^\.//' -i baseFiles
comm <(sort -n baseFiles|uniq) <(sort -n layerFiles|uniq|grep -v ${layer}) -1 -3 > newFiles
tar -rpf temp/layer.tar --mtime=0 --no-recursion --files-from newFiles 2>/dev/null || true
echo Adding meta
if [ -n "$parentID" ]; then
cat temp/json | jshon -s "$parentID" -i parent > tmpjson
mv tmpjson temp/json
fi
layerID=$(sha256sum temp/json|cut -d ' ' -f 1)
size=$(stat --printf="%s" temp/layer.tar)
cat temp/json | jshon -s "$layerID" -i id -n $size -i Size > tmpjson
mv tmpjson temp/json
mv temp image/$layerID
jshon -n object \
-n object -s "$layerID" -i "$imageTag" \
-i "$imageName" > image/repositories
chmod -R a-w image
echo Cooking the image
tar -C image --mtime=0 -c . | pigz -nT > $out
'';
in
result;
}
|