这篇文章发表于 187 天前,可能其部分内容已经发生变化,如有疑问可询问作者。

近几个月网络环境不佳,docker镜像的下载一直莫名其妙会中断,啥啥EOF的那种,然后又得从头docker pull。经常几个小时都下载不下来,让人抓狂!!!

换国内源,开代理,各种办法都试过,没啥好效果。根据排除法以为是自己的科学上网路由器啥的问题,结果来来回回断断续续检查了几个礼拜,但这个问题还是存在。

这里简单说下,各种国内源基本上都是那种对某些个镜像有效的,对于那种dockerhub上外国用户上传的镜像基本没啥作用。开代理的话其实也还是有时会出现问题,只要你的网络稍微不稳定,一个几个G大小的镜像,中途断掉是经常发生的事。还有那种修改/etc/hosts让register那啥域名指向那三个美国dockerhub那边ip的办法,因为这个DNS解析其实是没问题的所以屁用没有。

对于我这情况来说更新docker.io这个软件或者使用dockerproxy代理平台会有那么一点点效果,但还是下载中途会断。

所以最根本的解决办法是断点续传!

这周末又被折磨了,最终找到一位老哥的脚本,稍微修改了一下(运行下面脚本你就会知道用法,细节自行研究):

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
#!/usr/bin/bash
BW=8

if [[ "$1" == "BW"* ]]; then
NEWBW=`echo $1|cut -d: -f2`
if [ "$NEWBW" != "" ]; then
BW=$NEWBW
echo using bw limit $BW
fi
shift
fi

unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY
set -eo pipefail

# hello-world latest ef872312fe1b 3 months ago 910 B
# hello-world latest ef872312fe1bbc5e05aae626791a47ee9b032efa8f3bda39cc0be7b56bfe59b9 3 months ago 910 B

# debian latest f6fab3b798be 10 weeks ago 85.1 MB
# debian latest f6fab3b798be3174f45aa1eb731f8182705555f89c9026d8c1ef230cbf8301dd 10 weeks ago 85.1 MB

# check if essential commands are in our PATH
for cmd in curl jq go; do
if ! command -v $cmd &> /dev/null; then
echo >&2 "error: \"$cmd\" not found!"
exit 1
fi
done

usage() {
echo "usage: $0 dir image[:tag][@digest] ..."
echo " $0 /tmp/old-hello-world hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7"
[ -z "$1" ] || exit "$1"
}

dir="$1" # dir for building tar in
shift || usage 1 >&2

[ $# -gt 0 -a "$dir" ] || usage 2 >&2
mkdir -p "$dir"

# hacky workarounds for Bash 3 support (no associative arrays)
images=()
rm -f "$dir"/tags-*.tmp
manifestJsonEntries=()
doNotGenerateManifestJson=
# repositories[busybox]='"latest": "...", "ubuntu-14.04": "..."'

# bash v4 on Windows CI requires CRLF separator
newlineIFS=$'\n'
if [ "$(go env GOHOSTOS)" = 'windows' ]; then
major=$(echo ${BASH_VERSION%%[^0.9]} | cut -d. -f1)
if [ "$major" -ge 4 ]; then
newlineIFS=$'\r\n'
fi
fi

registryBase='https://registry-1.docker.io'
authBase='https://auth.docker.io'
authService='registry.docker.io'

# https://github.com/moby/moby/issues/33700
fetch_blob() {
FULL_FILE=0
local token="$1"; shift
local image="$1"; shift
local digest="$1"; shift
local targetFile="$1"; shift
local curlArgs=( "$@" )

local curlHeaders="$(
curl -S "${curlArgs[@]}" \
-H "Authorization: Bearer $token" \
"$registryBase/v2/$image/blobs/$digest" \
-o "$targetFile.headers" \
-D-
)"
curlHeaders="$(echo "$curlHeaders" | tr -d '\r')"
if grep -qE "^HTTP/[0-9].[0-9] 3" <<<"$curlHeaders"; then
local blobRedirect="$(echo "$curlHeaders" | awk -F ': ' 'tolower($1) == "location" { print $2; exit }')"
if [ -z "$blobRedirect" ]; then
echo >&2 "error: failed fetching '$image' blob '$digest'"
echo "$curlHeaders" | head -1 >&2
return 1
fi
#layer.tar gets handeled differently than small json files...
if [[ "$targetFile" == *"layer.tar"* ]]; then
#turn off the scripts cancel on any error settings..
set +eo pipefail

#loop until we get a 416 (server cannot accomodate resume byte range due to us having full file)
while :; do
#if the file already exists.. we will be resuming..
if [ -f "$targetFile" ];then
#getting current size of file we are resuming
CUR=`stat --printf="%s" $targetFile`
#use curl to get headers to find content-length of the full file
LEN=`curl -I -fL "${curlArgs[@]}" "$blobRedirect"|grep content-length|cut -d" " -f2`

#if we already have the entire file... lets stop curl from erroring with 416
if [ "$CUR" == "${LEN//[!0-9]/}" ]; then
FULL_FILE=1
break
fi
fi

HTTP_CODE=`curl -w %{http_code} -C - --tr-encoding --compressed --progress-bar -fL "${curlArgs[@]}" "$blobRedirect" -o "$targetFile"`
if [ "$HTTP_CODE" == "403" ]; then
#token expired so the server stopped allowing us to resume, lets return without setting FULL_FILE and itll restart this func w new token
FULL_FILE=0
break
fi

if [ "$HTTP_CODE" == "416" ]; then
FULL_FILE=1
break
fi

sleep 1
done
else
#small file.. needs no resume
curl -fSL "${curlArgs[@]}" "$blobRedirect" -o "$targetFile"
FULL_FILE=1
fi
#this could fail if we nested a call to this same function and it deletes .headers before it returns, and we try again..
rm -f "$targetFile.headers"
#turn back on scripts error checking
set -eo pipefail
fi
}

# handle 'application/vnd.docker.distribution.manifest.v2+json' manifest
handle_single_manifest_v2() {
local manifestJson="$1"; shift

local configDigest="$(echo "$manifestJson" | jq --raw-output '.config.digest')"
local imageId="${configDigest#*:}" # strip off "sha256:"

local configFile="$imageId.json"
fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s

local layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')"
local IFS="$newlineIFS"
local layers=( $layersFs )
unset IFS

echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..."
local layerId=
local layerFiles=()
for i in "${!layers[@]}"; do
local layerMeta="${layers[$i]}"

local layerMediaType="$(echo "$layerMeta" | jq --raw-output '.mediaType')"
local layerDigest="$(echo "$layerMeta" | jq --raw-output '.digest')"

# save the previous layer's ID
local parentId="$layerId"
# create a new fake layer ID based on this layer's digest and the previous layer's fake ID
layerId="$(echo "$parentId"$'\n'"$layerDigest" | sha256sum | cut -d' ' -f1)"
# this accounts for the possibility that an image contains the same layer twice (and thus has a duplicate digest value)

mkdir -p "$dir/$layerId"
echo '1.0' > "$dir/$layerId/VERSION"

if [ ! -s "$dir/$layerId/json" ]; then
local parentJson="$(printf ', parent: "%s"' "$parentId")"
local addJson="$(printf '{ id: "%s"%s }' "$layerId" "${parentId:+$parentJson}")"
# this starter JSON is taken directly from Docker's own "docker save" output for unimportant layers
jq "$addJson + ." > "$dir/$layerId/json" <<-'EOJSON'
{
"created": "0001-01-01T00:00:00Z",
"container_config": {
"Hostname": "",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": null,
"Cmd": null,
"Image": "",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": null,
"OnBuild": null,
"Labels": null
}
}
EOJSON
fi

case "$layerMediaType" in
application/vnd.docker.image.rootfs.diff.tar.gzip)
local layerTar="$layerId/layer.tar"
layerFiles=( "${layerFiles[@]}" "$layerTar" )
FULL_FILE=0
#loop until FULL_FILE is set in fetch_blob.. this is for bad/slow connections
while [ "$FULL_FILE" != "1" ];do
local token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress-bar
sleep 1
done

;;

*)
echo >&2 "error: unknown layer mediaType ($imageIdentifier, $layerDigest): '$layerMediaType'"
exit 1
;;
esac
done

# change "$imageId" to be the ID of the last layer we added (needed for old-style "repositories" file which is created later -- specifically for older Docker daemons)
imageId="$layerId"

# munge the top layer image manifest to have the appropriate image configuration for older daemons
local imageOldConfig="$(jq --raw-output --compact-output '{ id: .id } + if .parent then { parent: .parent } else {} end' "$dir/$imageId/json")"
jq --raw-output "$imageOldConfig + del(.history, .rootfs)" "$dir/$configFile" > "$dir/$imageId/json"

local manifestJsonEntry="$(
echo '{}' | jq --raw-output '. + {
Config: "'"$configFile"'",
RepoTags: ["'"${image#library\/}:$tag"'"],
Layers: '"$(echo '[]' | jq --raw-output ".$(for layerFile in "${layerFiles[@]}"; do echo " + [ \"$layerFile\" ]"; done)")"'
}'
)"
manifestJsonEntries=( "${manifestJsonEntries[@]}" "$manifestJsonEntry" )
}

while [ $# -gt 0 ]; do
imageTag="$1"
shift
image="${imageTag%%[:@]*}"
imageTag="${imageTag#*:}"
if [ "$imageTag" == "$image" ]; then
imageTag="latest"
fi
digest="${imageTag##*@}"
tag="${imageTag%%@*}"

# add prefix library if passed official image
if [[ "$image" != *"/"* ]]; then
image="library/$image"
fi

imageFile="${image//\//_}" # "/" can't be in filenames :)

token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"

manifestJson="$(
curl -fsSL \
-H "Authorization: Bearer $token" \
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
"$registryBase/v2/$image/manifests/$digest"
)"
if [ "${manifestJson:0:1}" != '{' ]; then
echo >&2 "error: /v2/$image/manifests/$digest returned something unexpected:"
echo >&2 " $manifestJson"
exit 1
fi

imageIdentifier="$image:$tag@$digest"

schemaVersion="$(echo "$manifestJson" | jq --raw-output '.schemaVersion')"
case "$schemaVersion" in
2)
mediaType="$(echo "$manifestJson" | jq --raw-output '.mediaType')"

case "$mediaType" in
application/vnd.docker.distribution.manifest.v2+json)
handle_single_manifest_v2 "$manifestJson"
;;
application/vnd.docker.distribution.manifest.list.v2+json)
layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.manifests[]')"
IFS="$newlineIFS"
layers=( $layersFs )
unset IFS

found=""
# parse first level multi-arch manifest
for i in "${!layers[@]}"; do
layerMeta="${layers[$i]}"
maniArch="$(echo "$layerMeta" | jq --raw-output '.platform.architecture')"
if [ "$maniArch" = "$(go env GOARCH)" ]; then
digest="$(echo "$layerMeta" | jq --raw-output '.digest')"
# get second level single manifest
submanifestJson="$(
curl -fsSL \
-H "Authorization: Bearer $token" \
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
"$registryBase/v2/$image/manifests/$digest"
)"
handle_single_manifest_v2 "$submanifestJson"
found="found"
break
fi
done
if [ -z "$found" ]; then
echo >&2 "error: manifest for $maniArch is not found"
exit 1
fi
;;
*)
echo >&2 "error: unknown manifest mediaType ($imageIdentifier): '$mediaType'"
exit 1
;;
esac
;;

1)
if [ -z "$doNotGenerateManifestJson" ]; then
echo >&2 "warning: '$imageIdentifier' uses schemaVersion '$schemaVersion'"
echo >&2 " this script cannot (currently) recreate the 'image config' to put in a 'manifest.json' (thus any schemaVersion 2+ images will be imported in the old way, and their 'docker history' will suffer)"
echo >&2
doNotGenerateManifestJson=1
fi

layersFs="$(echo "$manifestJson" | jq --raw-output '.fsLayers | .[] | .blobSum')"
IFS="$newlineIFS"
layers=( $layersFs )
unset IFS

history="$(echo "$manifestJson" | jq '.history | [.[] | .v1Compatibility]')"
imageId="$(echo "$history" | jq --raw-output '.[0]' | jq --raw-output '.id')"

echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..."
for i in "${!layers[@]}"; do
imageJson="$(echo "$history" | jq --raw-output ".[${i}]")"
layerId="$(echo "$imageJson" | jq --raw-output '.id')"
imageLayer="${layers[$i]}"

mkdir -p "$dir/$layerId"
echo '1.0' > "$dir/$layerId/VERSION"

echo "$imageJson" > "$dir/$layerId/json"

FULL_FILE=0
#loop until FULL_FILE is set in fetch_blob.. this is for bad/slow connections
while [ "$FULL_FILE" != "1" ];do
token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
fetch_blob "$token" "$image" "$imageLayer" "$dir/$layerId/layer.tar" --progress-bar
echo after fetch full file $FULL_FILE
sleep 1
done
done
;;

*)
echo >&2 "error: unknown manifest schemaVersion ($imageIdentifier): '$schemaVersion'"
exit 1
;;
esac

echo

if [ -s "$dir/tags-$imageFile.tmp" ]; then
echo -n ', ' >> "$dir/tags-$imageFile.tmp"
else
images=( "${images[@]}" "$image" )
fi
echo -n '"'"$tag"'": "'"$imageId"'"' >> "$dir/tags-$imageFile.tmp"
done

echo -n '{' > "$dir/repositories"
firstImage=1
for image in "${images[@]}"; do
imageFile="${image//\//_}" # "/" can't be in filenames :)
image="${image#library\/}"

[ "$firstImage" ] || echo -n ',' >> "$dir/repositories"
firstImage=
echo -n $'\n\t' >> "$dir/repositories"
echo -n '"'"$image"'": { '"$(cat "$dir/tags-$imageFile.tmp")"' }' >> "$dir/repositories"
done
echo -n $'\n}\n' >> "$dir/repositories"

rm -f "$dir"/tags-*.tmp

if [ -z "$doNotGenerateManifestJson" ] && [ "${#manifestJsonEntries[@]}" -gt 0 ]; then
echo '[]' | jq --raw-output ".$(for entry in "${manifestJsonEntries[@]}"; do echo " + [ $entry ]"; done)" > "$dir/manifest.json"
else
rm -f "$dir/manifest.json"
fi

echo "Download of images into '$dir' complete."
echo "Use something like the following to load the result into a Docker daemon:"
echo " tar -cC '$dir' . | docker load"

真的恶心死我了,搜了半天简中互联网大概是没人贴这种东西的,那我就简单发篇文章做做好事。算是给这几个月不顺的docker pull做个收尾吧。