7x31
/
public-image-mirror
mirror of https://github.com/DaoCloud/public-image-mirror.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactory

This commit is contained in:
Shiming Zhang 2021-11-18 19:13:57 +08:00
parent 6867da26c8
commit 6c1dbcdcb3
14 changed files with 300 additions and 321 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.github/workflows/sync-check.yml → .github/workflows/check.yml vendored
View File

@ -1,29 +1,29 @@
name: "Sync Check"
name: "Check"
on:
workflow_dispatch:
schedule:
- cron: "0 12 * * *"
- cron: "0 14 * * *"
jobs:
sync-check:
check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install dependencies
run: |
pip3 install yq
- name: Check
env:
INCREMENTAL: "true"
QUICKLY: "true"
PARALLET: "10"
run: |
./scripts/check-image.sh
./hack/diff.sh
- name: Update badge
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }} # Generated for update gist
BADGE_PREFIX: "gist://wzshiming/6e1f67a5184f93cefc5b2c670a5813e5"
QUICKLY: "true"
run: |
wget https://github.com/wzshiming/putingh/releases/download/v0.6.2/putingh_linux_amd64 -O /usr/local/bin/putingh && chmod +x /usr/local/bin/putingh
wget https://github.com/wzshiming/putingh/releases/download/v0.6.3/putingh_linux_amd64 -O /usr/local/bin/putingh && chmod +x /usr/local/bin/putingh
./hack/badge.sh
GH_TOKEN="${GH_TOKEN}" putingh "${BADGE_PREFIX}/daocloud-sync-badge.svg" badge.svg
GH_TOKEN="${GH_TOKEN}" putingh "${BADGE_PREFIX}/daocloud-sync.log" sync.log

16
.github/workflows/deep-sync-check.yml → .github/workflows/deep-check.yml vendored
View File

@ -1,29 +1,27 @@
name: "Deep Sync Check"
name: "Deep Check"
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * *"
- cron: "0 15 * * *"
jobs:
deep-sync-check:
check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install dependencies
run: |
pip3 install yq
- name: Check
env:
INCREMENTAL: "true"
PARALLET: "20"
PARALLET: "10"
run: |
./scripts/check-image.sh
./hack/diff.sh
- name: Update badge
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }} # Generated for update gist
BADGE_PREFIX: "gist://wzshiming/6e1f67a5184f93cefc5b2c670a5813e5"
run: |
wget https://github.com/wzshiming/putingh/releases/download/v0.6.2/putingh_linux_amd64 -O /usr/local/bin/putingh && chmod +x /usr/local/bin/putingh
wget https://github.com/wzshiming/putingh/releases/download/v0.6.3/putingh_linux_amd64 -O /usr/local/bin/putingh && chmod +x /usr/local/bin/putingh
./hack/badge.sh
GH_TOKEN="${GH_TOKEN}" putingh "${BADGE_PREFIX}/daocloud-deep-sync-badge.svg" badge.svg
GH_TOKEN="${GH_TOKEN}" putingh "${BADGE_PREFIX}/daocloud-deep-sync.log" sync.log

30
.github/workflows/deep-sync.yml vendored Normal file
View File

@ -0,0 +1,30 @@
name: Deep Sync
on:
schedule:
- cron: "0 7 * * *"
workflow_dispatch:
jobs:
sync-images:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Start Proxy
env:
only_proxy: ".m.daocloud.io"
run: |
wget https://github.com/wzshiming/bridge/releases/download/v0.7.6/bridge_linux_amd64 -O /usr/local/bin/bridge && chmod +x /usr/local/bin/bridge
nohup /usr/local/bin/bridge -b :8080 -p - ${{ secrets.PROTOCOL }} 2>/dev/null &
- name: Sync Images
env:
INCREMENTAL: "true"
SYNC: "true"
PARALLET: "5"
http_proxy: socks5://127.0.0.1:8080
https_proxy: socks5://127.0.0.1:8080
run: |
./hack/diff.sh

31
.github/workflows/sync.yml vendored Normal file
View File

@ -0,0 +1,31 @@
name: Sync
on:
schedule:
- cron: "0 * * * *"
workflow_dispatch:
jobs:
sync-images:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Start Proxy
env:
only_proxy: ".m.daocloud.io"
run: |
wget https://github.com/wzshiming/bridge/releases/download/v0.7.6/bridge_linux_amd64 -O /usr/local/bin/bridge && chmod +x /usr/local/bin/bridge
nohup /usr/local/bin/bridge -b :8080 -p - ${{ secrets.PROTOCOL }} 2>/dev/null &
- name: Sync Images
env:
INCREMENTAL: "true"
QUICKLY: "true"
SYNC: "true"
PARALLET: "5"
http_proxy: socks5://127.0.0.1:8080
https_proxy: socks5://127.0.0.1:8080
run: |
./hack/diff.sh

11
domain.txt
View File

@ -1,6 +1,5 @@
k8s.gcr.io=k8s-gcr.m.daocloud.io
gcr.io=gcr.m.daocloud.io
docker.io=docker.m.daocloud.io
quay.io=quay.m.daocloud.io
ghcr.io=ghcr.m.daocloud.io
k8s.gcr.io/=k8s-gcr.m.daocloud.io/
gcr.io/=gcr.m.daocloud.io/
docker.io/=docker.m.daocloud.io/
quay.io/=quay.m.daocloud.io/
ghcr.io/=ghcr.m.daocloud.io/

4
exclude.txt Normal file
View File

@ -0,0 +1,4 @@
.*[0-9a-f]{20,40}.*
.*windows.*
.*onbuild.*
sha256-.*

24
hack/badge.sh Executable file
View File

@ -0,0 +1,24 @@
#!/usr/bin/env bash
set -o errexit
set -o nounset
set -o pipefail
QUICKLY="${QUICKLY:-}"
cat *"_sync.log" > "sync.log"
sync="$(cat "sync.log" | grep " SYNCHRONIZED: " | wc -l | tr -d ' ' || :)"
unsync="$(cat "sync.log" | grep " NOT-SYNCHRONIZED: " | wc -l | tr -d ' ' || :)"
sum=$(($sync + $unsync))
if [[ "${sum}" -eq 0 ]]; then
echo "No sync"
exit 1
fi
if [[ "${QUICKLY}" == "true" ]]; then
echo "https://img.shields.io/badge/Sync-${sync}%2F${sum}-blue"
wget "https://img.shields.io/badge/Sync-${sync}%2F${sum}-blue" -O badge.svg
else
echo "https://img.shields.io/badge/Deep%20Sync-${sync}%2F${sum}-blue"
wget "https://img.shields.io/badge/Deep%20Sync-${sync}%2F${sum}-blue" -O badge.svg
fi

94
scripts/diff-image.sh → hack/diff-image.sh
View File

@ -4,7 +4,7 @@ set -o errexit
set -o nounset
set -o pipefail
# Output more information that is out of sync
# Output more information that is out of synchronize
DEBUG="${DEBUG:-}"
IMAGE1="${1:-}"
@ -28,6 +28,9 @@ SKIP="${SKIP:-}"
# Compare the number of tags in parallel
PARALLET="${PARALLET:-0}"
# Synchronize images from source to destination
SYNC="${SYNC:-}"
SELF="$(basename "${BASH_SOURCE[0]}")"
if [[ "${DEBUG}" == "true" ]]; then
@ -41,6 +44,7 @@ if [[ "${DEBUG}" == "true" ]]; then
echo "FOCUS: ${FOCUS}"
echo "SKIP: ${SKIP}"
echo "PARALLET: ${PARALLET}"
echo "SYNC: ${SYNC}"
fi
function check() {
@ -54,24 +58,25 @@ function check() {
echo " ${SELF}: <image1> <image2>"
echo " ${SELF}: <image1:tag> <image2:tag>"
echo "Env:"
echo " DEBUG=true # Output more information that is out of sync"
echo " DEBUG=true # Output more information that is out of synchronize"
echo " INCREMENTAL=true # Allow image2 to have more tags than image1"
echo " QUICKLY=true # Compare only tags that are in both images"
echo " FOCUS=<pattern> # Regexp that matches the tags"
echo " SKIP=<pattern> # Regexp that matches the tags that needs to be skipped"
echo " PARALLET=<size> # Compare the number of tags in parallel"
echo " SYNC=true # Synchronize images from source to destination"
return 2
fi
if [[ "${image1}" =~ ":" ]]; then
if [[ "${image2}" =~ ":" ]]; then
if [[ "${image1#*/}" =~ ":" ]]; then
if [[ "${image2#*/}" =~ ":" ]]; then
return 0
else
echo "${SELF}: ERROR: ${image1} and ${image2} must both be full images or not be tag references" >&2
return 2
fi
else
if [[ "${image2}" =~ ":" ]]; then
if [[ "${image2#*/}" =~ ":" ]]; then
echo "${SELF}: ERROR: ${image1} and ${image2} must both be full images or not be tag references" >&2
return 2
else
@ -80,6 +85,8 @@ function check() {
fi
}
emptyLayer="sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
function inspect() {
local image="${1:-}"
local raw=$(${SKOPEO} inspect --no-creds --raw --tls-verify=false "docker://${image}")
@ -92,7 +99,7 @@ function inspect() {
local schemaVersion=$(echo "${raw}" | ${JQ} -r '.schemaVersion')
case "${schemaVersion}" in
1)
echo "${raw}" | ${JQ} -r '.fsLayers[].blobSum' | tac
echo "${raw}" | ${JQ} -r '.fsLayers[].blobSum' | grep -v "${emptyLayer}" | tac
;;
2)
local mediaType=$(echo "${raw}" | ${JQ} -r '.mediaType // "" ')
@ -106,7 +113,7 @@ function inspect() {
case "${mediaType}" in
"layers" | "application/vnd.docker.distribution.manifest.v2+json")
echo "${raw}" | ${JQ} -r '.layers[].digest'
echo "${raw}" | ${JQ} -r '.layers[].digest' | grep -v "${emptyLayer}"
;;
"manifests" | "application/vnd.docker.distribution.manifest.list.v2+json")
local line=$(echo "${raw}" | ${JQ} -j '.manifests[] | .platform.architecture , " " , .platform.os , "\n"' | sort)
@ -140,15 +147,30 @@ function inspect() {
esac
}
function copy-image() {
local image1="${1:-}"
local image2="${2:-}"
${SKOPEO} copy --all --src-no-creds --dest-tls-verify=false --format oci "docker://${image1}" "docker://${image2}"
}
function list-tags() {
local image="${1:-}"
local raw="$(${SKOPEO} list-tags --tls-verify=false "docker://${image}" | ${JQ} -r '.Tags[]' | sort)"
local raw="$(${SKOPEO} list-tags --no-creds --tls-verify=false "docker://${image}" | ${JQ} -r '.Tags[]' | sort)"
if [[ "${FOCUS}" != "" ]]; then
local skip=$(echo "${raw}" | grep -v -E "${FOCUS}")
if [[ "${skip}" != "" ]]; then
echo "${SELF}: SKIP: ${image} with focus: ${FOCUS}:" ${skip} >&2
fi
raw="$(echo "${raw}" | grep -E "${FOCUS}" || :)"
fi
if [[ "${SKIP}" != "" ]]; then
local skip=$(echo "${raw}" | grep -E "${SKIP}")
if [[ "${skip}" != "" ]]; then
echo "${SELF}: SKIP: ${image} with skip: ${SKIP}:" ${skip} >&2
fi
raw="$(echo "${raw}" | grep -v -E "${SKIP}" || :)"
fi
echo "${raw}"
@ -162,16 +184,16 @@ function diff-image-with-tag() {
local tag1="${image1##*:}"
local tag2="${image2##*:}"
if [[ "${tag1}" != "${tag2}" ]]; then
echo "${SELF}: UNSYNC: ${image1} and ${image2} are not in synchronized" >&2
echo "${SELF}: NOT-SYNCHRONIZED: ${image1} and ${image2} are not in synchronized" >&2
return 1
fi
echo "${SELF}: SYNC: ${image1} and ${image2} are in synchronized" >&2
echo "${SELF}: SYNCHRONIZED: ${image1} and ${image2} are in synchronized" >&2
return 0
fi
local inspect2="$(inspect ${image2})"
if [[ "${inspect2}" == "" ]]; then
echo "${SELF}: UNSYNC: ${image1} and ${image2} are not in synchronized, ${image2} content is empty" >&2
echo "${SELF}: NOT-SYNCHRONIZED: ${image1} and ${image2} are not in synchronized, ${image2} content is empty" >&2
return 1
fi
@ -179,7 +201,7 @@ function diff-image-with-tag() {
local diff_raw=$(diff --unified <(echo "${inspect1}") <(echo "${inspect2}"))
if [[ "${diff_raw}" != "" ]]; then
echo "${SELF}: UNSYNC: ${image1} and ${image2} are not in synchronized" >&2
echo "${SELF}: NOT-SYNCHRONIZED: ${image1} and ${image2} are not in synchronized" >&2
if [[ "${DEBUG}" == "true" ]]; then
echo "DEBUG: image1 ${image1}:" >&2
echo "${inspect1}" >&2
@ -190,7 +212,7 @@ function diff-image-with-tag() {
fi
return 1
fi
echo "${SELF}: SYNC: ${image1} and ${image2} are in synchronized" >&2
echo "${SELF}: SYNCHRONIZED: ${image1} and ${image2} are in synchronized" >&2
}
function diff-image() {
@ -217,7 +239,7 @@ function diff-image() {
fi
if [[ "${reduce}" != "" ]] || [[ "${increase}" != "" ]]; then
echo "${SELF}: UNSYNC-TAGS: ${image1} and ${image2} are not in synchronized" >&2
echo "${SELF}: NOT-SYNCHRONIZED-TAGS: ${image1} and ${image2} are not in synchronized" >&2
if [[ "${DEBUG}" == "true" ]]; then
echo "DEBUG: image1 ${image1}:" >&2
echo "${tags1}" >&2
@ -227,15 +249,19 @@ function diff-image() {
echo "${diff_raw}" >&2
fi
for tag in ${reduce}; do
echo "${SELF}: UNSYNC: ${image1}:${tag} and ${image2}:${tag} are not in synchronized, ${image2}:${tag} does not exist" >&2
echo "${SELF}: NOT-SYNCHRONIZED: ${image1}:${tag} and ${image2}:${tag} are not in synchronized, ${image2}:${tag} does not exist" >&2
if [[ "${SYNC}" == "true" ]]; then
echo "${SELF}: SYNCHRONIZE: synchronize from ${image1}:${tag} to ${image2}:${tag}" >&2
copy-image "${image1}:${tag}" "${image2}:${tag}" >&2
fi
done
for tag in ${increase}; do
echo "${SELF}: UNSYNC: ${image1}:${tag} and ${image2}:${tag} are not in synchronized, ${image1}:${tag} does not exist" >&2
echo "${SELF}: NOT-SYNCHRONIZED: ${image1}:${tag} and ${image2}:${tag} are not in synchronized, ${image1}:${tag} does not exist" >&2
done
echo "${common}"
return 1
fi
echo "${SELF}: SYNC-TAGS: ${image1} and ${image2} are in synchronized" >&2
echo "${SELF}: SYNCHRONIZED-TAGS: ${image1} and ${image2} are in synchronized" >&2
echo "${common}"
return 0
}
@ -253,28 +279,46 @@ function main() {
local image1="${1:-}"
local image2="${2:-}"
if [[ "${image1}" =~ ":" ]]; then
diff-image-with-tag "${image1}" "${image2}" >/dev/null || return $?
if [[ "${image1#*/}" =~ ":" ]]; then
diff-image-with-tag "${image1}" "${image2}" >/dev/null || {
if [[ "${SYNC}" == "true" ]]; then
echo "${SELF}: SYNCHRONIZE: synchronize from ${image1}:${tag} to ${image2}:${tag}" >&2
copy-image "${image1}:${tag}" "${image2}:${tag}"
fi
return $?
}
return 0
fi
local list=$(diff-image "${image1}" "${image2}")
local unsync=()
if [[ "${QUICKLY}" == "true" ]] || [[ "${PARALLET}" -eq 0 ]]; then
local notsynced=()
if [[ "${PARALLET}" -eq 0 ]]; then
for tag in ${list}; do
diff-image-with-tag "${image1}:${tag}" "${image2}:${tag}" >/dev/null || unsync+=("${tag}")
diff-image-with-tag "${image1}:${tag}" "${image2}:${tag}" >/dev/null || {
if [[ "${SYNC}" == "true" ]]; then
echo "${SELF}: SYNCHRONIZE: synchronize from ${image1}:${tag} to ${image2}:${tag}" >&2
copy-image "${image1}:${tag}" "${image2}:${tag}"
fi
notsynced+=("${tag}")
}
done
else
for tag in ${list}; do
wait_jobs "${PARALLET}"
diff-image-with-tag "${image1}:${tag}" "${image2}:${tag}" >/dev/null || unsync+=("${tag}") &
diff-image-with-tag "${image1}:${tag}" "${image2}:${tag}" >/dev/null || {
if [[ "${SYNC}" == "true" ]]; then
echo "${SELF}: SYNCHRONIZE: synchronize from ${image1}:${tag} to ${image2}:${tag}" >&2
copy-image "${image1}:${tag}" "${image2}:${tag}"
fi
notsynced+=("${tag}")
} &
done
wait
fi
if [[ "${#unsync[@]}" -gt 0 ]]; then
echo "${SELF}: INFO: ${image1} and ${image2} are not in synchronized, there are unsynchronized tags ${#unsync[@]}: ${unsync[*]}" >&2
if [[ "${#notsynced[@]}" -gt 0 ]]; then
echo "${SELF}: INFO: ${image1} and ${image2} are not in synchronized, there are not synchronized tags ${#notsynced[@]}: ${notsynced[*]}" >&2
return 1
fi
}

74
hack/diff.sh Executable file
View File

@ -0,0 +1,74 @@
#!/usr/bin/env bash
set -o errexit
set -o nounset
set -o pipefail
source "$(dirname "${BASH_SOURCE}")/helper.sh"
cd "${ROOT}"
DEBUG="${DEBUG:-}"
INCREMENTAL="${INCREMENTAL:-}"
QUICKLY="${QUICKLY:-}"
SYNC="${SYNC:-}"
PARALLET="${PARALLET:-0}"
PARALLET_JOBS="${PARALLET_JOBS:-4}"
EXCLUDE="$(helper::exclude)"
declare -A DOMAIN_MAP=()
function wait_jobs() {
local job_num=${1:-3}
local perc=$(jobs -p | wc -l)
while [ "${perc}" -gt "${job_num}" ]; do
sleep 1
perc=$(jobs -p | wc -l)
done
}
function sync_with_domain() {
local domain="${1}"
local list=$(echo ${DOMAIN_MAP[${domain}]} | tr ' ' '\n' | shuf)
for image in ${list}; do
regex="${DEFAULT_REGEX}"
if [[ "${image#*/}" =~ ":" ]]; then
regex="${image##*:}"
image="${image%:*}"
fi
local to="$(helper::replace_domain "${domain}/${image}")"
local logfile="${to//\//_}_sync.log"
echo >"${logfile}"
DEBUG="${DEBUG}" SYNC="${SYNC}" QUICKLY="${QUICKLY}" INCREMENTAL="${INCREMENTAL}" PARALLET="${PARALLET}" FOCUS="${regex}" SKIP="${EXCLUDE}" ./hack/diff-image.sh "${domain}/${image}" "${to}" 2>&1 | tee -a "${logfile}" || {
echo "Error: diff image ${domain}/${image} $(helper::replace_domain "${domain}/${image}")"
}
done
}
function main() {
for image in $(helper::get_source); do
key="${image%%/*}"
val="${image#*/}"
if [[ -v "DOMAIN_MAP[${key}]" ]]; then
DOMAIN_MAP["${key}"]+=" ${val}"
else
DOMAIN_MAP["${key}"]="${val}"
fi
done
for domain in "${!DOMAIN_MAP[@]}"; do
if [[ "${PARALLET_JOBS}" -eq 0 ]]; then
sync_with_domain "${domain}"
else
wait_jobs "${PARALLET_JOBS}"
sync_with_domain "${domain}" &
fi
done
wait
}
main

3
hack/fmt.sh Executable file
View File

@ -0,0 +1,3 @@
#!/usr/bin/env bash
cat $1 | sort | tac > $1.tmp && mv $1.tmp $1

45
hack/helper.sh Executable file
View File

@ -0,0 +1,45 @@
#!/usr/bin/env bash
set -o errexit
set -o nounset
set -o pipefail
ROOT="${ROOT:-$(dirname "${BASH_SOURCE}")/..}"
DEFAULT_REGEX='^([a-z]+-)?[a-z]*[0-9]+(\.[0-9]+){0,2}(-.+)?$|^[a-z]+$'
SKOPEO="${SKOPEO:-skopeo}"
ROOT=$(realpath ${ROOT})
function helper::replace_domain() {
local domain="${1}"
local file="${2:-domain.txt}"
for line in $(cat ${file}); do
line="${line/ /}"
if [[ "${line}" == "" ]]; then
continue
fi
key="${line%=*}"
val="${line##*=}"
if [[ "${key}" == "" || "${val}" == "" ]]; then
continue
fi
if [[ "${domain}" =~ ^${key} ]]; then
echo "${domain}" | sed -e "s#^${key}#${val}#"
return 0
fi
done
echo "Error: domain ${domain} not found"
return 1
}
function helper::get_source() {
local source="${1:-mirror.txt}"
cat "${source}" | tr -d ' ' | grep -v -E '^$' | grep -v -E '^#'
}
function helper::exclude() {
local exclude="${1:-exclude.txt}"
local tmp=$(cat "${exclude}" | tr -d ' ' | grep -v -E '^$' | grep -v -E '^#' | tr '\n' '|')
echo "${tmp%|}"
}

31
not_sync.yaml
View File

@ -1,31 +0,0 @@
not_sync:
- image_pattern: quay.io/calico/.*
tag_patterns:
- ".*[0-9a-f]{7,40}.*"
- image_pattern: quay.io/tigera/.*
tag_patterns:
- ".*[0-9a-f]{7,40}.*"
- image_pattern: quay.io/jetstack/.*
tag_patterns:
- "[0-9a-f]{8,40}"
- image_pattern: docker.io/kubeovn/.*
tag_patterns:
- ".*[0-9a-f]{12,40}.*"
- image_pattern: docker.io/weaveworks/.*
tag_patterns:
- ".*[0-9a-f]{12,40}.*"
- image_pattern: docker.io/amazon/.*
tag_patterns:
- ".*[0-9a-f]{7,40}.*"
- image_pattern: docker.io/.*python
tag_patterns:
- ".*windows.*"
- image_pattern: docker.io/.*golang
tag_patterns:
- ".*windows.*"
- image_pattern: docker.io/.*node
tag_patterns:
- ".*onbuild.*"
- image_pattern: gcr.io/distroless/.*
tag_patterns:
- "sha256-.*"

88
scripts/check-image.sh
View File

@ -1,88 +0,0 @@
#!/usr/bin/env bash
set -o errexit
set -o nounset
set -o pipefail
DEBUG="${DEBUG:-}"
INCREMENTAL="${INCREMENTAL:-}"
QUICKLY="${QUICKLY:-}"
PARALLET="${PARALLET:-0}"
declare -A DOMAIN_MAP=()
for line in $(cat ./domain.txt); do
line="${line/ /}"
if [[ "$line" == "" ]]; then
continue
fi
key="${line%%=*}"
val="${line##*=}"
if [[ "${key}" == "" || "${val}" == "" ]]; then
echo "Error: invalid line: ${line}"
continue
fi
DOMAIN_MAP["${key}"]="${val}"
done
declare -A SKIP_MAP=()
for line in $(cat not_sync.yaml | yq -j '.not_sync[] | .image_pattern , "=", (.tag_patterns[] | . , "|" ) , "\n"' | sed "s/|$//g"); do
line="${line/ /}"
if [[ "$line" == "" ]]; then
continue
fi
key="${line%%=*}"
val="${line##*=}"
if [[ "${key}" == "" || "${val}" == "" ]]; then
echo "Error: invalid line: ${line}"
continue
fi
SKIP_MAP["${key}"]="${val}"
done
LOGFILE="./check-image.log"
echo >"${LOGFILE}"
count=0
images="$(cat ./mirror.txt | tr -d ' ' | grep -v -E '^$' | grep -v -E '^#' | shuf)"
images_count=$(echo "${images}" | wc -l | tr -d ' ')
for line in ${images}; do
line="${line/ /}"
if [[ "$line" == "" ]]; then
continue
fi
skip=""
for key in "${!SKIP_MAP[@]}"; do
if [[ "${line}" =~ ${key} ]]; then
skip+="${SKIP_MAP[$key]}|"
fi
done
skip="${skip%|}"
domain="${line%%/*}"
new_image=$(echo "${line}" | sed "s/^${domain}/${DOMAIN_MAP["${domain}"]}/g")
count=$((count + 1))
echo "Diff ${count}/${images_count} image ${line} ${new_image}"
DEBUG="${DEBUG}" QUICKLY="${QUICKLY}" INCREMENTAL="${INCREMENTAL}" PARALLET="${PARALLET}" SKIP="${skip}" ./scripts/diff-image.sh "${line}" "${new_image}" 2>&1 | tee -a "${LOGFILE}" || {
echo "Error: diff image ${line} ${new_image}"
}
done
cat "${LOGFILE}" | grep " UNSYNC: " | sort >sync.log
cat "${LOGFILE}" | grep " SYNC: " | sort >>sync.log
sync="$(cat "${LOGFILE}" | grep " SYNC: " | wc -l | tr -d ' ' || :)"
unsync="$(cat "${LOGFILE}" | grep " UNSYNC: " | wc -l | tr -d ' ' || :)"
sum=$(($sync + $unsync))
if [[ "${QUICKLY}" == "true" ]]; then
echo "https://img.shields.io/badge/Sync-${sync}%2F${sum}-blue"
wget "https://img.shields.io/badge/Sync-${sync}%2F${sum}-blue" -O badge.svg
else
echo "https://img.shields.io/badge/Deep%20Sync-${sync}%2F${sum}-blue"
wget "https://img.shields.io/badge/Deep%20Sync-${sync}%2F${sum}-blue" -O badge.svg
fi

154
scripts/sync-to-jp.py
View File

@ -1,154 +0,0 @@
import os
import re
import json
import yaml
import random
import subprocess
REPO_REPLACE_RULE1 = {
"k8s.gcr.io":"k8s-gcr-jp.m.daocloud.io"
}
REPO_REPLACE_RULE2 = {
"gcr.io":"gcr-jp.m.daocloud.io",
"docker.io":"docker-jp.m.daocloud.io",
"quay.io":"quay-jp.m.daocloud.io",
"ghcr.io":"ghcr-jp.m.daocloud.io"
}
# "":"",
INTERNAL_NETWORK = {
"k8s-gcr-jp.m.daocloud.io":"10.40.134.38:6001",
"gcr-jp.m.daocloud.io":"10.40.134.38:5001",
"docker-jp.m.daocloud.io":"10.40.134.38:7001",
"quay-jp.m.daocloud.io":"10.40.134.38:8001",
"ghcr-jp.m.daocloud.io":"10.40.134.38:9001"
}
REGISTRY_PASSWORD = os.environ["REGISTRY_PASSWORD"]
DELTA_MODE = os.environ.get("DELTA_MODE")
DRY_RUN = os.environ.get("DRY_RUN")
DOCKER_IO_USER = os.environ.get("DOCKER_IO_USER")
DOCKER_IO_PASS = os.environ.get("DOCKER_IO_PASS")
SKOPEO_CMD = "docker run --rm quay.io/containers/skopeo:latest"
# SKOPEO_CMD = "skopeo" # RUN without docker
def __run_cmd(cmd):
print(cmd)
if not DRY_RUN:
os.system(cmd)
def __dest_img(src_img):
dest_img = src_img
for k,v in REPO_REPLACE_RULE1.items():
dest_img = dest_img.replace(k,v)
for k,v in REPO_REPLACE_RULE2.items():
dest_img = dest_img.replace(k,v)
for k,v in INTERNAL_NETWORK.items():
dest_img = dest_img.replace(k,v)
return dest_img
def is_filtered(src_img):
y = []
with open("not_sync.yaml") as f:
y = yaml.safe_load(f)
for n in y.get('not_sync'):
if bool(re.match(n.get("image_pattern"), src_img)):
return True
return False
def __parse_tag(output):
# print(output)
try:
return json.loads(output).get('Tags',[])
except:
return []
def skepo_sync_one_tag(src_img,tag):
dest_img = __dest_img(src_img)
src_auth =''
if 'docker.io' in src_img and DOCKER_IO_USER:
src_auth = " --src-creds %s:%s " % (DOCKER_IO_USER,DOCKER_IO_PASS)
cmd = SKOPEO_CMD + " copy --all %s --dest-creds root:%s --dest-tls-verify=false -f oci docker://%s:%s docker://%s:%s" %(src_auth,REGISTRY_PASSWORD,src_img,tag,dest_img,tag)
__run_cmd(cmd)
def filter_tag(src_img,delta_tags):
y = []
need_to_sync = list(delta_tags)
with open("not_sync.yaml") as f:
y = yaml.safe_load(f)
for n in y.get('not_sync'):
if bool(re.match(n.get("image_pattern"), src_img)):
for t in delta_tags:
for tp in n.get("tag_patterns",[]):
if bool(re.match(tp, t)):
need_to_sync.remove(t)
return need_to_sync
def skepo_delta_sync(src_img,full_mode=False):
dest_img = __dest_img(src_img)
src_auth =''
if 'docker.io' in src_img and DOCKER_IO_USER:
src_auth = " --creds %s:%s " % (DOCKER_IO_USER,DOCKER_IO_PASS)
cmd = "skopeo list-tags %s docker://%s" % (src_auth,src_img)
print(cmd)
result = subprocess.run(cmd, shell=True,stdout=subprocess.PIPE)
src_tags = __parse_tag(result.stdout)
cmd = "skopeo list-tags --creds root:%s --tls-verify=false docker://%s" %(REGISTRY_PASSWORD,dest_img)
print(cmd)
result = subprocess.run(cmd, shell=True,stdout=subprocess.PIPE)
dest_tags = __parse_tag(result.stdout)
delta_tags = set(src_tags) - set(dest_tags)
need_to_sync = filter_tag(src_img,delta_tags)
filtered_num = len(delta_tags) - len(need_to_sync)
if full_mode:
need_to_sync = filter_tag(src_img,src_tags)
filtered_num = len(src_tags) - len(need_to_sync)
if 'latest' in src_tags and 'latest' not in need_to_sync:
need_to_sync.append('latest')
# print(src_img)
print("sync %s, src tag %s, dest tag %s, sync tag %s, filtered tag %s " % (src_img,len(src_tags),len(dest_tags),len(need_to_sync),filtered_num))
for tag in need_to_sync:
skepo_sync_one_tag(src_img,tag)
def skepo_full_sync(src_img):
if is_filtered(src_img):
skepo_delta_sync(src_img,full_mode=True)
else:
dest_img = __dest_img(src_img)
dest_img = "/".join(dest_img.split("/")[:-1])
src_auth = ""
if 'docker.io' in src_img and DOCKER_IO_USER:
src_auth = " --src-creds %s:%s " % (DOCKER_IO_USER,DOCKER_IO_PASS)
cmd = SKOPEO_CMD + " sync --all --src docker --dest docker %s --dest-tls-verify=false --dest-creds root:%s -f oci %s %s" %(src_auth,REGISTRY_PASSWORD,src_img,dest_img)
__run_cmd(cmd)
def main():
lines = []
with open("mirror.txt") as f:
lines = f.readlines()
random.shuffle(lines)
sync_cmds = []
for l in lines:
src_img = l.strip()
if DELTA_MODE:
print('use DELTA_MODE')
skepo_delta_sync(src_img)
else:
print('use FULL_SYNC_MODE')
skepo_full_sync(src_img)
if __name__ == "__main__":
# execute only if run as a script
main()