Merge remote-tracking branch 'origin/main'

.github/workflows/check.yml

@@ -10,6 +10,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: |
+          pip3 install yq
       - name: Check
         run: |
           ./scripts/check-image.sh

README.md

@@ -1,7 +1,7 @@
 # public-image-mirror
 
 ## 背景
-很多镜像都在国外。比如 gcr 。国内下载很忙，需要加速。
+很多镜像都在国外。比如 gcr 。国内下载很慢，需要加速。
 
 ## 目标
 
@@ -67,5 +67,5 @@ python scripts/sync-to-jp.py
 ## 最佳实践
 * 通过 加速 安装 kubeadm
 * 通过 加速 运行 artifacthub 上的镜像
-
+* 通过 加速 安装 kind
 

mirror.txt

@@ -3,9 +3,24 @@ docker.io/amazon/aws-ebs-csi-driver
 docker.io/busybox
 docker.io/centos
 docker.io/cloudnativelabs/kube-router
+docker.io/goharbor/chartmuseum-photon
+docker.io/goharbor/harbor-core
+docker.io/goharbor/harbor-db
+docker.io/goharbor/harbor-exporter
+docker.io/goharbor/harbor-jobservice
+docker.io/goharbor/harbor-portal
+docker.io/goharbor/harbor-registryctl
+docker.io/goharbor/nginx-photon
+docker.io/goharbor/notary-server-photon
+docker.io/goharbor/notary-signer-photon
+docker.io/goharbor/redis-photon
+docker.io/goharbor/registry-photon
+docker.io/goharbor/trivy-adapter-photon
 docker.io/haproxy
 docker.io/integratedcloudnative/ovn4nfv-k8s-plugin
 docker.io/k8scloudprovider/cinder-csi-plugin
+docker.io/kindest/node
+docker.io/kindest/haproxy
 docker.io/kubeovn/kube-ovn
 docker.io/kubernetesui/dashboard-amd64
 docker.io/kubernetesui/metrics-scraper
@@ -24,6 +39,7 @@ docker.io/registry
 docker.io/weaveworks/weave-kube
 docker.io/weaveworks/weave-npc
 docker.io/xueshanf/install-socat
+gcr.io/distroless/static
 gcr.io/google-containers/pause
 ghcr.io/k8snetworkplumbingwg/multus-cni
 ghcr.io/klts-io/kubernetes-lts/coredns

scripts/check-image.sh

@@ -21,16 +21,51 @@ for line in $(cat ./domain.txt); do
     DOMAIN_MAP["${key}"]="${val}"
 done
 
+declare -A EXCLUDED_MAP=()
+
+for line in $(cat not_sync.yaml | yq -j '.not_sync[] | .image_pattern , "=", (.tag_patterns[] | . , "|" ) , "\n"' | sed "s/|$//g"); do
+    line="${line/ /}"
+    if [[ "$line" == "" ]]; then
+        continue
+    fi
+    key="${line%%=*}"
+    val="${line##*=}"
+    if [[ "${key}" == "" || "${val}" == "" ]]; then
+        echo "Error: invalid line: ${line}"
+        continue
+    fi
+
+    EXCLUDED_MAP["${key}"]="${val}"
+done
+
+LOGFILE="./check-image.log"
+echo >"${LOGFILE}"
+
 for line in $(cat ./mirror.txt); do
     line="${line/ /}"
     if [[ "$line" == "" ]]; then
         continue
     fi
 
+    exclude=""
+    for key in "${!EXCLUDED_MAP[@]}"; do
+        if [[ "${line}" =~ ${key} ]]; then
+            exclude+="${EXCLUDED_MAP[$key]}|"
+        fi
+    done
+    exclude="${exclude%|}"
+
     domain="${line%%/*}"
     new_image=$(echo "${line}" | sed "s/^${domain}/${DOMAIN_MAP["${domain}"]}/g")
     echo "Diff image ${line} ${new_image}"
-    DEBUG=true INCREMENTAL=true ./scripts/diff-image.sh "${line}" "${new_image}" || {
+    DEBUG=true INCREMENTAL=true EXCLUDED="${exclude}" ./scripts/diff-image.sh "${line}" "${new_image}" 2>&1 | tee -a "${LOGFILE}" || {
         echo "Error: diff image ${line} ${new_image}"
     }
 done
+
+sync="$(cat "${LOGFILE}" | grep " SYNC: " | wc -l | tr -d ' ' || :)"
+nosync="$(cat "${LOGFILE}" | grep " NOSYNC: " | wc -l | tr -d ' ' || :)"
+sum=$(($sync + $nosync))
+
+echo https://img.shields.io/badge/Sync-${sync}%2F${sum}-blue
+wget "https://img.shields.io/badge/Sync-${sync}%2F${sum}-blue" -O sync.svg

scripts/diff-image.sh

@@ -13,6 +13,9 @@ DEBUG="${DEBUG:-}"
 # Allow image2 to have more tags than image1
 INCREMENTAL="${INCREMENTAL:-}"
 
+# Exclude tags that do not need to be checked
+EXCLUDED="${EXCLUDED:-}"
+
 SELF="$(basename "${BASH_SOURCE[0]}")"
 
 function check() {
@@ -50,27 +53,64 @@ function check() {
 
 function inspect() {
     local image="${1:-}"
-    if [[ "${DEBUG}" == "true" ]]; then
+    local raw=$(skopeo inspect --raw --tls-verify=false "docker://${image}")
-        echo skopeo inspect --retry-times=3 --raw --tls-verify=false "docker://${image}" >&2
+    if [[ "${raw}" == "" ]]; then
+        echo "skopeo inspect --raw --tls-verify=false docker://${image}" >&2
+        echo "ERROR: Failed to inspect ${image}" >&2
+        return 1
     fi
-    skopeo inspect --retry-times=3 --raw --tls-verify=false "docker://${image}"
+
+    local schemaVersion=$(echo "${raw}" | jq -r '.schemaVersion')
+    case "${schemaVersion}" in
+    1)
+        echo "${raw}" | jq -r '.fsLayers[].blobSum'
+        ;;
+    2)
+        local mediaType=$(echo "${raw}" | jq -r '.mediaType // "" ')
+        case "${mediaType}" in
+        "application/vnd.docker.distribution.manifest.v2+json" | "")
+            echo "${raw}" | jq -r '.layers[].digest'
+            ;;
+        "application/vnd.docker.distribution.manifest.list.v2+json")
+            echo "${raw}" | jq -j '.manifests[] | .platform.architecture , " " , .platform.os , " " , .digest , "\n"' | sort
+            ;;
+        *)
+            echo "skopeo inspect --raw --tls-verify=false docker://${image}" >&2
+            if [[ "${DEBUG}" == "true" ]]; then
+                echo "${raw}" >&2
+            fi
+            echo "${SELF}: ERROR: Unknown media type: ${mediaType}" >&2
+            return 2
+            ;;
+        esac
+        ;;
+    *)
+        echo "skopeo inspect --raw --tls-verify=false docker://${image}" >&2
+        if [[ "${DEBUG}" == "true" ]]; then
+            echo "${raw}" >&2
+        fi
+        echo "${SELF}: ERROR: Unknown schema version: ${schemaVersion}" >&2
+        return 2
+        ;;
+    esac
 }
 
 function list-tags() {
     local image="${1:-}"
-    if [[ "${DEBUG}" == "true" ]]; then
+    local raw="$(skopeo list-tags --tls-verify=false "docker://${image}" | jq -r '.Tags[]' | sort)"
-        echo skopeo list-tags --retry-times=3 --tls-verify=false "docker://${image}" >&2
+
+    if [[ "${EXCLUDED}" != "" ]]; then
+        raw="$(echo "${raw}" | grep -v -E "${EXCLUDED}" || :)"
     fi
-    skopeo list-tags --retry-times=3 --tls-verify=false "docker://${image}"
+    echo "${raw}"
 }
 
-function diff-image() {
+function diff-image-with-tag() {
     local image1="${1:-}"
     local image2="${2:-}"
 
-    if [[ "$image1" =~ ":" ]]; then
+    local inspect1="$(inspect ${image1})"
-        local inspect1="$(inspect ${image1} | jq -S 'del( .manifests[]?.mediaType, .layers[]?.mediaType, .config?, .mediaType?, .schemaVersion?, .signatures?)')"
+    local inspect2="$(inspect ${image2})"
-        local inspect2="$(inspect ${image2} | jq -S 'del( .manifests[]?.mediaType, .layers[]?.mediaType, .config?, .mediaType?, .schemaVersion?, .signatures?)')"
     local diff_raw=$(diff --unified <(echo "${inspect1}") <(echo "${inspect2}"))
 
     if [[ "${diff_raw}" != "" ]]; then
@@ -86,32 +126,44 @@ function diff-image() {
         return 1
     fi
     echo "${SELF}: SYNC: ${image1} and ${image2} are in synchronized" >&2
-        echo "${inspect1}"
+}
-    else
+
-        local inspect1="$(list-tags ${image1} | jq -S '.')"
+function diff-image() {
-        local inspect2="$(list-tags ${image2} | jq -S '.')"
+    local image1="${1:-}"
-        local diff_raw="$(diff --unified <(echo "${inspect1}" | jq -S '.Tags[]' | tr -d '"') <(echo "${inspect2}" | jq -S '.Tags[]' | tr -d '"'))"
+    local image2="${2:-}"
-        local diff_data="$(echo "${diff_raw}" | grep -v ' ' | grep -v -E '^---' | grep -v -E '^\+\+\+')"
+
+    local tags1="$(list-tags ${image1})"
+    local tags2="$(list-tags ${image2})"
+    local diff_raw="$(diff --unified <(echo "${tags1}") <(echo "${tags2}") | grep -v -E '^---' | grep -v -E '^\+\+\+' || :)"
+    local diff_data="$(echo "${diff_raw}" | grep -v -E '^ ' || :)"
 
     if [[ "${INCREMENTAL}" == "true" ]]; then
-            diff_data="$(echo "${diff_data}" | grep -v -E '^\+')"
+        diff_data="$(echo "${diff_data}" | grep -v -E '^\+' || :)"
     fi
 
     if [[ "${diff_data}" != "" ]]; then
-            echo "${SELF}: UNSYNC: ${image1} and ${image2} are not in synchronized" >&2
+        echo "${SELF}: UNSYNC-TAGS: ${image1} and ${image2} are not in synchronized" >&2
         if [[ "${DEBUG}" == "true" ]]; then
             echo "DEBUG: image1 ${image1}:" >&2
-                echo "${inspect1}" >&2
+            echo "${tags1}" >&2
             echo "DEBUG: image2 ${image2}:" >&2
-                echo "${inspect2}" >&2
+            echo "${tags2}" >&2
             echo "DEBUG: diff:" >&2
             echo "${diff_data}" >&2
         fi
+        for tag in $(echo "${diff_raw}" | grep -E '^-' || :); do
+            tag="${tag#-}"
+            echo "${SELF}: UNSYNC: ${image1}:${tag} and ${image2}:${tag} are not in synchronized, ${image2}:${tag} is empty" >&2
+        done
+        for tag in $(echo "${diff_raw}" | grep -E '^\+' || :); do
+            tag="${tag#+}"
+            echo "${SELF}: UNSYNC: ${image1}:${tag} and ${image2}:${tag} are not in synchronized, ${image1}:${tag} is empty" >&2
+        done
+        echo "$(echo "${diff_raw}" | grep -E '^ ' | tr -d ' ' || :)"
         return 1
     fi
-        echo "${SELF}: SYNC: ${image1} and ${image2} are in synchronized" >&2
+    echo "${SELF}: SYNC-TAGS: ${image1} and ${image2} are in synchronized" >&2
-        echo "${inspect1}"
+    echo "${tags1}"
-    fi
     return 0
 }
 
@@ -119,24 +171,20 @@ function main() {
     local image1="${1:-}"
     local image2="${2:-}"
 
-    raw=$(diff-image "${image1}" "${image2}")
     if [[ "${image1}" =~ ":" ]]; then
+        diff-image-with-tag "${image1}" "${image2}" >/dev/null || return $?
         return 0
     fi
 
-    local list=$(echo "${raw}" | jq '.Tags[]' | tr -d '"')
+    local list=$(diff-image "${image1}" "${image2}")
-    local total=$(echo "${list}" | wc -l | tr -d ' ')
-    local count=0
-    local unsync=()
 
+    local unsync=()
     for tag in ${list}; do
-        count=$((count + 1))
+        diff-image-with-tag "${image1}:${tag}" "${image2}:${tag}" >/dev/null || unsync+=("${tag}")
-        echo "${SELF}: DIFF ${count}/${total}: ${tag}"
-        diff-image "${image1}:${tag}" "${image2}:${tag}" >/dev/null || unsync+=("${tag}")
     done
 
     if [[ "${#unsync[@]}" -gt 0 ]]; then
-        echo "${SELF}: UNSYNC: ${image1} and ${image2} are not in synchronized, there are unsynchronized tags ${#unsync[@]}/${total}: ${unsync[*]}" >&2
+        echo "${SELF}: INFO: ${image1} and ${image2} are not in synchronized, there are unsynchronized tags ${#unsync[@]}: ${unsync[*]}" >&2
         return 1
     fi
 }