Merge remote-tracking branch 'origin/main'

2021-09-14 09:09:20 +08:00 · 2021-09-14 09:09:20 +08:00 · 59be4b7c9a
parent 09bbf64f95 e1143b8e4d
commit 59be4b7c9a
5 changed files with 162 additions and 60 deletions
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@ -10,6 +10,9 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: |
+          pip3 install yq
      - name: Check
        run: |
          ./scripts/check-image.sh
--- a/README.md
+++ b/README.md
@ -1,7 +1,7 @@
 # public-image-mirror

 ## 背景
-很多镜像都在国外。比如 gcr 。国内下载很忙，需要加速。
+很多镜像都在国外。比如 gcr 。国内下载很慢，需要加速。

 ## 目标

@ -67,5 +67,5 @@ python scripts/sync-to-jp.py
 ## 最佳实践
 * 通过 加速 安装 kubeadm
 * 通过 加速 运行 artifacthub 上的镜像
-
+* 通过 加速 安装 kind

--- a/mirror.txt
+++ b/mirror.txt
@ -3,9 +3,24 @@ docker.io/amazon/aws-ebs-csi-driver
 docker.io/busybox
 docker.io/centos
 docker.io/cloudnativelabs/kube-router
+docker.io/goharbor/chartmuseum-photon
+docker.io/goharbor/harbor-core
+docker.io/goharbor/harbor-db
+docker.io/goharbor/harbor-exporter
+docker.io/goharbor/harbor-jobservice
+docker.io/goharbor/harbor-portal
+docker.io/goharbor/harbor-registryctl
+docker.io/goharbor/nginx-photon
+docker.io/goharbor/notary-server-photon
+docker.io/goharbor/notary-signer-photon
+docker.io/goharbor/redis-photon
+docker.io/goharbor/registry-photon
+docker.io/goharbor/trivy-adapter-photon
 docker.io/haproxy
 docker.io/integratedcloudnative/ovn4nfv-k8s-plugin
 docker.io/k8scloudprovider/cinder-csi-plugin
+docker.io/kindest/node
+docker.io/kindest/haproxy
 docker.io/kubeovn/kube-ovn
 docker.io/kubernetesui/dashboard-amd64
 docker.io/kubernetesui/metrics-scraper
@ -24,6 +39,7 @@ docker.io/registry
 docker.io/weaveworks/weave-kube
 docker.io/weaveworks/weave-npc
 docker.io/xueshanf/install-socat
+gcr.io/distroless/static
 gcr.io/google-containers/pause
 ghcr.io/k8snetworkplumbingwg/multus-cni
 ghcr.io/klts-io/kubernetes-lts/coredns
--- a/scripts/check-image.sh
+++ b/scripts/check-image.sh
@ -21,16 +21,51 @@ for line in $(cat ./domain.txt); do
    DOMAIN_MAP["${key}"]="${val}"
 done

+declare -A EXCLUDED_MAP=()
+
+for line in $(cat not_sync.yaml | yq -j '.not_sync[] | .image_pattern , "=", (.tag_patterns[] | . , "|" ) , "\n"' | sed "s/|$//g"); do
+    line="${line/ /}"
+    if [[ "$line" == "" ]]; then
+        continue
+    fi
+    key="${line%%=*}"
+    val="${line##*=}"
+    if [[ "${key}" == "" || "${val}" == "" ]]; then
+        echo "Error: invalid line: ${line}"
+        continue
+    fi
+
+    EXCLUDED_MAP["${key}"]="${val}"
+done
+
+LOGFILE="./check-image.log"
+echo >"${LOGFILE}"
+
 for line in $(cat ./mirror.txt); do
    line="${line/ /}"
    if [[ "$line" == "" ]]; then
        continue
    fi

+    exclude=""
+    for key in "${!EXCLUDED_MAP[@]}"; do
+        if [[ "${line}" =~ ${key} ]]; then
+            exclude+="${EXCLUDED_MAP[$key]}|"
+        fi
+    done
+    exclude="${exclude%|}"
+
    domain="${line%%/*}"
    new_image=$(echo "${line}" | sed "s/^${domain}/${DOMAIN_MAP["${domain}"]}/g")
    echo "Diff image ${line} ${new_image}"
-    DEBUG=true INCREMENTAL=true ./scripts/diff-image.sh "${line}" "${new_image}" || {
+    DEBUG=true INCREMENTAL=true EXCLUDED="${exclude}" ./scripts/diff-image.sh "${line}" "${new_image}" 2>&1 | tee -a "${LOGFILE}" || {
        echo "Error: diff image ${line} ${new_image}"
    }
 done
+
+sync="$(cat "${LOGFILE}" | grep " SYNC: " | wc -l | tr -d ' ' || :)"
+nosync="$(cat "${LOGFILE}" | grep " NOSYNC: " | wc -l | tr -d ' ' || :)"
+sum=$(($sync + $nosync))
+
+echo https://img.shields.io/badge/Sync-${sync}%2F${sum}-blue
+wget "https://img.shields.io/badge/Sync-${sync}%2F${sum}-blue" -O sync.svg
--- a/scripts/diff-image.sh
+++ b/scripts/diff-image.sh
@ -13,6 +13,9 @@ DEBUG="${DEBUG:-}"
 # Allow image2 to have more tags than image1
 INCREMENTAL="${INCREMENTAL:-}"

+# Exclude tags that do not need to be checked
+EXCLUDED="${EXCLUDED:-}"
+
 SELF="$(basename "${BASH_SOURCE[0]}")"

 function check() {
@ -50,68 +53,117 @@ function check() {

 function inspect() {
    local image="${1:-}"
-    if [[ "${DEBUG}" == "true" ]]; then
-        echo skopeo inspect --retry-times=3 --raw --tls-verify=false "docker://${image}" >&2
+    local raw=$(skopeo inspect --raw --tls-verify=false "docker://${image}")
+    if [[ "${raw}" == "" ]]; then
+        echo "skopeo inspect --raw --tls-verify=false docker://${image}" >&2
+        echo "ERROR: Failed to inspect ${image}" >&2
+        return 1
    fi
-    skopeo inspect --retry-times=3 --raw --tls-verify=false "docker://${image}"
+
+    local schemaVersion=$(echo "${raw}" | jq -r '.schemaVersion')
+    case "${schemaVersion}" in
+    1)
+        echo "${raw}" | jq -r '.fsLayers[].blobSum'
+        ;;
+    2)
+        local mediaType=$(echo "${raw}" | jq -r '.mediaType // "" ')
+        case "${mediaType}" in
+        "application/vnd.docker.distribution.manifest.v2+json" | "")
+            echo "${raw}" | jq -r '.layers[].digest'
+            ;;
+        "application/vnd.docker.distribution.manifest.list.v2+json")
+            echo "${raw}" | jq -j '.manifests[] | .platform.architecture , " " , .platform.os , " " , .digest , "\n"' | sort
+            ;;
+        *)
+            echo "skopeo inspect --raw --tls-verify=false docker://${image}" >&2
+            if [[ "${DEBUG}" == "true" ]]; then
+                echo "${raw}" >&2
+            fi
+            echo "${SELF}: ERROR: Unknown media type: ${mediaType}" >&2
+            return 2
+            ;;
+        esac
+        ;;
+    *)
+        echo "skopeo inspect --raw --tls-verify=false docker://${image}" >&2
+        if [[ "${DEBUG}" == "true" ]]; then
+            echo "${raw}" >&2
+        fi
+        echo "${SELF}: ERROR: Unknown schema version: ${schemaVersion}" >&2
+        return 2
+        ;;
+    esac
 }

 function list-tags() {
    local image="${1:-}"
-    if [[ "${DEBUG}" == "true" ]]; then
-        echo skopeo list-tags --retry-times=3 --tls-verify=false "docker://${image}" >&2
+    local raw="$(skopeo list-tags --tls-verify=false "docker://${image}" | jq -r '.Tags[]' | sort)"
+
+    if [[ "${EXCLUDED}" != "" ]]; then
+        raw="$(echo "${raw}" | grep -v -E "${EXCLUDED}" || :)"
    fi
-    skopeo list-tags --retry-times=3 --tls-verify=false "docker://${image}"
+    echo "${raw}"
+}
+
+function diff-image-with-tag() {
+    local image1="${1:-}"
+    local image2="${2:-}"
+
+    local inspect1="$(inspect ${image1})"
+    local inspect2="$(inspect ${image2})"
+    local diff_raw=$(diff --unified <(echo "${inspect1}") <(echo "${inspect2}"))
+
+    if [[ "${diff_raw}" != "" ]]; then
+        echo "${SELF}: UNSYNC: ${image1} and ${image2} are not in synchronized" >&2
+        if [[ "${DEBUG}" == "true" ]]; then
+            echo "DEBUG: image1 ${image1}:" >&2
+            echo "${inspect1}" >&2
+            echo "DEBUG: image2 ${image2}:" >&2
+            echo "${inspect2}" >&2
+            echo "diff:" >&2
+            echo "${diff_raw}" >&2
+        fi
+        return 1
+    fi
+    echo "${SELF}: SYNC: ${image1} and ${image2} are in synchronized" >&2
 }

 function diff-image() {
    local image1="${1:-}"
    local image2="${2:-}"

-    if [[ "$image1" =~ ":" ]]; then
-        local inspect1="$(inspect ${image1} | jq -S 'del( .manifests[]?.mediaType, .layers[]?.mediaType, .config?, .mediaType?, .schemaVersion?, .signatures?)')"
-        local inspect2="$(inspect ${image2} | jq -S 'del( .manifests[]?.mediaType, .layers[]?.mediaType, .config?, .mediaType?, .schemaVersion?, .signatures?)')"
-        local diff_raw=$(diff --unified <(echo "${inspect1}") <(echo "${inspect2}"))
+    local tags1="$(list-tags ${image1})"
+    local tags2="$(list-tags ${image2})"
+    local diff_raw="$(diff --unified <(echo "${tags1}") <(echo "${tags2}") | grep -v -E '^---' | grep -v -E '^\+\+\+' || :)"
+    local diff_data="$(echo "${diff_raw}" | grep -v -E '^ ' || :)"

-        if [[ "${diff_raw}" != "" ]]; then
-            echo "${SELF}: UNSYNC: ${image1} and ${image2} are not in synchronized" >&2
-            if [[ "${DEBUG}" == "true" ]]; then
-                echo "DEBUG: image1 ${image1}:" >&2
-                echo "${inspect1}" >&2
-                echo "DEBUG: image2 ${image2}:" >&2
-                echo "${inspect2}" >&2
-                echo "diff:" >&2
-                echo "${diff_raw}" >&2
-            fi
-            return 1
-        fi
-        echo "${SELF}: SYNC: ${image1} and ${image2} are in synchronized" >&2
-        echo "${inspect1}"
-    else
-        local inspect1="$(list-tags ${image1} | jq -S '.')"
-        local inspect2="$(list-tags ${image2} | jq -S '.')"
-        local diff_raw="$(diff --unified <(echo "${inspect1}" | jq -S '.Tags[]' | tr -d '"') <(echo "${inspect2}" | jq -S '.Tags[]' | tr -d '"'))"
-        local diff_data="$(echo "${diff_raw}" | grep -v ' ' | grep -v -E '^---' | grep -v -E '^\+\+\+')"
-
-        if [[ "${INCREMENTAL}" == "true" ]]; then
-            diff_data="$(echo "${diff_data}" | grep -v -E '^\+')"
-        fi
-
-        if [[ "${diff_data}" != "" ]]; then
-            echo "${SELF}: UNSYNC: ${image1} and ${image2} are not in synchronized" >&2
-            if [[ "${DEBUG}" == "true" ]]; then
-                echo "DEBUG: image1 ${image1}:" >&2
-                echo "${inspect1}" >&2
-                echo "DEBUG: image2 ${image2}:" >&2
-                echo "${inspect2}" >&2
-                echo "DEBUG: diff:" >&2
-                echo "${diff_data}" >&2
-            fi
-            return 1
-        fi
-        echo "${SELF}: SYNC: ${image1} and ${image2} are in synchronized" >&2
-        echo "${inspect1}"
+    if [[ "${INCREMENTAL}" == "true" ]]; then
+        diff_data="$(echo "${diff_data}" | grep -v -E '^\+' || :)"
    fi
+
+    if [[ "${diff_data}" != "" ]]; then
+        echo "${SELF}: UNSYNC-TAGS: ${image1} and ${image2} are not in synchronized" >&2
+        if [[ "${DEBUG}" == "true" ]]; then
+            echo "DEBUG: image1 ${image1}:" >&2
+            echo "${tags1}" >&2
+            echo "DEBUG: image2 ${image2}:" >&2
+            echo "${tags2}" >&2
+            echo "DEBUG: diff:" >&2
+            echo "${diff_data}" >&2
+        fi
+        for tag in $(echo "${diff_raw}" | grep -E '^-' || :); do
+            tag="${tag#-}"
+            echo "${SELF}: UNSYNC: ${image1}:${tag} and ${image2}:${tag} are not in synchronized, ${image2}:${tag} is empty" >&2
+        done
+        for tag in $(echo "${diff_raw}" | grep -E '^\+' || :); do
+            tag="${tag#+}"
+            echo "${SELF}: UNSYNC: ${image1}:${tag} and ${image2}:${tag} are not in synchronized, ${image1}:${tag} is empty" >&2
+        done
+        echo "$(echo "${diff_raw}" | grep -E '^ ' | tr -d ' ' || :)"
+        return 1
+    fi
+    echo "${SELF}: SYNC-TAGS: ${image1} and ${image2} are in synchronized" >&2
+    echo "${tags1}"
    return 0
 }

@ -119,24 +171,20 @@ function main() {
    local image1="${1:-}"
    local image2="${2:-}"

-    raw=$(diff-image "${image1}" "${image2}")
    if [[ "${image1}" =~ ":" ]]; then
+        diff-image-with-tag "${image1}" "${image2}" >/dev/null || return $?
        return 0
    fi

-    local list=$(echo "${raw}" | jq '.Tags[]' | tr -d '"')
-    local total=$(echo "${list}" | wc -l | tr -d ' ')
-    local count=0
-    local unsync=()
+    local list=$(diff-image "${image1}" "${image2}")

+    local unsync=()
    for tag in ${list}; do
-        count=$((count + 1))
-        echo "${SELF}: DIFF ${count}/${total}: ${tag}"
-        diff-image "${image1}:${tag}" "${image2}:${tag}" >/dev/null || unsync+=("${tag}")
+        diff-image-with-tag "${image1}:${tag}" "${image2}:${tag}" >/dev/null || unsync+=("${tag}")
    done

    if [[ "${#unsync[@]}" -gt 0 ]]; then
-        echo "${SELF}: UNSYNC: ${image1} and ${image2} are not in synchronized, there are unsynchronized tags ${#unsync[@]}/${total}: ${unsync[*]}" >&2
+        echo "${SELF}: INFO: ${image1} and ${image2} are not in synchronized, there are unsynchronized tags ${#unsync[@]}: ${unsync[*]}" >&2
        return 1
    fi
 }