Refactory

.github/workflows/check.yml

@@ -1,29 +1,29 @@
-name: "Sync Check"
+name: "Check"
 on:
   workflow_dispatch:
 
   schedule:
-    - cron: "0 12 * * *"
+    - cron: "0 14 * * *"
 
 jobs:
-  sync-check:
+  check:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install dependencies
-        run: |
-          pip3 install yq
       - name: Check
         env:
           INCREMENTAL: "true"
           QUICKLY: "true"
+          PARALLET: "10"
         run: |
-          ./scripts/check-image.sh
+          ./hack/diff.sh
       - name: Update badge
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }} # Generated for update gist
           BADGE_PREFIX: "gist://wzshiming/6e1f67a5184f93cefc5b2c670a5813e5"
+          QUICKLY: "true"  
         run: |
-          wget https://github.com/wzshiming/putingh/releases/download/v0.6.2/putingh_linux_amd64 -O /usr/local/bin/putingh && chmod +x /usr/local/bin/putingh
+          wget https://github.com/wzshiming/putingh/releases/download/v0.6.3/putingh_linux_amd64 -O /usr/local/bin/putingh && chmod +x /usr/local/bin/putingh
+          ./hack/badge.sh
           GH_TOKEN="${GH_TOKEN}" putingh "${BADGE_PREFIX}/daocloud-sync-badge.svg" badge.svg
           GH_TOKEN="${GH_TOKEN}" putingh "${BADGE_PREFIX}/daocloud-sync.log" sync.log

.github/workflows/deep-check.yml

@@ -1,29 +1,27 @@
-name: "Deep Sync Check"
+name: "Deep Check"
 on:
   workflow_dispatch:
 
   schedule:
-    - cron: "0 0 * * *"
+    - cron: "0 15 * * *"
 
 jobs:
-  deep-sync-check:
+  check:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install dependencies
-        run: |
-          pip3 install yq
       - name: Check
         env:
           INCREMENTAL: "true"
-          PARALLET: "20"
+          PARALLET: "10"
         run: |
-          ./scripts/check-image.sh
+          ./hack/diff.sh
       - name: Update badge
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }} # Generated for update gist
           BADGE_PREFIX: "gist://wzshiming/6e1f67a5184f93cefc5b2c670a5813e5"
         run: |
-          wget https://github.com/wzshiming/putingh/releases/download/v0.6.2/putingh_linux_amd64 -O /usr/local/bin/putingh && chmod +x /usr/local/bin/putingh
+          wget https://github.com/wzshiming/putingh/releases/download/v0.6.3/putingh_linux_amd64 -O /usr/local/bin/putingh && chmod +x /usr/local/bin/putingh
+          ./hack/badge.sh
           GH_TOKEN="${GH_TOKEN}" putingh "${BADGE_PREFIX}/daocloud-deep-sync-badge.svg" badge.svg
           GH_TOKEN="${GH_TOKEN}" putingh "${BADGE_PREFIX}/daocloud-deep-sync.log" sync.log

.github/workflows/deep-sync.yml

@@ -0,0 +1,30 @@
+name: Deep Sync
+
+on:
+  schedule:
+    - cron: "0 7 * * *"
+
+  workflow_dispatch:
+
+jobs:
+  sync-images:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Start Proxy
+        env:
+          only_proxy: ".m.daocloud.io"
+        run: |
+          wget https://github.com/wzshiming/bridge/releases/download/v0.7.6/bridge_linux_amd64 -O /usr/local/bin/bridge && chmod +x /usr/local/bin/bridge
+          nohup /usr/local/bin/bridge -b :8080 -p - ${{ secrets.PROTOCOL }} 2>/dev/null &
+
+      - name: Sync Images
+        env:
+          INCREMENTAL: "true"
+          SYNC: "true"
+          PARALLET: "5"
+          http_proxy: socks5://127.0.0.1:8080
+          https_proxy: socks5://127.0.0.1:8080
+        run: |
+          ./hack/diff.sh

.github/workflows/sync.yml

@@ -0,0 +1,31 @@
+name: Sync
+
+on:
+  schedule:
+    - cron: "0 * * * *"
+
+  workflow_dispatch:
+
+jobs:
+  sync-images:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Start Proxy
+        env:
+          only_proxy: ".m.daocloud.io"
+        run: |
+          wget https://github.com/wzshiming/bridge/releases/download/v0.7.6/bridge_linux_amd64 -O /usr/local/bin/bridge && chmod +x /usr/local/bin/bridge
+          nohup /usr/local/bin/bridge -b :8080 -p - ${{ secrets.PROTOCOL }} 2>/dev/null &
+
+      - name: Sync Images
+        env:
+          INCREMENTAL: "true"
+          QUICKLY: "true"
+          SYNC: "true"
+          PARALLET: "5"
+          http_proxy: socks5://127.0.0.1:8080
+          https_proxy: socks5://127.0.0.1:8080
+        run: |
+          ./hack/diff.sh

domain.txt

@@ -1,6 +1,5 @@
-k8s.gcr.io=k8s-gcr.m.daocloud.io
+k8s.gcr.io/=k8s-gcr.m.daocloud.io/
-gcr.io=gcr.m.daocloud.io
+gcr.io/=gcr.m.daocloud.io/
-docker.io=docker.m.daocloud.io
+docker.io/=docker.m.daocloud.io/
-quay.io=quay.m.daocloud.io
+quay.io/=quay.m.daocloud.io/
-ghcr.io=ghcr.m.daocloud.io
+ghcr.io/=ghcr.m.daocloud.io/
-

exclude.txt

@@ -0,0 +1,4 @@
+.*[0-9a-f]{20,40}.*
+.*windows.*
+.*onbuild.*
+sha256-.*

hack/badge.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+QUICKLY="${QUICKLY:-}"
+cat *"_sync.log" > "sync.log"
+sync="$(cat "sync.log" | grep " SYNCHRONIZED: " | wc -l | tr -d ' ' || :)"
+unsync="$(cat "sync.log" | grep " NOT-SYNCHRONIZED: " | wc -l | tr -d ' ' || :)"
+sum=$(($sync + $unsync))
+
+if [[ "${sum}" -eq 0 ]]; then
+    echo "No sync"
+    exit 1    
+fi
+
+if [[ "${QUICKLY}" == "true" ]]; then
+    echo "https://img.shields.io/badge/Sync-${sync}%2F${sum}-blue"
+    wget "https://img.shields.io/badge/Sync-${sync}%2F${sum}-blue" -O badge.svg
+else
+    echo "https://img.shields.io/badge/Deep%20Sync-${sync}%2F${sum}-blue"
+    wget "https://img.shields.io/badge/Deep%20Sync-${sync}%2F${sum}-blue" -O badge.svg
+fi

hack/diff-image.sh

@@ -4,7 +4,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-# Output more information that is out of sync
+# Output more information that is out of synchronize
 DEBUG="${DEBUG:-}"
 
 IMAGE1="${1:-}"
@@ -28,6 +28,9 @@ SKIP="${SKIP:-}"
 # Compare the number of tags in parallel
 PARALLET="${PARALLET:-0}"
 
+# Synchronize images from source to destination
+SYNC="${SYNC:-}"
+
 SELF="$(basename "${BASH_SOURCE[0]}")"
 
 if [[ "${DEBUG}" == "true" ]]; then
@@ -41,6 +44,7 @@ if [[ "${DEBUG}" == "true" ]]; then
     echo "FOCUS:       ${FOCUS}"
     echo "SKIP:        ${SKIP}"
     echo "PARALLET:    ${PARALLET}"
+    echo "SYNC:        ${SYNC}"
 fi
 
 function check() {
@@ -54,24 +58,25 @@ function check() {
         echo " ${SELF}: <image1> <image2>"
         echo " ${SELF}: <image1:tag> <image2:tag>"
         echo "Env:"
-        echo " DEBUG=true         # Output more information that is out of sync"
+        echo " DEBUG=true         # Output more information that is out of synchronize"
         echo " INCREMENTAL=true   # Allow image2 to have more tags than image1"
         echo " QUICKLY=true       # Compare only tags that are in both images"
         echo " FOCUS=<pattern>    # Regexp that matches the tags"
         echo " SKIP=<pattern>     # Regexp that matches the tags that needs to be skipped"
         echo " PARALLET=<size>    # Compare the number of tags in parallel"
+        echo " SYNC=true          # Synchronize images from source to destination"
         return 2
     fi
 
-    if [[ "${image1}" =~ ":" ]]; then
+    if [[ "${image1#*/}" =~ ":" ]]; then
-        if [[ "${image2}" =~ ":" ]]; then
+        if [[ "${image2#*/}" =~ ":" ]]; then
             return 0
         else
             echo "${SELF}: ERROR: ${image1} and ${image2} must both be full images or not be tag references" >&2
             return 2
         fi
     else
-        if [[ "${image2}" =~ ":" ]]; then
+        if [[ "${image2#*/}" =~ ":" ]]; then
             echo "${SELF}: ERROR: ${image1} and ${image2} must both be full images or not be tag references" >&2
             return 2
         else
@@ -80,6 +85,8 @@ function check() {
     fi
 }
 
+emptyLayer="sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
+
 function inspect() {
     local image="${1:-}"
     local raw=$(${SKOPEO} inspect --no-creds --raw --tls-verify=false "docker://${image}")
@@ -92,7 +99,7 @@ function inspect() {
     local schemaVersion=$(echo "${raw}" | ${JQ} -r '.schemaVersion')
     case "${schemaVersion}" in
     1)
-        echo "${raw}" | ${JQ} -r '.fsLayers[].blobSum' | tac
+        echo "${raw}" | ${JQ} -r '.fsLayers[].blobSum' | grep -v "${emptyLayer}" | tac
         ;;
     2)
         local mediaType=$(echo "${raw}" | ${JQ} -r '.mediaType // "" ')
@@ -106,7 +113,7 @@ function inspect() {
 
         case "${mediaType}" in
         "layers" | "application/vnd.docker.distribution.manifest.v2+json")
-            echo "${raw}" | ${JQ} -r '.layers[].digest'
+            echo "${raw}" | ${JQ} -r '.layers[].digest' | grep -v "${emptyLayer}"
             ;;
         "manifests" | "application/vnd.docker.distribution.manifest.list.v2+json")
             local line=$(echo "${raw}" | ${JQ} -j '.manifests[] | .platform.architecture , " " , .platform.os , "\n"' | sort)
@@ -140,15 +147,30 @@ function inspect() {
     esac
 }
 
+function copy-image() {
+    local image1="${1:-}"
+    local image2="${2:-}"
+
+    ${SKOPEO} copy --all --src-no-creds --dest-tls-verify=false --format oci "docker://${image1}" "docker://${image2}"
+}
+
 function list-tags() {
     local image="${1:-}"
-    local raw="$(${SKOPEO} list-tags --tls-verify=false "docker://${image}" | ${JQ} -r '.Tags[]' | sort)"
+    local raw="$(${SKOPEO} list-tags --no-creds --tls-verify=false "docker://${image}" | ${JQ} -r '.Tags[]' | sort)"
 
     if [[ "${FOCUS}" != "" ]]; then
+        local skip=$(echo "${raw}" | grep -v -E "${FOCUS}")
+        if [[ "${skip}" != "" ]]; then
+            echo "${SELF}: SKIP: ${image} with focus: ${FOCUS}:" ${skip} >&2
+        fi
         raw="$(echo "${raw}" | grep -E "${FOCUS}" || :)"
     fi
 
     if [[ "${SKIP}" != "" ]]; then
+        local skip=$(echo "${raw}" | grep -E "${SKIP}")
+        if [[ "${skip}" != "" ]]; then
+            echo "${SELF}: SKIP: ${image} with skip: ${SKIP}:" ${skip} >&2
+        fi
         raw="$(echo "${raw}" | grep -v -E "${SKIP}" || :)"
     fi
     echo "${raw}"
@@ -162,16 +184,16 @@ function diff-image-with-tag() {
         local tag1="${image1##*:}"
         local tag2="${image2##*:}"
         if [[ "${tag1}" != "${tag2}" ]]; then
-            echo "${SELF}: UNSYNC: ${image1} and ${image2} are not in synchronized" >&2
+            echo "${SELF}: NOT-SYNCHRONIZED: ${image1} and ${image2} are not in synchronized" >&2
             return 1
         fi
-        echo "${SELF}: SYNC: ${image1} and ${image2} are in synchronized" >&2
+        echo "${SELF}: SYNCHRONIZED: ${image1} and ${image2} are in synchronized" >&2
         return 0
     fi
 
     local inspect2="$(inspect ${image2})"
     if [[ "${inspect2}" == "" ]]; then
-        echo "${SELF}: UNSYNC: ${image1} and ${image2} are not in synchronized, ${image2} content is empty" >&2
+        echo "${SELF}: NOT-SYNCHRONIZED: ${image1} and ${image2} are not in synchronized, ${image2} content is empty" >&2
         return 1
     fi
 
@@ -179,7 +201,7 @@ function diff-image-with-tag() {
     local diff_raw=$(diff --unified <(echo "${inspect1}") <(echo "${inspect2}"))
 
     if [[ "${diff_raw}" != "" ]]; then
-        echo "${SELF}: UNSYNC: ${image1} and ${image2} are not in synchronized" >&2
+        echo "${SELF}: NOT-SYNCHRONIZED: ${image1} and ${image2} are not in synchronized" >&2
         if [[ "${DEBUG}" == "true" ]]; then
             echo "DEBUG: image1 ${image1}:" >&2
             echo "${inspect1}" >&2
@@ -190,7 +212,7 @@ function diff-image-with-tag() {
         fi
         return 1
     fi
-    echo "${SELF}: SYNC: ${image1} and ${image2} are in synchronized" >&2
+    echo "${SELF}: SYNCHRONIZED: ${image1} and ${image2} are in synchronized" >&2
 }
 
 function diff-image() {
@@ -217,7 +239,7 @@ function diff-image() {
     fi
 
     if [[ "${reduce}" != "" ]] || [[ "${increase}" != "" ]]; then
-        echo "${SELF}: UNSYNC-TAGS: ${image1} and ${image2} are not in synchronized" >&2
+        echo "${SELF}: NOT-SYNCHRONIZED-TAGS: ${image1} and ${image2} are not in synchronized" >&2
         if [[ "${DEBUG}" == "true" ]]; then
             echo "DEBUG: image1 ${image1}:" >&2
             echo "${tags1}" >&2
@@ -227,15 +249,19 @@ function diff-image() {
             echo "${diff_raw}" >&2
         fi
         for tag in ${reduce}; do
-            echo "${SELF}: UNSYNC: ${image1}:${tag} and ${image2}:${tag} are not in synchronized, ${image2}:${tag} does not exist" >&2
+            echo "${SELF}: NOT-SYNCHRONIZED: ${image1}:${tag} and ${image2}:${tag} are not in synchronized, ${image2}:${tag} does not exist" >&2
+            if [[ "${SYNC}" == "true" ]]; then
+                echo "${SELF}: SYNCHRONIZE: synchronize from ${image1}:${tag} to ${image2}:${tag}" >&2
+                copy-image "${image1}:${tag}" "${image2}:${tag}" >&2
+            fi
         done
         for tag in ${increase}; do
-            echo "${SELF}: UNSYNC: ${image1}:${tag} and ${image2}:${tag} are not in synchronized, ${image1}:${tag} does not exist" >&2
+            echo "${SELF}: NOT-SYNCHRONIZED: ${image1}:${tag} and ${image2}:${tag} are not in synchronized, ${image1}:${tag} does not exist" >&2
         done
         echo "${common}"
         return 1
     fi
-    echo "${SELF}: SYNC-TAGS: ${image1} and ${image2} are in synchronized" >&2
+    echo "${SELF}: SYNCHRONIZED-TAGS: ${image1} and ${image2} are in synchronized" >&2
     echo "${common}"
     return 0
 }
@@ -253,28 +279,46 @@ function main() {
     local image1="${1:-}"
     local image2="${2:-}"
 
-    if [[ "${image1}" =~ ":" ]]; then
+    if [[ "${image1#*/}" =~ ":" ]]; then
-        diff-image-with-tag "${image1}" "${image2}" >/dev/null || return $?
+        diff-image-with-tag "${image1}" "${image2}" >/dev/null || {
+            if [[ "${SYNC}" == "true" ]]; then
+                echo "${SELF}: SYNCHRONIZE: synchronize from ${image1}:${tag} to ${image2}:${tag}" >&2
+                copy-image "${image1}:${tag}" "${image2}:${tag}"
+            fi
+            return $?
+        }
         return 0
     fi
 
     local list=$(diff-image "${image1}" "${image2}")
 
-    local unsync=()
+    local notsynced=()
-    if [[ "${QUICKLY}" == "true" ]] || [[ "${PARALLET}" -eq 0 ]]; then
+    if [[ "${PARALLET}" -eq 0 ]]; then
         for tag in ${list}; do
-            diff-image-with-tag "${image1}:${tag}" "${image2}:${tag}" >/dev/null || unsync+=("${tag}")
+            diff-image-with-tag "${image1}:${tag}" "${image2}:${tag}" >/dev/null || {
+                if [[ "${SYNC}" == "true" ]]; then
+                    echo "${SELF}: SYNCHRONIZE: synchronize from ${image1}:${tag} to ${image2}:${tag}" >&2
+                    copy-image "${image1}:${tag}" "${image2}:${tag}"
+                fi
+                notsynced+=("${tag}")
+            }
         done
     else
         for tag in ${list}; do
             wait_jobs "${PARALLET}"
-            diff-image-with-tag "${image1}:${tag}" "${image2}:${tag}" >/dev/null || unsync+=("${tag}") &
+            diff-image-with-tag "${image1}:${tag}" "${image2}:${tag}" >/dev/null || {
+                if [[ "${SYNC}" == "true" ]]; then
+                    echo "${SELF}: SYNCHRONIZE: synchronize from ${image1}:${tag} to ${image2}:${tag}" >&2
+                    copy-image "${image1}:${tag}" "${image2}:${tag}"
+                fi
+                notsynced+=("${tag}")
+            } &
         done
         wait
     fi
 
-    if [[ "${#unsync[@]}" -gt 0 ]]; then
+    if [[ "${#notsynced[@]}" -gt 0 ]]; then
-        echo "${SELF}: INFO: ${image1} and ${image2} are not in synchronized, there are unsynchronized tags ${#unsync[@]}: ${unsync[*]}" >&2
+        echo "${SELF}: INFO: ${image1} and ${image2} are not in synchronized, there are not synchronized tags ${#notsynced[@]}: ${notsynced[*]}" >&2
         return 1
     fi
 }

hack/diff.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source "$(dirname "${BASH_SOURCE}")/helper.sh"
+cd "${ROOT}"
+
+DEBUG="${DEBUG:-}"
+INCREMENTAL="${INCREMENTAL:-}"
+QUICKLY="${QUICKLY:-}"
+SYNC="${SYNC:-}"
+PARALLET="${PARALLET:-0}"
+PARALLET_JOBS="${PARALLET_JOBS:-4}"
+EXCLUDE="$(helper::exclude)"
+
+declare -A DOMAIN_MAP=()
+
+function wait_jobs() {
+    local job_num=${1:-3}
+    local perc=$(jobs -p | wc -l)
+    while [ "${perc}" -gt "${job_num}" ]; do
+        sleep 1
+        perc=$(jobs -p | wc -l)
+    done
+}
+
+function sync_with_domain() {
+    local domain="${1}"
+
+    local list=$(echo ${DOMAIN_MAP[${domain}]} | tr ' ' '\n' | shuf)
+    for image in ${list}; do
+        regex="${DEFAULT_REGEX}"
+        if [[ "${image#*/}" =~ ":" ]]; then
+            regex="${image##*:}"
+            image="${image%:*}"
+        fi
+        local to="$(helper::replace_domain "${domain}/${image}")"
+
+        local logfile="${to//\//_}_sync.log"
+        echo >"${logfile}"
+
+        DEBUG="${DEBUG}" SYNC="${SYNC}" QUICKLY="${QUICKLY}" INCREMENTAL="${INCREMENTAL}" PARALLET="${PARALLET}" FOCUS="${regex}" SKIP="${EXCLUDE}" ./hack/diff-image.sh "${domain}/${image}" "${to}" 2>&1 | tee -a "${logfile}" || {
+            echo "Error: diff image ${domain}/${image} $(helper::replace_domain "${domain}/${image}")"
+        }
+    done
+}
+
+function main() {
+
+    for image in $(helper::get_source); do
+        key="${image%%/*}"
+        val="${image#*/}"
+        if [[ -v "DOMAIN_MAP[${key}]" ]]; then
+            DOMAIN_MAP["${key}"]+=" ${val}"
+        else
+            DOMAIN_MAP["${key}"]="${val}"
+        fi
+    done
+
+    for domain in "${!DOMAIN_MAP[@]}"; do
+        if [[ "${PARALLET_JOBS}" -eq 0 ]]; then
+            sync_with_domain "${domain}"
+        else
+            wait_jobs "${PARALLET_JOBS}"
+            sync_with_domain "${domain}" &
+        fi
+    done
+
+    wait
+}
+
+main

hack/fmt.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+cat $1 | sort | tac > $1.tmp && mv $1.tmp $1

hack/helper.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+ROOT="${ROOT:-$(dirname "${BASH_SOURCE}")/..}"
+DEFAULT_REGEX='^([a-z]+-)?[a-z]*[0-9]+(\.[0-9]+){0,2}(-.+)?$|^[a-z]+$'
+SKOPEO="${SKOPEO:-skopeo}"
+ROOT=$(realpath ${ROOT})
+
+function helper::replace_domain() {
+    local domain="${1}"
+    local file="${2:-domain.txt}"
+    for line in $(cat ${file}); do
+        line="${line/ /}"
+        if [[ "${line}" == "" ]]; then
+            continue
+        fi
+
+        key="${line%=*}"
+        val="${line##*=}"
+        if [[ "${key}" == "" || "${val}" == "" ]]; then
+            continue
+        fi
+
+        if [[ "${domain}" =~ ^${key} ]]; then
+            echo "${domain}" | sed -e "s#^${key}#${val}#"
+            return 0
+        fi
+    done
+    echo "Error: domain ${domain} not found"
+    return 1
+}
+
+function helper::get_source() {
+    local source="${1:-mirror.txt}"
+    cat "${source}" | tr -d ' ' | grep -v -E '^$' | grep -v -E '^#'
+}
+
+function helper::exclude() {
+    local exclude="${1:-exclude.txt}"
+    local tmp=$(cat "${exclude}" | tr -d ' ' | grep -v -E '^$' | grep -v -E '^#' | tr '\n' '|')
+    echo "${tmp%|}"
+}

not_sync.yaml

@@ -1,31 +0,0 @@
-not_sync:
-- image_pattern: quay.io/calico/.*
-  tag_patterns:
-  - ".*[0-9a-f]{7,40}.*"
-- image_pattern: quay.io/tigera/.*
-  tag_patterns:
-  - ".*[0-9a-f]{7,40}.*"
-- image_pattern: quay.io/jetstack/.*
-  tag_patterns:
-  - "[0-9a-f]{8,40}"
-- image_pattern: docker.io/kubeovn/.*
-  tag_patterns:
-  - ".*[0-9a-f]{12,40}.*"
-- image_pattern: docker.io/weaveworks/.*
-  tag_patterns:
-  - ".*[0-9a-f]{12,40}.*"
-- image_pattern: docker.io/amazon/.*
-  tag_patterns:
-  - ".*[0-9a-f]{7,40}.*"
-- image_pattern: docker.io/.*python
-  tag_patterns:
-  - ".*windows.*"
-- image_pattern: docker.io/.*golang
-  tag_patterns:
-  - ".*windows.*"
-- image_pattern: docker.io/.*node
-  tag_patterns:
-  - ".*onbuild.*"
-- image_pattern: gcr.io/distroless/.*
-  tag_patterns:
-  - "sha256-.*"

scripts/check-image.sh

@@ -1,88 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-DEBUG="${DEBUG:-}"
-INCREMENTAL="${INCREMENTAL:-}"
-QUICKLY="${QUICKLY:-}"
-PARALLET="${PARALLET:-0}"
-
-declare -A DOMAIN_MAP=()
-
-for line in $(cat ./domain.txt); do
-    line="${line/ /}"
-    if [[ "$line" == "" ]]; then
-        continue
-    fi
-    key="${line%%=*}"
-    val="${line##*=}"
-    if [[ "${key}" == "" || "${val}" == "" ]]; then
-        echo "Error: invalid line: ${line}"
-        continue
-    fi
-
-    DOMAIN_MAP["${key}"]="${val}"
-done
-
-declare -A SKIP_MAP=()
-
-for line in $(cat not_sync.yaml | yq -j '.not_sync[] | .image_pattern , "=", (.tag_patterns[] | . , "|" ) , "\n"' | sed "s/|$//g"); do
-    line="${line/ /}"
-    if [[ "$line" == "" ]]; then
-        continue
-    fi
-    key="${line%%=*}"
-    val="${line##*=}"
-    if [[ "${key}" == "" || "${val}" == "" ]]; then
-        echo "Error: invalid line: ${line}"
-        continue
-    fi
-
-    SKIP_MAP["${key}"]="${val}"
-done
-
-LOGFILE="./check-image.log"
-echo >"${LOGFILE}"
-
-count=0
-images="$(cat ./mirror.txt | tr -d ' ' | grep -v -E '^$' | grep -v -E '^#' | shuf)"
-images_count=$(echo "${images}" | wc -l | tr -d ' ')
-for line in ${images}; do
-    line="${line/ /}"
-    if [[ "$line" == "" ]]; then
-        continue
-    fi
-
-    skip=""
-    for key in "${!SKIP_MAP[@]}"; do
-        if [[ "${line}" =~ ${key} ]]; then
-            skip+="${SKIP_MAP[$key]}|"
-        fi
-    done
-    skip="${skip%|}"
-
-    domain="${line%%/*}"
-    new_image=$(echo "${line}" | sed "s/^${domain}/${DOMAIN_MAP["${domain}"]}/g")
-    count=$((count + 1))
-    echo "Diff ${count}/${images_count} image ${line} ${new_image}"
-    DEBUG="${DEBUG}" QUICKLY="${QUICKLY}" INCREMENTAL="${INCREMENTAL}" PARALLET="${PARALLET}" SKIP="${skip}" ./scripts/diff-image.sh "${line}" "${new_image}" 2>&1 | tee -a "${LOGFILE}" || {
-        echo "Error: diff image ${line} ${new_image}"
-    }
-done
-
-cat "${LOGFILE}" | grep " UNSYNC: " | sort >sync.log
-cat "${LOGFILE}" | grep " SYNC: " | sort >>sync.log
-
-sync="$(cat "${LOGFILE}" | grep " SYNC: " | wc -l | tr -d ' ' || :)"
-unsync="$(cat "${LOGFILE}" | grep " UNSYNC: " | wc -l | tr -d ' ' || :)"
-sum=$(($sync + $unsync))
-
-if [[ "${QUICKLY}" == "true" ]]; then
-    echo "https://img.shields.io/badge/Sync-${sync}%2F${sum}-blue"
-    wget "https://img.shields.io/badge/Sync-${sync}%2F${sum}-blue" -O badge.svg
-else
-    echo "https://img.shields.io/badge/Deep%20Sync-${sync}%2F${sum}-blue"
-    wget "https://img.shields.io/badge/Deep%20Sync-${sync}%2F${sum}-blue" -O badge.svg
-fi

scripts/sync-to-jp.py

@@ -1,154 +0,0 @@
-import os
-import re
-import json
-import yaml
-import random
-import subprocess
-
-REPO_REPLACE_RULE1 = {
-    "k8s.gcr.io":"k8s-gcr-jp.m.daocloud.io"
-}
-
-REPO_REPLACE_RULE2 = {
-    "gcr.io":"gcr-jp.m.daocloud.io",
-    "docker.io":"docker-jp.m.daocloud.io",
-    "quay.io":"quay-jp.m.daocloud.io",
-    "ghcr.io":"ghcr-jp.m.daocloud.io"
-}
-#    "":"",
-
-INTERNAL_NETWORK = {
-    "k8s-gcr-jp.m.daocloud.io":"10.40.134.38:6001",
-    "gcr-jp.m.daocloud.io":"10.40.134.38:5001",
-    "docker-jp.m.daocloud.io":"10.40.134.38:7001",
-    "quay-jp.m.daocloud.io":"10.40.134.38:8001",
-    "ghcr-jp.m.daocloud.io":"10.40.134.38:9001"
-}
-
-
-REGISTRY_PASSWORD = os.environ["REGISTRY_PASSWORD"]
-DELTA_MODE = os.environ.get("DELTA_MODE") 
-
-DRY_RUN = os.environ.get("DRY_RUN")
-DOCKER_IO_USER = os.environ.get("DOCKER_IO_USER")
-DOCKER_IO_PASS = os.environ.get("DOCKER_IO_PASS")
-
-SKOPEO_CMD = "docker run --rm quay.io/containers/skopeo:latest"
-# SKOPEO_CMD = "skopeo" # RUN without docker
-
-def __run_cmd(cmd):
-    print(cmd)
-    if not DRY_RUN:
-        os.system(cmd)
-
-def __dest_img(src_img):
-    dest_img = src_img
-    for k,v in REPO_REPLACE_RULE1.items():
-        dest_img = dest_img.replace(k,v)
-    for k,v in REPO_REPLACE_RULE2.items():
-        dest_img = dest_img.replace(k,v)
-    for k,v in INTERNAL_NETWORK.items():
-        dest_img = dest_img.replace(k,v)
-    return dest_img
-
-
-def is_filtered(src_img):
-    y = []
-    with open("not_sync.yaml") as f:
-        y = yaml.safe_load(f)
-    for n in y.get('not_sync'):
-        if bool(re.match(n.get("image_pattern"), src_img)):
-            return True
-    return False
-
-def __parse_tag(output):
-    # print(output)
-    try:
-        return json.loads(output).get('Tags',[])
-    except:
-        return []
-
-def skepo_sync_one_tag(src_img,tag):
-    dest_img = __dest_img(src_img)
-    src_auth =''
-    if 'docker.io' in src_img and DOCKER_IO_USER:
-        src_auth = " --src-creds %s:%s " % (DOCKER_IO_USER,DOCKER_IO_PASS)
-    cmd = SKOPEO_CMD + " copy --all %s --dest-creds root:%s  --dest-tls-verify=false -f oci docker://%s:%s docker://%s:%s" %(src_auth,REGISTRY_PASSWORD,src_img,tag,dest_img,tag)
-    __run_cmd(cmd)
-
-def filter_tag(src_img,delta_tags):
-    y = []
-    need_to_sync = list(delta_tags)
-    with open("not_sync.yaml") as f:
-        y = yaml.safe_load(f)
-    for n in y.get('not_sync'):
-        if bool(re.match(n.get("image_pattern"), src_img)):
-            for t in delta_tags:
-                for tp in n.get("tag_patterns",[]):
-                    if bool(re.match(tp, t)):
-                        need_to_sync.remove(t)
-    return need_to_sync
-
-def skepo_delta_sync(src_img,full_mode=False):
-    dest_img = __dest_img(src_img)
-    src_auth =''
-    if 'docker.io' in src_img and DOCKER_IO_USER:
-        src_auth = " --creds %s:%s " % (DOCKER_IO_USER,DOCKER_IO_PASS)
-    cmd = "skopeo list-tags %s docker://%s" % (src_auth,src_img)
-    print(cmd)
-    result = subprocess.run(cmd, shell=True,stdout=subprocess.PIPE)
-    src_tags = __parse_tag(result.stdout)
-    cmd = "skopeo list-tags  --creds root:%s  --tls-verify=false docker://%s" %(REGISTRY_PASSWORD,dest_img)
-    print(cmd)
-    result = subprocess.run(cmd, shell=True,stdout=subprocess.PIPE)
-    dest_tags = __parse_tag(result.stdout)
-    delta_tags = set(src_tags) - set(dest_tags)
-
-
-    need_to_sync =  filter_tag(src_img,delta_tags)
-    filtered_num = len(delta_tags) - len(need_to_sync)
-    if full_mode:
-        need_to_sync = filter_tag(src_img,src_tags)
-        filtered_num = len(src_tags) - len(need_to_sync)
-    
-    if 'latest' in src_tags and 'latest' not in need_to_sync:
-        need_to_sync.append('latest')
-
-    # print(src_img)
-    print("sync %s, src tag %s, dest tag %s, sync tag %s, filtered tag %s " % (src_img,len(src_tags),len(dest_tags),len(need_to_sync),filtered_num))
-    
-    for tag in need_to_sync:
-        skepo_sync_one_tag(src_img,tag)
-
-def skepo_full_sync(src_img):
-    if is_filtered(src_img):
-        skepo_delta_sync(src_img,full_mode=True)
-    else:
-        dest_img = __dest_img(src_img)
-        dest_img = "/".join(dest_img.split("/")[:-1])
-        src_auth = ""
-        if 'docker.io' in src_img and DOCKER_IO_USER:
-            src_auth = " --src-creds %s:%s " % (DOCKER_IO_USER,DOCKER_IO_PASS)
-        cmd = SKOPEO_CMD + " sync --all --src docker --dest docker %s --dest-tls-verify=false --dest-creds root:%s -f oci %s %s" %(src_auth,REGISTRY_PASSWORD,src_img,dest_img)
-        __run_cmd(cmd)
-
-
-def main():
-    lines = []
-    with open("mirror.txt") as f:
-        lines = f.readlines()
-    random.shuffle(lines)
-    sync_cmds = []
-    for l in lines:
-        src_img = l.strip()
-        if DELTA_MODE:
-            print('use DELTA_MODE')
-            skepo_delta_sync(src_img)
-        else:
-            print('use FULL_SYNC_MODE')
-            skepo_full_sync(src_img)
-        
-
-if __name__ == "__main__":
-    # execute only if run as a script
-    main()