From 45d0fadbb2394030dc395471c07226c35ed92d99 Mon Sep 17 00:00:00 2001 From: yankay Date: Sun, 12 Sep 2021 14:54:42 +0800 Subject: [PATCH 1/2] delta mode --- README.md | 5 ++++ scripts/sync-to-jp.py | 69 +++++++++++++++++++++++++++++++++---------- 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index dc18ee1..37bbf24 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,11 @@ TODO ## 安装运行 +安装skepo +``` +yum install skopeo --nobest +``` + 运行同步程序 diff --git a/scripts/sync-to-jp.py b/scripts/sync-to-jp.py index fca6e76..561e38b 100644 --- a/scripts/sync-to-jp.py +++ b/scripts/sync-to-jp.py @@ -1,5 +1,7 @@ import os +import json import random +import subprocess REPO_REPLACE_RULE1 = { "k8s.gcr.io":"k8s-gcr-jp.m.daocloud.io" @@ -23,40 +25,77 @@ INTERNAL_NETWORK = { REGISTRY_PASSWORD = os.environ["REGISTRY_PASSWORD"] +DELTA_MODE = os.environ.get("DELTA_MODE") +DRY_RUN = os.environ.get("DRY_RUN") +# SKEPO_CMD = "docker run --rm quay.io/containers/skopeo:latest" +SKEPO_CMD = "skopeo" # RUN without docker -SKEPO_CMD = "docker run --rm quay.io/containers/skopeo:latest" -# SKEPO_CMD = "skepo" # RUN without docker +def __run_cmd(cmd): + print(cmd) + if not DRY_RUN: + os.system(cmd) -def skepo_sync_cmd(src_img): - src_img = src_img.strip() - - dest_img = "/".join(src_img.split("/")[:-1]) +def __dest_img(src_img): + dest_img = src_img for k,v in REPO_REPLACE_RULE1.items(): dest_img = dest_img.replace(k,v) for k,v in REPO_REPLACE_RULE2.items(): dest_img = dest_img.replace(k,v) for k,v in INTERNAL_NETWORK.items(): dest_img = dest_img.replace(k,v) + return dest_img +def skepo_full_sync(src_img): + dest_img = __dest_img(src_img) + dest_img = "/".join(dest_img.split("/")[:-1]) cmd = SKEPO_CMD + " sync --src docker --dest docker --dest-tls-verify=false --dest-creds root:%s -f oci %s %s" %(REGISTRY_PASSWORD,src_img,dest_img) -# print(src_img) -# print(dest_img) -# print(cmd) - return cmd + __run_cmd(cmd) + +def __parse_tag(output): + # print(output) + try: + return json.loads(output).get('Tags',[]) + except: + return [] + +def skepo_delta_sync(src_img): + dest_img = __dest_img(src_img) + cmd = "skopeo list-tags docker://" + src_img + result = subprocess.run(cmd, shell=True,stdout=subprocess.PIPE) + src_tags = __parse_tag(result.stdout) + cmd = "skopeo list-tags --creds root:%s --tls-verify=false docker://%s" %(REGISTRY_PASSWORD,dest_img) + print(cmd) + result = subprocess.run(cmd, shell=True,stdout=subprocess.PIPE) + dest_tags = __parse_tag(result.stdout) + need_to_sync = set(src_tags) - set(dest_tags) + + print(src_img) + print("src %s, dest %s " % (len(src_tags),len(dest_tags))) + print(need_to_sync) + if len(need_to_sync) == 0: + if 'latest' in src_tags: + cmd = "skopeo copy --dest-creds root:%s --dest-tls-verify=false -f oci docker://%s:latest docker://%s:latest" %(REGISTRY_PASSWORD,src_img,dest_img) + __run_cmd(cmd) + else: + skepo_full_sync(src_img) + + # print(src_tags) def main(): lines = [] with open("mirror.txt") as f: lines = f.readlines() + random.shuffle(lines) sync_cmds = [] for l in lines: - sync_cmds.append(skepo_sync_cmd(l)) - random.shuffle(sync_cmds) - for c in sync_cmds: - print(c) - os.system(c) + src_img = l.strip() + if DELTA_MODE: + skepo_delta_sync(src_img) + else: + skepo_full_sync(src_img) + if __name__ == "__main__": # execute only if run as a script From 1d5e2f45cf916dd86b454cb10045c84803eca732 Mon Sep 17 00:00:00 2001 From: yankay Date: Sun, 12 Sep 2021 15:18:59 +0800 Subject: [PATCH 2/2] delta mode --- README.md | 2 +- scripts/sync-to-jp.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 37bbf24..1f5f079 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ TODO ## 安装运行 -安装skepo +安装skepo. ``` yum install skopeo --nobest ``` diff --git a/scripts/sync-to-jp.py b/scripts/sync-to-jp.py index 561e38b..65d8282 100644 --- a/scripts/sync-to-jp.py +++ b/scripts/sync-to-jp.py @@ -28,8 +28,8 @@ REGISTRY_PASSWORD = os.environ["REGISTRY_PASSWORD"] DELTA_MODE = os.environ.get("DELTA_MODE") DRY_RUN = os.environ.get("DRY_RUN") -# SKEPO_CMD = "docker run --rm quay.io/containers/skopeo:latest" -SKEPO_CMD = "skopeo" # RUN without docker +SKOPEO_CMD = "docker run --rm quay.io/containers/skopeo:latest" +# SKOPEO_CMD = "skopeo" # RUN without docker def __run_cmd(cmd): print(cmd) @@ -50,7 +50,7 @@ def __dest_img(src_img): def skepo_full_sync(src_img): dest_img = __dest_img(src_img) dest_img = "/".join(dest_img.split("/")[:-1]) - cmd = SKEPO_CMD + " sync --src docker --dest docker --dest-tls-verify=false --dest-creds root:%s -f oci %s %s" %(REGISTRY_PASSWORD,src_img,dest_img) + cmd = SKOPEO_CMD + " sync --src docker --dest docker --dest-tls-verify=false --dest-creds root:%s -f oci %s %s" %(REGISTRY_PASSWORD,src_img,dest_img) __run_cmd(cmd) def __parse_tag(output): @@ -66,17 +66,17 @@ def skepo_delta_sync(src_img): result = subprocess.run(cmd, shell=True,stdout=subprocess.PIPE) src_tags = __parse_tag(result.stdout) cmd = "skopeo list-tags --creds root:%s --tls-verify=false docker://%s" %(REGISTRY_PASSWORD,dest_img) - print(cmd) + # print(cmd) result = subprocess.run(cmd, shell=True,stdout=subprocess.PIPE) dest_tags = __parse_tag(result.stdout) need_to_sync = set(src_tags) - set(dest_tags) - print(src_img) - print("src %s, dest %s " % (len(src_tags),len(dest_tags))) - print(need_to_sync) + # print(src_img) + print("sync %s, src tag %s, dest tag %s " % (src_img,len(src_tags),len(dest_tags))) + # print(need_to_sync) if len(need_to_sync) == 0: if 'latest' in src_tags: - cmd = "skopeo copy --dest-creds root:%s --dest-tls-verify=false -f oci docker://%s:latest docker://%s:latest" %(REGISTRY_PASSWORD,src_img,dest_img) + cmd = SKOPEO_CMD + " copy --dest-creds root:%s --dest-tls-verify=false -f oci docker://%s:latest docker://%s:latest" %(REGISTRY_PASSWORD,src_img,dest_img) __run_cmd(cmd) else: skepo_full_sync(src_img)