diff --git a/README.md b/README.md index a59aeeb..12cf839 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,11 @@ TODO ## 安装运行 +安装skepo. +``` +yum install skopeo --nobest +``` + 运行同步程序 diff --git a/scripts/sync-to-jp.py b/scripts/sync-to-jp.py index fca6e76..65d8282 100644 --- a/scripts/sync-to-jp.py +++ b/scripts/sync-to-jp.py @@ -1,5 +1,7 @@ import os +import json import random +import subprocess REPO_REPLACE_RULE1 = { "k8s.gcr.io":"k8s-gcr-jp.m.daocloud.io" @@ -23,40 +25,77 @@ INTERNAL_NETWORK = { REGISTRY_PASSWORD = os.environ["REGISTRY_PASSWORD"] +DELTA_MODE = os.environ.get("DELTA_MODE") +DRY_RUN = os.environ.get("DRY_RUN") +SKOPEO_CMD = "docker run --rm quay.io/containers/skopeo:latest" +# SKOPEO_CMD = "skopeo" # RUN without docker -SKEPO_CMD = "docker run --rm quay.io/containers/skopeo:latest" -# SKEPO_CMD = "skepo" # RUN without docker +def __run_cmd(cmd): + print(cmd) + if not DRY_RUN: + os.system(cmd) -def skepo_sync_cmd(src_img): - src_img = src_img.strip() - - dest_img = "/".join(src_img.split("/")[:-1]) +def __dest_img(src_img): + dest_img = src_img for k,v in REPO_REPLACE_RULE1.items(): dest_img = dest_img.replace(k,v) for k,v in REPO_REPLACE_RULE2.items(): dest_img = dest_img.replace(k,v) for k,v in INTERNAL_NETWORK.items(): dest_img = dest_img.replace(k,v) + return dest_img - cmd = SKEPO_CMD + " sync --src docker --dest docker --dest-tls-verify=false --dest-creds root:%s -f oci %s %s" %(REGISTRY_PASSWORD,src_img,dest_img) -# print(src_img) -# print(dest_img) -# print(cmd) - return cmd +def skepo_full_sync(src_img): + dest_img = __dest_img(src_img) + dest_img = "/".join(dest_img.split("/")[:-1]) + cmd = SKOPEO_CMD + " sync --src docker --dest docker --dest-tls-verify=false --dest-creds root:%s -f oci %s %s" %(REGISTRY_PASSWORD,src_img,dest_img) + __run_cmd(cmd) + +def __parse_tag(output): + # print(output) + try: + return json.loads(output).get('Tags',[]) + except: + return [] + +def skepo_delta_sync(src_img): + dest_img = __dest_img(src_img) + cmd = "skopeo list-tags docker://" + src_img + result = subprocess.run(cmd, shell=True,stdout=subprocess.PIPE) + src_tags = __parse_tag(result.stdout) + cmd = "skopeo list-tags --creds root:%s --tls-verify=false docker://%s" %(REGISTRY_PASSWORD,dest_img) + # print(cmd) + result = subprocess.run(cmd, shell=True,stdout=subprocess.PIPE) + dest_tags = __parse_tag(result.stdout) + need_to_sync = set(src_tags) - set(dest_tags) + + # print(src_img) + print("sync %s, src tag %s, dest tag %s " % (src_img,len(src_tags),len(dest_tags))) + # print(need_to_sync) + if len(need_to_sync) == 0: + if 'latest' in src_tags: + cmd = SKOPEO_CMD + " copy --dest-creds root:%s --dest-tls-verify=false -f oci docker://%s:latest docker://%s:latest" %(REGISTRY_PASSWORD,src_img,dest_img) + __run_cmd(cmd) + else: + skepo_full_sync(src_img) + + # print(src_tags) def main(): lines = [] with open("mirror.txt") as f: lines = f.readlines() + random.shuffle(lines) sync_cmds = [] for l in lines: - sync_cmds.append(skepo_sync_cmd(l)) - random.shuffle(sync_cmds) - for c in sync_cmds: - print(c) - os.system(c) + src_img = l.strip() + if DELTA_MODE: + skepo_delta_sync(src_img) + else: + skepo_full_sync(src_img) + if __name__ == "__main__": # execute only if run as a script