init repo

.travis.yml

@@ -0,0 +1,9 @@
+services:
+  - docker
+before_install:
+  - docker login -u${user_name} -p${pass_word}
+  - chmod +x ./deploy.sh
+  - ./deploy.sh
+
+script: 
+  - echo hello

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 AnJia
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.tpl

@@ -0,0 +1,58 @@
+Google Container Registry Mirror [last sync ${current_date} UTC]
+-------
+
+[![Sync Status](https://travis-ci.org/anjia0532/gcr.io_mirror.svg?branch=sync)](https://travis-ci.org/anjia0532/gcr.io_mirror)
+
+Syntax
+-------
+
+```bash
+gcr.io/namespace/image_name:image_tag 
+#eq
+${user_name}/namespace.image_name:image_tag
+```
+
+Example
+-------
+
+```bash
+docker pull gcr.io/google-containers/federation-controller-manager-arm64:v1.3.1-beta.1 
+# eq 
+docker pull ${user_name}/google-containers.federation-controller-manager-arm64:v1.3.1-beta.1
+```
+
+ReTag ${user_name} images to gcr.io 
+-------
+
+```bash
+# replace gcr.io/google-containers/federation-controller-manager-arm64:v1.3.1-beta.1 to real image
+# this will convert gcr.io/google-containers/federation-controller-manager-arm64:v1.3.1-beta.1 
+# to ${user_name}/google-containers.federation-controller-manager-arm64:v1.3.1-beta.1 and pull it
+
+images=$(cat img.txt)
+#or 
+#images=$(cat <<EOF
+# gcr.io/google-containers/federation-controller-manager-arm64:v1.3.1-beta.1
+# gcr.io/google-containers/federation-controller-manager-arm64:v1.3.1-beta.1
+# gcr.io/google-containers/federation-controller-manager-arm64:v1.3.1-beta.1
+#EOF)
+
+eval $(echo ${images}|
+        sed 's/gcr\.io/${user_name}/g;s/\//\./g;s/ /\n/g;s/${user_name}\./${user_name}\//g' |
+        uniq |
+        awk '{print "docker pull "$1";"}'
+       )
+
+# this code will retag all of ${user_name}'s image from local  e.g. ${user_name}/google-containers.federation-controller-manager-arm64:v1.3.1-beta.1 
+# to gcr.io/google-containers/federation-controller-manager-arm64:v1.3.1-beta.1
+
+for img in $(docker images --format "{{.Repository}}:{{.Tag}}"| grep "${user_name}"); do
+  n=$(echo ${img}| awk -F'[/.:]' '{printf "gcr.io/%s/%s",$2,$3}')
+  tag=$(echo ${img}| awk -F'[:]' '{printf ":%s",$2}')
+  docker tag $img "${n}${tag}"
+done
+```
+
+[Changelog](./CHANGES.md)
+-------
+

deploy.sh

@@ -0,0 +1,202 @@
+#!/usr/bin/env bash
+
+SECONDS=0
+
+source ./process-utils.sh
+process_init 30
+
+red='\033[0;31m'
+green='\033[0;32m'
+yellow='\033[0;33m'
+plain='\033[0m'
+
+[[ -d "gcr.io_mirror" ]] && rm -rf ./gcr.io_mirror
+
+git clone "https://github.com/${user_name}/gcr.io_mirror.git"
+
+function init_namespace()
+{
+  n=$1
+  echo -e "${yellow}init gcr.io/$n's image...${plain}"
+  # get all of the gcr images
+  imgs=$(curl -ks 'https://console.cloud.google.com/m/gcr/entities/list' \
+           -H "Cookie: ${cookie}" \
+           -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.7 Safari/537.36' \
+           -H 'Content-Type: application/json;charset=UTF-8' \
+           -H 'Accept: application/json, text/plain, */*' \
+           --data-binary '["'${n}'",null,null,[]]' |
+           grep -P '"' |
+           sed 's/"gcr.ListEntities"//' |
+           cut -d '"' -f2 |
+           sort |
+           uniq)
+
+  for img in ${imgs[@]}  ; do
+   process_run "init_imgs $img"
+  done
+  wait ${!}
+}
+
+function init_imgs()
+{
+  img=$1
+  echo -e "${yellow}init gcr.io/$n/${img}'s image...${plain}"
+  # get all  tags for this image
+  gcr_content=$(curl -ks -X GET https://gcr.io/v2/${n}/${img}/tags/list)
+  dir=gcr.io_mirror/${n}/${img}/
+
+  # if this image dir not exits
+  [[ ! -d ${dir} ]] && mkdir -p ${dir};
+
+  # create img tmp file,named by tag's name, set access's time,modify's time by this image manifest's timeUploadedMs
+  echo ${gcr_content} | jq -r '.manifest|to_entries[]|select(.value.tag|length>0)|{k: .key,t: .value.tag[0],v: .value.timeUploadedMs} | "tf=${dir}"+.t+".tmp;echo "+.k+">${tf};touch -amd \"$(date \"+%F %T\" -d @" + .v[0:10] +")\" ${tf}"' | while read i; do
+    eval $i
+  done
+}
+
+function compare()
+{
+
+  echo -e "${yellow}compare image diff ...${plain}"
+  find ./gcr.io_mirror/ -name "*.tmp" | while read t
+  do
+    dir=$(dirname $t)
+    name=$(basename $t .tmp)
+    if [ -e ${dir}/${name}.tag ] && [ $(cat ${dir}/${name}.tag)x = $(cat $t)x ]; then
+      rm -rf $t;
+    else
+      [[ -e ${dir}/${name}.tag ]] && rm -rf ${dir}/${name}.tag
+    fi
+  done
+}
+
+function pull_push_diff()
+{
+  n=$1
+  img=$2
+  all_of_imgs=$(find ./gcr.io_mirror -type f \( ! -iname "*.md" \) |wc -l)
+  current_ns_imgs=$(find ./gcr.io_mirror/${n}/ -type f \( ! -iname "*.md" \) |wc -l)
+  tmps=($(find ./gcr.io_mirror/${n}/${img}/ -type f \( -iname "*.tmp" \) -exec basename {} .tmp \; | uniq))
+  
+  echo -e "${red}wait for mirror${plain}/${yellow}gcr.io/${n}/* images${plain}/${green}all of images${plain}:${red}${#tmps[@]}${plain}/${yellow}${current_ns_imgs}${plain}/${green}${all_of_imgs}${plain}"
+  
+  for tag in ${tmps[@]} ; do
+    echo -e "${yellow}mirror ${n}/${img}/${tag}...${plain}"
+    lock=./gcr.io_mirror/${n}/${img}/${tag}.lck
+    [[ -e $lock ]] && continue;
+    echo "${tag}">$lock
+    
+    docker pull gcr.io/${n}/${img}:${tag}
+    docker tag gcr.io/${n}/${img}:${tag} ${user_name}/${n}.${img}:${tag}
+    docker push ${user_name}/${n}.${img}:${tag}
+    
+    [[ -e ./commit.lck ]] && break
+    
+    mv ./gcr.io_mirror/${n}/${img}/${tag}.tmp ./gcr.io_mirror/${n}/${img}/${tag}.tag
+    
+    echo -e "[gcr.io/${n}/${image}:${tag}](https://hub.docker.com/r/${user_name}/${n}.${image}/tags/)\n\n" >> CHANGES.md
+    rm -rf $lock
+  done
+}
+
+function mirror()
+{
+  num=$(find ./gcr.io_mirror/ -type f \( -iname "*.tmp" \) |wc -l)
+  if [ $num -eq 0 ]; then
+    ns=$(cat ./gcr_namespaces 2>/dev/null || echo google-containers)
+    for n in ${ns[@]}  ; do
+      process_run "init_namespace $n"
+    done
+    wait ${!}
+  fi
+  
+  compare
+  
+  tmps=$(find ./gcr.io_mirror/ -type f \( -iname "*.tmp" \) -exec dirname {} \; | uniq | cut -d'/' -f3-4)
+  
+  if [ -z "$tmps" ]; then
+    echo -e "${red} wait for push ${tmps[@]}"
+    for img in ${tmps[@]} ; do
+      n=$(echo ${img}|cut -d'/' -f1)
+      image=$(echo ${img}|cut -d'/' -f2)
+      process_run "pull_push_diff $n $image"
+    done
+    wait ${!}
+  fi
+  
+  images=($(find ./gcr.io_mirror/ -type f -name "*.tag" |uniq|sort))
+  
+  cp ./gcr.io_mirror/CHANGES.md ./CHANGES1.md 2>/dev/null
+  
+  find ./gcr.io_mirror/ -type f -name "*.md" -exec rm -rf {} \;
+    
+  for img in ${images[@]} ; do
+    n=$(echo ${img}|cut -d'/' -f3)
+    image=$(echo ${img}|cut -d'/' -f4)
+    tag=$(basename ${img} .tag)
+    mkdir -p ./gcr.io_mirror/${n}/${image}
+    if [ ! -e ./gcr.io_mirror/${n}/${image}/README.md ]; then
+      echo -e "\n[gcr.io/${n}/${image}](https://hub.docker.com/r/${user_name}/${n}.${image}/tags/)\n-----\n\n" >> ./gcr.io_mirror/${n}/${image}/README.md
+      echo -e "\n[gcr.io/${n}/${image}](https://hub.docker.com/r/${user_name}/${n}.${image}/tags/)\n-----\n\n" >> ./gcr.io_mirror/${n}/README.md
+    fi
+    
+    echo -e "[gcr.io/${n}/${image}:${tag}](https://hub.docker.com/r/${user_name}/${n}.${image}/tags/)\n\n" >> ./gcr.io_mirror/${n}/${image}/README.md
+  done
+  
+  commit
+}
+
+function commit()
+{
+  echo 1 > ./commit.lck
+  ns=($(cat ./gcr_namespaces 2>/dev/null || echo google-containers))
+  readme=./gcr.io_mirror/README.md
+  export current_date=$(date +'%Y-%m-%d %H:%M')
+  envsubst < README.tpl >"${readme}"
+  
+  echo -e "Mirror ${#ns[@]} namespaces image from gcr.io\n-----\n\n" >> "${readme}"
+  for n in ${ns[@]} ; do
+    echo -e "[gcr.io/${n}/*](./${n}/README.md)\n\n" >> "${readme}"
+  done
+  
+  if [ -s CHANGES.md ]; then
+    (echo -e "## $(date +'%Y-%m-%d %H:%M') \n" && cat CHANGES.md && cat CHANGES1.md 2>/dev/null) >> gcr.io_mirror/CHANGES.md
+  fi
+  
+  echo -e "${red} commit to github master"
+  git -C ./gcr.io_mirror pull
+  git -C ./gcr.io_mirror add -A
+  git -C ./gcr.io_mirror commit -m "sync gcr.io's images at $(date +'%Y-%m-%d %H:%M')"
+  git -C ./gcr.io_mirror push -f "https://${user_name}:${GH_TOKEN}@github.com/${user_name}/gcr.io_mirror.git" master:master
+  
+  echo -e "${red} commit to github master:done"
+  echo 1 > ./commit.done
+}
+
+mirror &
+
+while true;
+do
+  duration=$SECONDS
+  if [[ -e ./commit.done ]] || [[ $duration -ge $sec ]]; then
+    
+    [[ $duration -ge $sec ]] && echo -e "${red} more than $(expr $sec / 60) min,abort this build"
+    
+    [[ ! -e ./commit.done ]] && commit
+    
+    IFS=$'\n'; for i in $(jobs); do echo "$i"; done
+    kill $(jobs -p)
+    IFS=$'\n'; for i in $(jobs); do echo "$i"; done
+    break
+    
+  else
+    docker_dir=$(docker info | grep "Docker Root Dir" | cut -d':' -f2)
+    used=$(df -h ${docker_dir}|awk '{if(NR>1)print $5}')
+    echo -e "${red} duration:${duration}s, docker root dir :${docker_dir}:used:${used}"
+    [[ ${used} > '70%' ]] && docker system prune -f -a
+    sleep 30
+  fi
+done
+
+sleep 120
+echo "${red} bye bye"

gcr_namespaces

@@ -0,0 +1,6 @@
+google-samples
+kubernetes-helm
+k8s-minikube
+tf-on-k8s-dogfood
+spinnaker-marketplace
+google-containers

process-utils.sh

@@ -0,0 +1,36 @@
+_PROCESS_PIPE_NAME="/tmp/$(cat /proc/sys/kernel/random/uuid)"
+_PROCESS_PIPE_ID=10
+
+function _create_pipe()
+{
+
+  mkfifo ${_PROCESS_PIPE_NAME}
+  eval exec "${_PROCESS_PIPE_ID}""<>${_PROCESS_PIPE_NAME}"
+
+  for ((i=0; i< $1; i++))
+  do
+    echo >&${_PROCESS_PIPE_ID}
+  done
+}
+
+function process_init()
+{
+  _create_pipe $1
+}
+
+function process_run()
+{
+  cmd=$1
+  
+  if [ -z "$cmd" ]; then
+    echo "please input command to run"
+    exit 1
+  fi
+  
+  read -u${_PROCESS_PIPE_ID}
+  {
+    $cmd
+    echo >&${_PROCESS_PIPE_ID}
+  }&
+}
+