init repo
This commit is contained in:
anjia0532
commit
5c54de249f
|
|
@ -0,0 +1,9 @@
|
|||
services:
|
||||
- docker
|
||||
before_install:
|
||||
- docker login -u${user_name} -p${pass_word}
|
||||
- chmod +x ./deploy.sh
|
||||
- ./deploy.sh
|
||||
|
||||
script:
|
||||
- echo hello
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2017 AnJia
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
Google Container Registry Mirror [last sync ${current_date} UTC]
|
||||
-------
|
||||
|
||||
[](https://travis-ci.org/anjia0532/gcr.io_mirror)
|
||||
|
||||
Syntax
|
||||
-------
|
||||
|
||||
```bash
|
||||
gcr.io/namespace/image_name:image_tag
|
||||
#eq
|
||||
${user_name}/namespace.image_name:image_tag
|
||||
```
|
||||
|
||||
Example
|
||||
-------
|
||||
|
||||
```bash
|
||||
docker pull gcr.io/google-containers/federation-controller-manager-arm64:v1.3.1-beta.1
|
||||
# eq
|
||||
docker pull ${user_name}/google-containers.federation-controller-manager-arm64:v1.3.1-beta.1
|
||||
```
|
||||
|
||||
ReTag ${user_name} images to gcr.io
|
||||
-------
|
||||
|
||||
```bash
|
||||
# replace gcr.io/google-containers/federation-controller-manager-arm64:v1.3.1-beta.1 to real image
|
||||
# this will convert gcr.io/google-containers/federation-controller-manager-arm64:v1.3.1-beta.1
|
||||
# to ${user_name}/google-containers.federation-controller-manager-arm64:v1.3.1-beta.1 and pull it
|
||||
|
||||
images=$(cat img.txt)
|
||||
#or
|
||||
#images=$(cat <<EOF
|
||||
# gcr.io/google-containers/federation-controller-manager-arm64:v1.3.1-beta.1
|
||||
# gcr.io/google-containers/federation-controller-manager-arm64:v1.3.1-beta.1
|
||||
# gcr.io/google-containers/federation-controller-manager-arm64:v1.3.1-beta.1
|
||||
#EOF)
|
||||
|
||||
eval $(echo ${images}|
|
||||
sed 's/gcr\.io/${user_name}/g;s/\//\./g;s/ /\n/g;s/${user_name}\./${user_name}\//g' |
|
||||
uniq |
|
||||
awk '{print "docker pull "$1";"}'
|
||||
)
|
||||
|
||||
# this code will retag all of ${user_name}'s image from local e.g. ${user_name}/google-containers.federation-controller-manager-arm64:v1.3.1-beta.1
|
||||
# to gcr.io/google-containers/federation-controller-manager-arm64:v1.3.1-beta.1
|
||||
|
||||
for img in $(docker images --format "{{.Repository}}:{{.Tag}}"| grep "${user_name}"); do
|
||||
n=$(echo ${img}| awk -F'[/.:]' '{printf "gcr.io/%s/%s",$2,$3}')
|
||||
tag=$(echo ${img}| awk -F'[:]' '{printf ":%s",$2}')
|
||||
docker tag $img "${n}${tag}"
|
||||
done
|
||||
```
|
||||
|
||||
[Changelog](./CHANGES.md)
|
||||
-------
|
||||
|
||||
|
|
@ -0,0 +1,202 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
SECONDS=0
|
||||
|
||||
source ./process-utils.sh
|
||||
process_init 30
|
||||
|
||||
red='\033[0;31m'
|
||||
green='\033[0;32m'
|
||||
yellow='\033[0;33m'
|
||||
plain='\033[0m'
|
||||
|
||||
[[ -d "gcr.io_mirror" ]] && rm -rf ./gcr.io_mirror
|
||||
|
||||
git clone "https://github.com/${user_name}/gcr.io_mirror.git"
|
||||
|
||||
function init_namespace()
|
||||
{
|
||||
n=$1
|
||||
echo -e "${yellow}init gcr.io/$n's image...${plain}"
|
||||
# get all of the gcr images
|
||||
imgs=$(curl -ks 'https://console.cloud.google.com/m/gcr/entities/list' \
|
||||
-H "Cookie: ${cookie}" \
|
||||
-H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.7 Safari/537.36' \
|
||||
-H 'Content-Type: application/json;charset=UTF-8' \
|
||||
-H 'Accept: application/json, text/plain, */*' \
|
||||
--data-binary '["'${n}'",null,null,[]]' |
|
||||
grep -P '"' |
|
||||
sed 's/"gcr.ListEntities"//' |
|
||||
cut -d '"' -f2 |
|
||||
sort |
|
||||
uniq)
|
||||
|
||||
for img in ${imgs[@]} ; do
|
||||
process_run "init_imgs $img"
|
||||
done
|
||||
wait ${!}
|
||||
}
|
||||
|
||||
function init_imgs()
|
||||
{
|
||||
img=$1
|
||||
echo -e "${yellow}init gcr.io/$n/${img}'s image...${plain}"
|
||||
# get all tags for this image
|
||||
gcr_content=$(curl -ks -X GET https://gcr.io/v2/${n}/${img}/tags/list)
|
||||
dir=gcr.io_mirror/${n}/${img}/
|
||||
|
||||
# if this image dir not exits
|
||||
[[ ! -d ${dir} ]] && mkdir -p ${dir};
|
||||
|
||||
# create img tmp file,named by tag's name, set access's time,modify's time by this image manifest's timeUploadedMs
|
||||
echo ${gcr_content} | jq -r '.manifest|to_entries[]|select(.value.tag|length>0)|{k: .key,t: .value.tag[0],v: .value.timeUploadedMs} | "tf=${dir}"+.t+".tmp;echo "+.k+">${tf};touch -amd \"$(date \"+%F %T\" -d @" + .v[0:10] +")\" ${tf}"' | while read i; do
|
||||
eval $i
|
||||
done
|
||||
}
|
||||
|
||||
function compare()
|
||||
{
|
||||
|
||||
echo -e "${yellow}compare image diff ...${plain}"
|
||||
find ./gcr.io_mirror/ -name "*.tmp" | while read t
|
||||
do
|
||||
dir=$(dirname $t)
|
||||
name=$(basename $t .tmp)
|
||||
if [ -e ${dir}/${name}.tag ] && [ $(cat ${dir}/${name}.tag)x = $(cat $t)x ]; then
|
||||
rm -rf $t;
|
||||
else
|
||||
[[ -e ${dir}/${name}.tag ]] && rm -rf ${dir}/${name}.tag
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function pull_push_diff()
|
||||
{
|
||||
n=$1
|
||||
img=$2
|
||||
all_of_imgs=$(find ./gcr.io_mirror -type f \( ! -iname "*.md" \) |wc -l)
|
||||
current_ns_imgs=$(find ./gcr.io_mirror/${n}/ -type f \( ! -iname "*.md" \) |wc -l)
|
||||
tmps=($(find ./gcr.io_mirror/${n}/${img}/ -type f \( -iname "*.tmp" \) -exec basename {} .tmp \; | uniq))
|
||||
|
||||
echo -e "${red}wait for mirror${plain}/${yellow}gcr.io/${n}/* images${plain}/${green}all of images${plain}:${red}${#tmps[@]}${plain}/${yellow}${current_ns_imgs}${plain}/${green}${all_of_imgs}${plain}"
|
||||
|
||||
for tag in ${tmps[@]} ; do
|
||||
echo -e "${yellow}mirror ${n}/${img}/${tag}...${plain}"
|
||||
lock=./gcr.io_mirror/${n}/${img}/${tag}.lck
|
||||
[[ -e $lock ]] && continue;
|
||||
echo "${tag}">$lock
|
||||
|
||||
docker pull gcr.io/${n}/${img}:${tag}
|
||||
docker tag gcr.io/${n}/${img}:${tag} ${user_name}/${n}.${img}:${tag}
|
||||
docker push ${user_name}/${n}.${img}:${tag}
|
||||
|
||||
[[ -e ./commit.lck ]] && break
|
||||
|
||||
mv ./gcr.io_mirror/${n}/${img}/${tag}.tmp ./gcr.io_mirror/${n}/${img}/${tag}.tag
|
||||
|
||||
echo -e "[gcr.io/${n}/${image}:${tag}](https://hub.docker.com/r/${user_name}/${n}.${image}/tags/)\n\n" >> CHANGES.md
|
||||
rm -rf $lock
|
||||
done
|
||||
}
|
||||
|
||||
function mirror()
|
||||
{
|
||||
num=$(find ./gcr.io_mirror/ -type f \( -iname "*.tmp" \) |wc -l)
|
||||
if [ $num -eq 0 ]; then
|
||||
ns=$(cat ./gcr_namespaces 2>/dev/null || echo google-containers)
|
||||
for n in ${ns[@]} ; do
|
||||
process_run "init_namespace $n"
|
||||
done
|
||||
wait ${!}
|
||||
fi
|
||||
|
||||
compare
|
||||
|
||||
tmps=$(find ./gcr.io_mirror/ -type f \( -iname "*.tmp" \) -exec dirname {} \; | uniq | cut -d'/' -f3-4)
|
||||
|
||||
if [ -z "$tmps" ]; then
|
||||
echo -e "${red} wait for push ${tmps[@]}"
|
||||
for img in ${tmps[@]} ; do
|
||||
n=$(echo ${img}|cut -d'/' -f1)
|
||||
image=$(echo ${img}|cut -d'/' -f2)
|
||||
process_run "pull_push_diff $n $image"
|
||||
done
|
||||
wait ${!}
|
||||
fi
|
||||
|
||||
images=($(find ./gcr.io_mirror/ -type f -name "*.tag" |uniq|sort))
|
||||
|
||||
cp ./gcr.io_mirror/CHANGES.md ./CHANGES1.md 2>/dev/null
|
||||
|
||||
find ./gcr.io_mirror/ -type f -name "*.md" -exec rm -rf {} \;
|
||||
|
||||
for img in ${images[@]} ; do
|
||||
n=$(echo ${img}|cut -d'/' -f3)
|
||||
image=$(echo ${img}|cut -d'/' -f4)
|
||||
tag=$(basename ${img} .tag)
|
||||
mkdir -p ./gcr.io_mirror/${n}/${image}
|
||||
if [ ! -e ./gcr.io_mirror/${n}/${image}/README.md ]; then
|
||||
echo -e "\n[gcr.io/${n}/${image}](https://hub.docker.com/r/${user_name}/${n}.${image}/tags/)\n-----\n\n" >> ./gcr.io_mirror/${n}/${image}/README.md
|
||||
echo -e "\n[gcr.io/${n}/${image}](https://hub.docker.com/r/${user_name}/${n}.${image}/tags/)\n-----\n\n" >> ./gcr.io_mirror/${n}/README.md
|
||||
fi
|
||||
|
||||
echo -e "[gcr.io/${n}/${image}:${tag}](https://hub.docker.com/r/${user_name}/${n}.${image}/tags/)\n\n" >> ./gcr.io_mirror/${n}/${image}/README.md
|
||||
done
|
||||
|
||||
commit
|
||||
}
|
||||
|
||||
function commit()
|
||||
{
|
||||
echo 1 > ./commit.lck
|
||||
ns=($(cat ./gcr_namespaces 2>/dev/null || echo google-containers))
|
||||
readme=./gcr.io_mirror/README.md
|
||||
export current_date=$(date +'%Y-%m-%d %H:%M')
|
||||
envsubst < README.tpl >"${readme}"
|
||||
|
||||
echo -e "Mirror ${#ns[@]} namespaces image from gcr.io\n-----\n\n" >> "${readme}"
|
||||
for n in ${ns[@]} ; do
|
||||
echo -e "[gcr.io/${n}/*](./${n}/README.md)\n\n" >> "${readme}"
|
||||
done
|
||||
|
||||
if [ -s CHANGES.md ]; then
|
||||
(echo -e "## $(date +'%Y-%m-%d %H:%M') \n" && cat CHANGES.md && cat CHANGES1.md 2>/dev/null) >> gcr.io_mirror/CHANGES.md
|
||||
fi
|
||||
|
||||
echo -e "${red} commit to github master"
|
||||
git -C ./gcr.io_mirror pull
|
||||
git -C ./gcr.io_mirror add -A
|
||||
git -C ./gcr.io_mirror commit -m "sync gcr.io's images at $(date +'%Y-%m-%d %H:%M')"
|
||||
git -C ./gcr.io_mirror push -f "https://${user_name}:${GH_TOKEN}@github.com/${user_name}/gcr.io_mirror.git" master:master
|
||||
|
||||
echo -e "${red} commit to github master:done"
|
||||
echo 1 > ./commit.done
|
||||
}
|
||||
|
||||
mirror &
|
||||
|
||||
while true;
|
||||
do
|
||||
duration=$SECONDS
|
||||
if [[ -e ./commit.done ]] || [[ $duration -ge $sec ]]; then
|
||||
|
||||
[[ $duration -ge $sec ]] && echo -e "${red} more than $(expr $sec / 60) min,abort this build"
|
||||
|
||||
[[ ! -e ./commit.done ]] && commit
|
||||
|
||||
IFS=$'\n'; for i in $(jobs); do echo "$i"; done
|
||||
kill $(jobs -p)
|
||||
IFS=$'\n'; for i in $(jobs); do echo "$i"; done
|
||||
break
|
||||
|
||||
else
|
||||
docker_dir=$(docker info | grep "Docker Root Dir" | cut -d':' -f2)
|
||||
used=$(df -h ${docker_dir}|awk '{if(NR>1)print $5}')
|
||||
echo -e "${red} duration:${duration}s, docker root dir :${docker_dir}:used:${used}"
|
||||
[[ ${used} > '70%' ]] && docker system prune -f -a
|
||||
sleep 30
|
||||
fi
|
||||
done
|
||||
|
||||
sleep 120
|
||||
echo "${red} bye bye"
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
google-samples
|
||||
kubernetes-helm
|
||||
k8s-minikube
|
||||
tf-on-k8s-dogfood
|
||||
spinnaker-marketplace
|
||||
google-containers
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
_PROCESS_PIPE_NAME="/tmp/$(cat /proc/sys/kernel/random/uuid)"
|
||||
_PROCESS_PIPE_ID=10
|
||||
|
||||
function _create_pipe()
|
||||
{
|
||||
|
||||
mkfifo ${_PROCESS_PIPE_NAME}
|
||||
eval exec "${_PROCESS_PIPE_ID}""<>${_PROCESS_PIPE_NAME}"
|
||||
|
||||
for ((i=0; i< $1; i++))
|
||||
do
|
||||
echo >&${_PROCESS_PIPE_ID}
|
||||
done
|
||||
}
|
||||
|
||||
function process_init()
|
||||
{
|
||||
_create_pipe $1
|
||||
}
|
||||
|
||||
function process_run()
|
||||
{
|
||||
cmd=$1
|
||||
|
||||
if [ -z "$cmd" ]; then
|
||||
echo "please input command to run"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
read -u${_PROCESS_PIPE_ID}
|
||||
{
|
||||
$cmd
|
||||
echo >&${_PROCESS_PIPE_ID}
|
||||
}&
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue