支持 gcr.io k8s.gcr.io registry.k8s.io quay.io ghcr.io 等镜像库

This commit is contained in:
AnJia 2022-09-06 15:10:54 +08:00
parent a9743d3808
commit e93d0a31ab
5 changed files with 374 additions and 194 deletions

View File

@ -7,204 +7,20 @@ on:
types: [created]
workflow_dispatch:
env:
REPO_NAME: ${{ github.event.repository.name }}
GH_USER: anjia0532
jobs:
build:
runs-on: ubuntu-latest
outputs:
GCR_IMAGE: ${{ steps.pullIssuesPorter.outputs.GCR_IMAGE }}
# ISSUE_NUMBER: ${{ steps.pullIssuesPorter.outputs.ISSUE_NUMBER }}
# MY_DOCKER_IMAGE_NAME: ${{ steps.transferImage.outputs.MY_DOCKER_IMAGE_NAME }}
SUCCESS: ${{ steps.successCheck.outputs.SUCCESS }}
if: contains(github.event.issue.labels.*.name, 'porter')
steps:
- name: Log into docker hub
uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
- name: get porter issues
id: pullIssuesPorter
uses: actions/github-script@v3.1.0
with:
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
const fs = require('fs')
let gcr_image
let title
let issues_author
const ev = JSON.parse(
fs.readFileSync(process.env.GITHUB_EVENT_PATH, 'utf8')
)
let issue_number = (ev.issues || {'number': -1})['number']
if(issue_number>0){
const issuesResponse = await github.issues.get({
owner: '${{ env.GH_USER }}',
repo: '${{ env.REPO_NAME }}',
issue_number: issue_number
})
title = issuesResponse.title
console.log('issues opened trigger')
}else{
const issuesResponse = await github.issues.listForRepo({
owner: '${{ env.GH_USER }}',
repo: '${{ env.REPO_NAME }}',
state: "open",
labels: "porter",
sort: "created",
direction: "desc",
per_page: 1
})
- name: 检出代码
uses: actions/checkout@v3
if (Array.isArray(issuesResponse["data"]) && issuesResponse["data"].length) {
title = issuesResponse["data"][0]["title"]
issue_number = issuesResponse["data"][0]["number"]
issues_author = issuesResponse["data"][0]["user"]["login"]
}
console.log("schedule trigger")
}
if(issue_number>0){
let start = 0
if (title.includes("[PORTER]")){
start = 8
}
gcr_image = title.substring(start).trim()
issues_body=''
is_error=false
if( gcr_image.includes("@")){
// 不支持带摘要 k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660
is_error=true
issues_body='@'+issues_author+' 拉取镜像不支持带摘要信息,请去除 @部分'
}else if( !gcr_image.includes("gcr.io")){
// 只支持 k8s.gcr.io 和 gcr.io
is_error=true
issues_body='@'+issues_author+' 不是说了么,只支持 k8s.gcr.io 和 gcr.io其他源请自己想办法'
}else{
issues_body='构建进展 [https://github.com/${{ env.GH_USER }}/${{ env.REPO_NAME }}/actions/runs/${{ github.run_id }}](https://github.com/${{ env.GH_USER }}/${{ env.REPO_NAME }}/actions/runs/${{ github.run_id }})'
}
const issuesComment = await github.issues.createComment({
owner: '${{ env.GH_USER }}',
repo: '${{ env.REPO_NAME }}',
issue_number: issue_number,
body: issues_body
});
console.log("create issues comment resp:", issuesComment["status"]);
console.log("gcr_image from issues is ", gcr_image,", issue_number is ",issue_number, ",issues_author is ", issues_author)
if(is_error){
core.setFailed("Error");
}
}else{
core.setFailed("No Images");
}
core.setOutput('GCR_IMAGE', gcr_image)
core.setOutput('ISSUE_NUMBER', issue_number)
- name: 设置 golang 环境
uses: actions/setup-go@v3
with:
go-version: 1.18
- name: Retrieve transfer image name
run: |
echo "::set-output name=MY_DOCKER_IMAGE_NAME::$(echo ${{ steps.pullIssuesPorter.outputs.GCR_IMAGE }} | sed 's/k8s\.gcr\.io/${{ secrets.DOCKER_HUB_USERNAME }}\/google-containers/g;s/gcr\.io/${{ secrets.DOCKER_HUB_USERNAME }}/g;s/\//\./g;s/ /\n/g;s/${{ secrets.DOCKER_HUB_USERNAME }}\./${{ secrets.DOCKER_HUB_USERNAME }}\//g')"
id: transferImage
- name: 运行 go 代码
run: go run main.go --github.token=${{ secrets.GITHUB_TOKEN }} --github.user=${{ github.repository_owner }} --github.repo=${{ github.event.repository.name }} --docker.registry=${{ secrets.DOCKER_REGISTRY }} --docker.namespace=${{ secrets.DOCKER_NAMESPACE }} --docker.user=${{ secrets.DOCKER_USER }} --docker.password=${{ secrets.DOCKER_PASSWORD }} --github.run_id=${{ github.run_id }}
- name: pull from gcr.io and push to docker hub
shell: bash
run: |
docker pull ${{ steps.pullIssuesPorter.outputs.GCR_IMAGE }}
echo ${{ steps.transferImage.outputs.MY_DOCKER_IMAGE_NAME }}
docker images
docker tag ${{ steps.pullIssuesPorter.outputs.GCR_IMAGE }} ${{ steps.transferImage.outputs.MY_DOCKER_IMAGE_NAME }}
docker push ${{ steps.transferImage.outputs.MY_DOCKER_IMAGE_NAME }}
- name: success check
id: successCheck
uses: actions/github-script@v3.1.0
if: ${{ success() }}
with:
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
core.setOutput('SUCCESS', true)
- name: Close Porter Issues
id: closePorterIssues
uses: actions/github-script@v3.1.0
if: ${{ always() }}
with:
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
if (${{ steps.pullIssuesPorter.outputs.ISSUE_NUMBER }} > 0){
const issuesResponse = await github.issues.update({
owner: '${{ env.GH_USER }}',
repo: '${{ env.REPO_NAME }}',
issue_number: ${{ steps.pullIssuesPorter.outputs.ISSUE_NUMBER }},
state: 'closed'
})
console.log("update issues resp:", issuesResponse["status"] == 200 ? "success" : "failed" )
let body = "转换失败,详见 [构建任务](https://github.com/${{ env.GH_USER }}/${{ env.REPO_NAME }}/actions/runs/${{ github.run_id }})"
let success = String(${{ steps.successCheck.outputs.SUCCESS }}).toLowerCase() == "true"
console.log("is success?", success)
let labels = []
if(success){
body = "转换完成 <br/>\n```bash \n#原镜像\n${{ steps.pullIssuesPorter.outputs.GCR_IMAGE }}\n\n\n#转换后镜像\n${{ steps.transferImage.outputs.MY_DOCKER_IMAGE_NAME }}\n\n\n#下载并重命名镜像\ndocker pull ${{ steps.transferImage.outputs.MY_DOCKER_IMAGE_NAME }}\ndocker tag ${{ steps.transferImage.outputs.MY_DOCKER_IMAGE_NAME }} ${{ steps.pullIssuesPorter.outputs.GCR_IMAGE }}\ndocker images | grep $(echo ${{ steps.pullIssuesPorter.outputs.GCR_IMAGE }}|awk -F':' '{print $1}')\n\n\n# 也可以用我写的脚本\nwget https://raw.githubusercontent.com/anjia0532/gcr.io_mirror/master/pull-k8s-image.sh && chmod +x pull-k8s-image.sh\n./pull-k8s-image.sh ${{ steps.pullIssuesPorter.outputs.GCR_IMAGE }}\n```"
labels=['success']
}else{
const jobsResponse = await github.actions.listJobsForWorkflowRun({
owner: '${{ env.GH_USER }}',
repo: '${{ env.REPO_NAME }}',
run_id: ${{ github.run_id }}
})
console.log("jobs",jobsResponse['data'])
body+="\n\n 日志:\n\n"
for(let job of jobsResponse['data']['jobs']){
body+="- ["+job.name+"]("+job.html_url+")"
}
labels=['failed']
}
let gcrImg = "${{ steps.pullIssuesPorter.outputs.GCR_IMAGE }}"
let colonIndex = gcrImg.indexOf(":")
if (colonIndex > 0) {
gcrImg = gcrImg.substr(0,colonIndex)
}
let names = gcrImg.split("/")
let registry = names[0]
names=names.splice(1,5)
if("k8s.gcr.io" == registry){
body+="\n\n\n k8s.gcr.io 镜像标签 Api(需梯子) <https://k8s.gcr.io/v2/"+names.join("/")+"/tags/list>\n\n k8s.gcr.io 镜像标签 UI(需梯子) <https://console.cloud.google.com/gcr/images/k8s-artifacts-prod/us/"+names.join("/")+">\n\n"
}else{
let uri=names[0]+"/global/"+names.slice(1,5).join("/")
if(names.length==1){
uri="global/"+names[0]
}
body+="\n\n\n gcr.io 镜像标签 Api(需梯子) <https://gcr.io/v2/"+names.join("/")+"/tags/list>\n\n gcr.io 镜像标签 UI(需梯子) <https://console.cloud.google.com/gcr/images/"+uri+">\n\n"
}
const issuesComment = await github.issues.createComment({
owner: '${{ env.GH_USER }}',
repo: '${{ env.REPO_NAME }}',
issue_number: ${{ steps.pullIssuesPorter.outputs.ISSUE_NUMBER }},
body: body
});
console.log("create issues comment resp:", issuesComment["status"] == 201 ? "success" : "failed" )
if(labels){
await github.issues.addLabels({
owner: '${{ env.GH_USER }}',
repo: '${{ env.REPO_NAME }}',
issue_number: ${{ steps.pullIssuesPorter.outputs.ISSUE_NUMBER }},
labels: labels
});
}
}

View File

@ -41,7 +41,23 @@ issues的内容无所谓可以为空
**注意:**
本项目目前仅支持 gcr.io和k8s.gcr.io 镜像
本项目目前仅支持 `gcr.io` , `k8s.gcr.io` , `registry.k8s.io` , `quay.io`, `ghcr.io` 镜像,其余镜像源可以提 Issues 反馈或者自己 Fork 一份,修改 `rules.yaml`
Fork/分叉代码自行维护
-------
- 必须: <https://github.com/anjia0532/gcr.io_mirror/fork> 点击连接在自己账号下分叉出 `gcr.io_mirror` 项目
- 可选: 修改 [./rules.yaml](./rules.yaml) 增加暂未支持的镜像库
- 在 [./settings/secrets/actions](./settings/secrets/actions) 创建自己的参数
`DOCKER_REGISTRY`: 如果推到 docker hub 为空即可
`DOCKER_NAMESPACE`: 如果推到 docker hub ,则是自己的 docker hub 账号(不带@email部分),例如我的 anjia0532
`DOCKER_USER`: 如果推到 docker hub,则是 docker hub 账号(不带@email部分),例如我的 anjia0532
`DOCKER_PASSWORD`: 如果推到 docker hub则是 docker hub 密码
k8s.gcr.io 和 gcr.io 镜像tags
------

35
go.mod Normal file
View File

@ -0,0 +1,35 @@
module image-mirror
go 1.18
require (
github.com/docker/docker v20.10.17+incompatible
github.com/google/go-github/v47 v47.0.0
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
)
require (
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.3.2 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
google.golang.org/appengine v1.6.7 // indirect
gotest.tools/v3 v3.3.0 // indirect
)

308
main.go Normal file
View File

@ -0,0 +1,308 @@
package main
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"github.com/google/go-github/v47/github"
"golang.org/x/oauth2"
"gopkg.in/alecthomas/kingpin.v2"
"gopkg.in/yaml.v3"
"io"
"io/ioutil"
"os"
"regexp"
"strings"
"text/template"
)
func main() {
ctx := context.Background()
var (
ghToken = kingpin.Flag("github.token", "Github token.").Short('t').String()
ghUser = kingpin.Flag("github.user", "Github Owner.").Short('u').String()
ghRepo = kingpin.Flag("github.repo", "Github Repo.").Short('p').String()
registry = kingpin.Flag("docker.registry", "Docker Registry.").Short('r').Default("").String()
registryNamespace = kingpin.Flag("docker.namespace", "Docker Registry Namespace.").Short('n').String()
registryUserName = kingpin.Flag("docker.user", "Docker Registry User.").Short('a').String()
registryPassword = kingpin.Flag("docker.password", "Docker Registry Password.").Short('p').String()
runId = kingpin.Flag("github.run_id", "Github Run Id.").Short('i').String()
)
kingpin.HelpFlag.Short('h')
kingpin.Parse()
config := &Config{
GhToken: *ghToken,
GhUser: *ghUser,
Repo: *ghRepo,
Registry: *registry,
RegistryNamespace: *registryNamespace,
RegistryUserName: *registryUserName,
RegistryPassword: *registryPassword,
RunId: *runId,
Rules: map[string]string{
"^gcr.io": "",
"^k8s.gcr.io": "google-containers",
"^registry.k8s.io": "google-containers",
"^quay.io": "quay",
"^ghcr.io": "ghcr",
},
}
rulesFile, err := ioutil.ReadFile("rules.yaml")
if err == nil {
rules := make(map[string]string)
err2 := yaml.Unmarshal(rulesFile, &rules)
if err2 == nil {
config.Rules = rules
}
}
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: config.GhToken},
)
tc := oauth2.NewClient(ctx, ts)
cli := github.NewClient(tc)
issues, err := getIssues(cli, ctx, config)
if err != nil {
fmt.Println("查询 Issues 报错,", err.Error())
os.Exit(-1)
}
if len(issues) == 0 {
fmt.Println("暂无需要搬运的镜像")
os.Exit(0)
}
// 可以用协程,但是懒得写
issue := issues[0]
fmt.Println("添加 构建进展 Comment")
commentIssues(issue, cli, ctx, "[构建进展](https://github.com/"+config.GhUser+"/"+config.Repo+"/actions/runs/"+config.RunId+")")
err, originImageName, targetImageName := mirrorByIssues(issue, config)
if err != nil {
commentErr := commentIssues(issue, cli, ctx, err.Error())
if commentErr != nil {
fmt.Println("提交 comment 报错", commentErr)
}
}
result := struct {
Success bool
Registry string
OriginImageName string
TargetImageName string
GhUser string
Repo string
RunId string
}{
Success: err == nil,
Registry: config.Registry,
OriginImageName: originImageName,
TargetImageName: targetImageName,
GhUser: *ghUser,
Repo: *ghRepo,
RunId: *runId,
}
var buf bytes.Buffer
tmpl, err := template.New("result").Parse(resultTpl)
err = tmpl.Execute(&buf, &result)
fmt.Println("添加 转换结果 Comment")
commentIssues(issue, cli, ctx, strings.ReplaceAll(buf.String(), "^", "`"))
fmt.Println("添加 转换结果 Label")
issuesAddLabels(issue, cli, ctx, result.Success)
fmt.Println("关闭 Issues")
issuesClose(issue, cli, ctx)
}
var resultTpl = `
{{ if .Success }}
{{ if .Registry }}
**转换完成**\n
^^^bash
#原镜像\n
{{ .OriginImageName }}\n\n\n
#转换后镜像\n
{{ .TargetImageName }}\n\n\n
#下载并重命名镜像\n
docker pull {{ .TargetImageName }}\n
docker tag {{ .TargetImageName }} {{ .originImageName }}\n
docker images | grep $(echo {{ .OriginImageName }} |awk -F':' '{print $1}')\n\n\n
^^^
{{ end }}
{{ else }}
**转换失败**\n
详见 [构建任务](https://github.com/{{ .GhUser }}/{{ .Repo }}/actions/runs/{{ .RunId }})
{{ end }}
`
func issuesClose(issues *github.Issue, cli *github.Client, ctx context.Context) {
names := strings.Split(*issues.RepositoryURL, "/")
state := "closed"
cli.Issues.Edit(ctx, names[len(names)-2], names[len(names)-1], issues.GetNumber(), &github.IssueRequest{
State: &state,
})
}
func issuesAddLabels(issues *github.Issue, cli *github.Client, ctx context.Context, success bool) {
names := strings.Split(*issues.RepositoryURL, "/")
label := "success"
if !success {
label = "failed"
}
cli.Issues.AddLabelsToIssue(ctx, names[len(names)-2], names[len(names)-1], issues.GetNumber(), []string{label})
}
func commentIssues(issues *github.Issue, cli *github.Client, ctx context.Context, comment string) error {
names := strings.Split(*issues.RepositoryURL, "/")
_, _, err := cli.Issues.CreateComment(ctx, names[len(names)-2], names[len(names)-1], issues.GetNumber(), &github.IssueComment{
Body: &comment,
})
return err
}
func mirrorByIssues(issues *github.Issue, config *Config) (err error, originImageName string, targetImageName string) {
// 去掉前缀 [PORTER] 整体去除前后空格
originImageName = strings.TrimSpace(strings.Replace(*issues.Title, "[PORTER]", "", 1))
targetImageName = originImageName
if strings.ContainsAny(originImageName, "@") {
return errors.New("@" + *issues.GetUser().Login + " 不支持同步带摘要信息的镜像"), originImageName, targetImageName
}
registrys := []string{}
for k, v := range config.Rules {
targetImageName = regexp.MustCompile(k).ReplaceAllString(targetImageName, v)
registrys = append(registrys, k)
}
if strings.EqualFold(targetImageName, originImageName) {
return errors.New("@" + *issues.GetUser().Login + " 暂不支持同步" + originImageName + ",目前仅支持同步 `" + strings.Join(registrys, " ,") + "`镜像"), originImageName, targetImageName
}
if len(config.RegistryNamespace) > 0 {
targetImageName = config.RegistryNamespace + targetImageName
}
if len(config.Registry) > 0 {
targetImageName = config.Registry + "/" + targetImageName
}
fmt.Println("source:", originImageName, " , target:", targetImageName)
cli, ctx, err := dockerLogin(config)
if err != nil {
return errors.New("@" + config.GhUser + " ,docker login 报错 `" + err.Error() + "`"), originImageName, targetImageName
}
//execCmd("docker", "login", config.Registry, "-u", config.RegistryUserName, "-p", config.RegistryPassword)
//execCmd("docker", "pull", originImageName)
err = dockerPull(originImageName, cli, ctx)
if err != nil {
return errors.New("@" + *issues.GetUser().Login + " ,docker pull 报错 `" + err.Error() + "`"), originImageName, targetImageName
}
//execCmd("docker", "tag", originImageName, targetImageName)
err = dockerTag(originImageName, targetImageName, cli, ctx)
if err != nil {
return errors.New("@" + config.GhUser + " ,docker tag 报错 `" + err.Error() + "`"), originImageName, targetImageName
}
//execCmd("docker", "push", targetImageName)
err = dockerPush(targetImageName, cli, ctx, config)
if err != nil {
return errors.New("@" + config.GhUser + " ,docker push 报错 `" + err.Error() + "`"), originImageName, targetImageName
}
return nil, originImageName, targetImageName
}
func dockerLogin(config *Config) (*client.Client, context.Context, error) {
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return nil, nil, err
}
fmt.Println("docker login, server: ", config.Registry, " user: ", config.RegistryUserName, ", password: ***")
authConfig := types.AuthConfig{
Username: config.RegistryUserName,
Password: config.RegistryPassword,
ServerAddress: config.Registry,
}
ctx := context.Background()
_, err = cli.RegistryLogin(ctx, authConfig)
if err != nil {
return nil, nil, err
}
return cli, ctx, nil
}
func dockerPull(originImageName string, cli *client.Client, ctx context.Context) error {
fmt.Println("docker pull ", originImageName)
pullOut, err := cli.ImagePull(ctx, originImageName, types.ImagePullOptions{})
if err != nil {
return err
}
defer pullOut.Close()
io.Copy(os.Stdout, pullOut)
return nil
}
func dockerTag(originImageName string, targetImageName string, cli *client.Client, ctx context.Context) error {
fmt.Println("docker tag ", originImageName, " ", targetImageName)
err := cli.ImageTag(ctx, originImageName, targetImageName)
return err
}
func dockerPush(targetImageName string, cli *client.Client, ctx context.Context, config *Config) error {
fmt.Println("docker push ", targetImageName)
authConfig := types.AuthConfig{
Username: config.RegistryUserName,
Password: config.RegistryPassword,
ServerAddress: config.Registry,
}
encodedJSON, err := json.Marshal(authConfig)
if err != nil {
return err
}
authStr := base64.URLEncoding.EncodeToString(encodedJSON)
pushOut, err := cli.ImagePush(ctx, targetImageName, types.ImagePushOptions{
RegistryAuth: authStr,
})
if err != nil {
return err
}
defer pushOut.Close()
io.Copy(os.Stdout, pushOut)
return nil
}
type Config struct {
GhToken string `yaml:"gh_token"`
GhUser string `yaml:"gh_user"`
Repo string `yaml:"repo"`
Registry string `yaml:"registry"`
RegistryNamespace string `yaml:"registry_namespace"`
RegistryUserName string `yaml:"registry_user_name"`
RegistryPassword string `yaml:"registry_password"`
Rules map[string]string `yaml:"rules"`
RunId string `yaml:"run_id"`
}
func getIssues(cli *github.Client, ctx context.Context, config *Config) ([]*github.Issue, error) {
issues, _, err := cli.Issues.ListByRepo(ctx, config.GhUser, config.Repo, &github.IssueListByRepoOptions{
//State: "closed",
State: "open",
Labels: []string{"porter"},
Sort: "created",
Direction: "desc",
// 防止被滥用每次最多只能拉20条虽然可以递归但是没必要。
//ListOptions: github.ListOptions{Page: 1, PerPage: 20},
// 考虑了下,每次还是只允许转一个吧
ListOptions: github.ListOptions{Page: 1, PerPage: 1},
})
return issues, err
}

5
rules.yaml Normal file
View File

@ -0,0 +1,5 @@
"^gcr.io": ""
"^k8s.gcr.io": "google-containers"
"^registry.k8s.io": "google-containers"
"^quay.io": "quay"
"^ghcr.io": "ghcr"