From 03d74ddd141addbce3473df8e4ecd37f9784d93f Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 11 Mar 2020 14:01:02 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5git=20=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/main.go | 3 +- backend/routes/git.go | 30 +- backend/services/git.go | 298 ++++++++++---- .../src/components/Settings/GitSettings.vue | 380 +++++++++++------- frontend/src/views/spider/SpiderDetail.vue | 2 +- 5 files changed, 494 insertions(+), 219 deletions(-) diff --git a/backend/main.go b/backend/main.go index 8819c0da..35496b58 100644 --- a/backend/main.go +++ b/backend/main.go @@ -259,8 +259,9 @@ func main() { // 文件 authGroup.GET("/file", routes.GetFile) // 获取文件 // Git - authGroup.GET("/git/branches", routes.GetGitBranches) // 获取 Git 分支 + authGroup.GET("/git/branches", routes.GetGitRemoteBranches) // 获取 Git 分支 authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥 + authGroup.GET("/git/commits", routes.GetGitCommits) // 获取 Git Commits } } diff --git a/backend/routes/git.go b/backend/routes/git.go index 56fbc683..2b76dc6e 100644 --- a/backend/routes/git.go +++ b/backend/routes/git.go @@ -1,16 +1,19 @@ package routes import ( + "crawlab/model" "crawlab/services" "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" "net/http" ) -func GetGitBranches(c *gin.Context) { +func GetGitRemoteBranches(c *gin.Context) { url := c.Query("url") - branches, err := services.GetGitBranches(url) + branches, err := services.GetGitRemoteBranches(url) if err != nil { HandleError(http.StatusInternalServerError, c, err) + return } c.JSON(http.StatusOK, Response{ Status: "ok", @@ -27,3 +30,26 @@ func GetGitSshPublicKey(c *gin.Context) { Data: content, }) } + +func GetGitCommits(c *gin.Context) { + spiderId := c.Query("spider_id") + if spiderId == "" || !bson.IsObjectIdHex(spiderId) { + HandleErrorF(http.StatusInternalServerError, c, "invalid request") + return + } + spider, err := model.GetSpider(bson.ObjectIdHex(spiderId)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + commits, err := services.GetGitCommits(spider) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: commits, + }) +} diff --git a/backend/services/git.go b/backend/services/git.go index 1bb8b3b9..e4626dd7 100644 --- a/backend/services/git.go +++ b/backend/services/git.go @@ -12,6 +12,7 @@ import ( "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh" "io/ioutil" "net/url" @@ -21,6 +22,7 @@ import ( "regexp" "runtime/debug" "strings" + "time" ) var GitCron *GitCronScheduler @@ -29,6 +31,101 @@ type GitCronScheduler struct { cron *cron.Cron } +type GitBranch struct { + Hash string `json:"hash"` + Name string `json:"name"` + Label string `json:"label"` +} + +type GitTag struct { + Hash string `json:"hash"` + Name string `json:"name"` + Label string `json:"label"` +} + +type GitCommit struct { + Hash string `json:"hash"` + TreeHash string `json:"tree_hash"` + Author string `json:"author"` + Email string `json:"email"` + Message string `json:"message"` + IsHead bool `json:"is_head"` + Ts time.Time `json:"ts"` + Branches []GitBranch `json:"branches"` + Tags []GitTag `json:"tags"` +} + +func (g *GitCronScheduler) Start() error { + c := cron.New(cron.WithSeconds()) + + // 启动cron服务 + g.cron.Start() + + // 更新任务列表 + if err := g.Update(); err != nil { + log.Errorf("update scheduler error: %s", err.Error()) + debug.PrintStack() + return err + } + + // 每30秒更新一次任务列表 + spec := "*/30 * * * * *" + if _, err := c.AddFunc(spec, UpdateGitCron); err != nil { + log.Errorf("add func update schedulers error: %s", err.Error()) + debug.PrintStack() + return err + } + + return nil +} + +func (g *GitCronScheduler) RemoveAll() { + entries := g.cron.Entries() + for i := 0; i < len(entries); i++ { + g.cron.Remove(entries[i].ID) + } +} + +func (g *GitCronScheduler) Update() error { + // 删除所有定时任务 + g.RemoveAll() + + // 获取开启 Git 自动同步的爬虫 + spiders, err := model.GetSpiderAllList(bson.M{"git_auto_sync": true}) + if err != nil { + log.Errorf("get spider list error: %s", err.Error()) + debug.PrintStack() + return err + } + + // 遍历任务列表 + for _, s := range spiders { + // 添加到定时任务 + if err := g.AddJob(s); err != nil { + log.Errorf("add job error: %s, job: %s, cron: %s", err.Error(), s.Name, s.GitSyncFrequency) + debug.PrintStack() + return err + } + } + + return nil +} + +func (g *GitCronScheduler) AddJob(s model.Spider) error { + spec := s.GitSyncFrequency + + // 添加定时任务 + _, err := g.cron.AddFunc(spec, AddGitCronJob(s)) + if err != nil { + log.Errorf("add func task error: %s", err.Error()) + debug.PrintStack() + return err + } + + return nil +} + +// 保存爬虫Git同步错误 func SaveSpiderGitSyncError(s model.Spider, errMsg string) { s, _ = model.GetSpider(s.Id) s.GitSyncError = errMsg @@ -39,7 +136,8 @@ func SaveSpiderGitSyncError(s model.Spider, errMsg string) { } } -func GetGitBranches(url string) (branches []string, err error) { +// 获得Git分支 +func GetGitRemoteBranches(url string) (branches []string, err error) { var stdout bytes.Buffer var stderr bytes.Buffer @@ -63,6 +161,7 @@ func GetGitBranches(url string) (branches []string, err error) { return branches, nil } +// 重置爬虫Git func ResetSpiderGit(s model.Spider) (err error) { // 删除文件夹 if err := os.RemoveAll(s.Src); err != nil { @@ -86,6 +185,7 @@ func ResetSpiderGit(s model.Spider) (err error) { return nil } +// 同步爬虫Git func SyncSpiderGit(s model.Spider) (err error) { // 如果 .git 不存在,初始化一个仓库 if !utils.Exists(path.Join(s.Src, ".git")) { @@ -165,6 +265,7 @@ func SyncSpiderGit(s model.Spider) (err error) { RemoteName: "origin", Force: true, Auth: auth, + Tags: git.AllTags, }) // 获得 WorkTree @@ -178,8 +279,10 @@ func SyncSpiderGit(s model.Spider) (err error) { // 拉取 repo if err := wt.Pull(&git.PullOptions{ - RemoteName: "origin", - Auth: auth, + RemoteName: "origin", + Auth: auth, + ReferenceName: plumbing.HEAD, + SingleBranch: false, }); err != nil { if err.Error() == "already up-to-date" { // 检查是否为 Scrapy @@ -221,76 +324,7 @@ func SyncSpiderGit(s model.Spider) (err error) { return nil } -func (g *GitCronScheduler) Start() error { - c := cron.New(cron.WithSeconds()) - - // 启动cron服务 - g.cron.Start() - - // 更新任务列表 - if err := g.Update(); err != nil { - log.Errorf("update scheduler error: %s", err.Error()) - debug.PrintStack() - return err - } - - // 每30秒更新一次任务列表 - spec := "*/30 * * * * *" - if _, err := c.AddFunc(spec, UpdateGitCron); err != nil { - log.Errorf("add func update schedulers error: %s", err.Error()) - debug.PrintStack() - return err - } - - return nil -} - -func (g *GitCronScheduler) RemoveAll() { - entries := g.cron.Entries() - for i := 0; i < len(entries); i++ { - g.cron.Remove(entries[i].ID) - } -} - -func (g *GitCronScheduler) Update() error { - // 删除所有定时任务 - g.RemoveAll() - - // 获取开启 Git 自动同步的爬虫 - spiders, err := model.GetSpiderAllList(bson.M{"git_auto_sync": true}) - if err != nil { - log.Errorf("get spider list error: %s", err.Error()) - debug.PrintStack() - return err - } - - // 遍历任务列表 - for _, s := range spiders { - // 添加到定时任务 - if err := g.AddJob(s); err != nil { - log.Errorf("add job error: %s, job: %s, cron: %s", err.Error(), s.Name, s.GitSyncFrequency) - debug.PrintStack() - return err - } - } - - return nil -} - -func (g *GitCronScheduler) AddJob(s model.Spider) error { - spec := s.GitSyncFrequency - - // 添加定时任务 - _, err := g.cron.AddFunc(spec, AddGitCronJob(s)) - if err != nil { - log.Errorf("add func task error: %s", err.Error()) - debug.PrintStack() - return err - } - - return nil -} - +// 添加Git定时任务 func AddGitCronJob(s model.Spider) func() { return func() { if err := SyncSpiderGit(s); err != nil { @@ -301,6 +335,7 @@ func AddGitCronJob(s model.Spider) func() { } } +// 更新Git定时任务 func UpdateGitCron() { if err := GitCron.Update(); err != nil { log.Errorf(err.Error()) @@ -308,6 +343,7 @@ func UpdateGitCron() { } } +// 获取SSH公钥 func GetGitSshPublicKey() string { if !utils.Exists(path.Join(os.Getenv("HOME"), ".ssh")) || !utils.Exists(path.Join(os.Getenv("HOME"), ".ssh", "id_rsa")) || @@ -322,3 +358,119 @@ func GetGitSshPublicKey() string { } return string(content) } + +func GetGitBranches(s model.Spider) (branches []GitBranch, err error) { + // 打开 repo + repo, err := git.PlainOpen(s.Src) + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return branches, err + } + + iter, err := repo.Branches() + if iter == nil { + return branches, nil + } + if err := iter.ForEach(func(reference *plumbing.Reference) error { + branches = append(branches, GitBranch{ + Hash: reference.Hash().String(), + Name: reference.Name().String(), + Label: reference.Name().Short(), + }) + return nil + }); err != nil { + return branches, err + } + + return branches, nil +} + +func GetGitTags(s model.Spider) (tags []GitTag, err error) { + // 打开 repo + repo, err := git.PlainOpen(s.Src) + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return tags, err + } + + iter, err := repo.Tags() + if iter == nil { + return tags, nil + } + if err := iter.ForEach(func(reference *plumbing.Reference) error { + tags = append(tags, GitTag{ + Hash: reference.Hash().String(), + Name: reference.Name().String(), + Label: reference.Name().Short(), + }) + return nil + }); err != nil { + return tags, err + } + + return tags, nil +} + +func GetHeadHash(repo *git.Repository) string { + head, _ := repo.Head() + return head.Hash().String() +} + +func GetGitCommits(s model.Spider) (commits []GitCommit, err error) { + // 打开 repo + repo, err := git.PlainOpen(s.Src) + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return commits, err + } + + // 获取分支列表 + branches, err := GetGitBranches(s) + branchesDict := make(map[string][]GitBranch) + for _, b := range branches { + branchesDict[b.Hash] = append(branchesDict[b.Hash], b) + } + + // 获取标签列表 + tags, err := GetGitTags(s) + tagsDict := make(map[string][]GitTag) + for _, t := range tags { + tagsDict[t.Hash] = append(tagsDict[t.Hash], t) + } + + // 获取日志遍历器 + iter, err := repo.Log(&git.LogOptions{ + All: true, + }) + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return commits, err + } + + // 遍历日志 + if err := iter.ForEach(func(commit *object.Commit) error { + gc := GitCommit{ + Hash: commit.Hash.String(), + TreeHash: commit.TreeHash.String(), + Message: commit.Message, + Author: commit.Author.Name, + Email: commit.Author.Email, + Ts: commit.Author.When, + IsHead: commit.Hash.String() == GetHeadHash(repo), + Branches: branchesDict[commit.Hash.String()], + Tags: tagsDict[commit.Hash.String()], + } + commits = append(commits, gc) + return nil + }); err != nil { + log.Error(err.Error()) + debug.PrintStack() + return commits, err + } + + return commits, nil +} diff --git a/frontend/src/components/Settings/GitSettings.vue b/frontend/src/components/Settings/GitSettings.vue index 4a8c791b..80d8bd72 100644 --- a/frontend/src/components/Settings/GitSettings.vue +++ b/frontend/src/components/Settings/GitSettings.vue @@ -1,159 +1,217 @@ @@ -318,4 +391,27 @@ export default { text-align: right; margin-top: 10px; } + + .git-settings .log { + height: calc(100vh - 280px); + overflow: auto; + } + + .git-settings .log .commit { + border-top: 1px solid rgb(244, 244, 245); + padding: 10px 0; + } + + .git-settings .log .commit .row { + display: flex; + justify-content: space-between; + } + + .git-settings .log .el-timeline-item { + /*cursor: pointer;*/ + } + + .git-settings .log .commit .row .tags .el-tag { + margin-right: 5px; + } diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue index 02038caf..2b137a79 100644 --- a/frontend/src/views/spider/SpiderDetail.vue +++ b/frontend/src/views/spider/SpiderDetail.vue @@ -22,7 +22,7 @@ - +