加入git 日志

This commit is contained in:
marvzhang
2020-03-11 14:01:02 +08:00
parent 5d79bf3fe8
commit 03d74ddd14
5 changed files with 494 additions and 219 deletions

View File

@@ -259,8 +259,9 @@ func main() {
// 文件
authGroup.GET("/file", routes.GetFile) // 获取文件
// Git
authGroup.GET("/git/branches", routes.GetGitBranches) // 获取 Git 分支
authGroup.GET("/git/branches", routes.GetGitRemoteBranches) // 获取 Git 分支
authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥
authGroup.GET("/git/commits", routes.GetGitCommits) // 获取 Git Commits
}
}

View File

@@ -1,16 +1,19 @@
package routes
import (
"crawlab/model"
"crawlab/services"
"github.com/gin-gonic/gin"
"github.com/globalsign/mgo/bson"
"net/http"
)
func GetGitBranches(c *gin.Context) {
func GetGitRemoteBranches(c *gin.Context) {
url := c.Query("url")
branches, err := services.GetGitBranches(url)
branches, err := services.GetGitRemoteBranches(url)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
@@ -27,3 +30,26 @@ func GetGitSshPublicKey(c *gin.Context) {
Data: content,
})
}
func GetGitCommits(c *gin.Context) {
spiderId := c.Query("spider_id")
if spiderId == "" || !bson.IsObjectIdHex(spiderId) {
HandleErrorF(http.StatusInternalServerError, c, "invalid request")
return
}
spider, err := model.GetSpider(bson.ObjectIdHex(spiderId))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
commits, err := services.GetGitCommits(spider)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: commits,
})
}

View File

@@ -12,6 +12,7 @@ import (
"gopkg.in/src-d/go-git.v4"
"gopkg.in/src-d/go-git.v4/config"
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/plumbing/object"
"gopkg.in/src-d/go-git.v4/plumbing/transport/ssh"
"io/ioutil"
"net/url"
@@ -21,6 +22,7 @@ import (
"regexp"
"runtime/debug"
"strings"
"time"
)
var GitCron *GitCronScheduler
@@ -29,6 +31,101 @@ type GitCronScheduler struct {
cron *cron.Cron
}
type GitBranch struct {
Hash string `json:"hash"`
Name string `json:"name"`
Label string `json:"label"`
}
type GitTag struct {
Hash string `json:"hash"`
Name string `json:"name"`
Label string `json:"label"`
}
type GitCommit struct {
Hash string `json:"hash"`
TreeHash string `json:"tree_hash"`
Author string `json:"author"`
Email string `json:"email"`
Message string `json:"message"`
IsHead bool `json:"is_head"`
Ts time.Time `json:"ts"`
Branches []GitBranch `json:"branches"`
Tags []GitTag `json:"tags"`
}
func (g *GitCronScheduler) Start() error {
c := cron.New(cron.WithSeconds())
// 启动cron服务
g.cron.Start()
// 更新任务列表
if err := g.Update(); err != nil {
log.Errorf("update scheduler error: %s", err.Error())
debug.PrintStack()
return err
}
// 每30秒更新一次任务列表
spec := "*/30 * * * * *"
if _, err := c.AddFunc(spec, UpdateGitCron); err != nil {
log.Errorf("add func update schedulers error: %s", err.Error())
debug.PrintStack()
return err
}
return nil
}
func (g *GitCronScheduler) RemoveAll() {
entries := g.cron.Entries()
for i := 0; i < len(entries); i++ {
g.cron.Remove(entries[i].ID)
}
}
func (g *GitCronScheduler) Update() error {
// 删除所有定时任务
g.RemoveAll()
// 获取开启 Git 自动同步的爬虫
spiders, err := model.GetSpiderAllList(bson.M{"git_auto_sync": true})
if err != nil {
log.Errorf("get spider list error: %s", err.Error())
debug.PrintStack()
return err
}
// 遍历任务列表
for _, s := range spiders {
// 添加到定时任务
if err := g.AddJob(s); err != nil {
log.Errorf("add job error: %s, job: %s, cron: %s", err.Error(), s.Name, s.GitSyncFrequency)
debug.PrintStack()
return err
}
}
return nil
}
func (g *GitCronScheduler) AddJob(s model.Spider) error {
spec := s.GitSyncFrequency
// 添加定时任务
_, err := g.cron.AddFunc(spec, AddGitCronJob(s))
if err != nil {
log.Errorf("add func task error: %s", err.Error())
debug.PrintStack()
return err
}
return nil
}
// 保存爬虫Git同步错误
func SaveSpiderGitSyncError(s model.Spider, errMsg string) {
s, _ = model.GetSpider(s.Id)
s.GitSyncError = errMsg
@@ -39,7 +136,8 @@ func SaveSpiderGitSyncError(s model.Spider, errMsg string) {
}
}
func GetGitBranches(url string) (branches []string, err error) {
// 获得Git分支
func GetGitRemoteBranches(url string) (branches []string, err error) {
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -63,6 +161,7 @@ func GetGitBranches(url string) (branches []string, err error) {
return branches, nil
}
// 重置爬虫Git
func ResetSpiderGit(s model.Spider) (err error) {
// 删除文件夹
if err := os.RemoveAll(s.Src); err != nil {
@@ -86,6 +185,7 @@ func ResetSpiderGit(s model.Spider) (err error) {
return nil
}
// 同步爬虫Git
func SyncSpiderGit(s model.Spider) (err error) {
// 如果 .git 不存在,初始化一个仓库
if !utils.Exists(path.Join(s.Src, ".git")) {
@@ -165,6 +265,7 @@ func SyncSpiderGit(s model.Spider) (err error) {
RemoteName: "origin",
Force: true,
Auth: auth,
Tags: git.AllTags,
})
// 获得 WorkTree
@@ -178,8 +279,10 @@ func SyncSpiderGit(s model.Spider) (err error) {
// 拉取 repo
if err := wt.Pull(&git.PullOptions{
RemoteName: "origin",
Auth: auth,
RemoteName: "origin",
Auth: auth,
ReferenceName: plumbing.HEAD,
SingleBranch: false,
}); err != nil {
if err.Error() == "already up-to-date" {
// 检查是否为 Scrapy
@@ -221,76 +324,7 @@ func SyncSpiderGit(s model.Spider) (err error) {
return nil
}
func (g *GitCronScheduler) Start() error {
c := cron.New(cron.WithSeconds())
// 启动cron服务
g.cron.Start()
// 更新任务列表
if err := g.Update(); err != nil {
log.Errorf("update scheduler error: %s", err.Error())
debug.PrintStack()
return err
}
// 每30秒更新一次任务列表
spec := "*/30 * * * * *"
if _, err := c.AddFunc(spec, UpdateGitCron); err != nil {
log.Errorf("add func update schedulers error: %s", err.Error())
debug.PrintStack()
return err
}
return nil
}
func (g *GitCronScheduler) RemoveAll() {
entries := g.cron.Entries()
for i := 0; i < len(entries); i++ {
g.cron.Remove(entries[i].ID)
}
}
func (g *GitCronScheduler) Update() error {
// 删除所有定时任务
g.RemoveAll()
// 获取开启 Git 自动同步的爬虫
spiders, err := model.GetSpiderAllList(bson.M{"git_auto_sync": true})
if err != nil {
log.Errorf("get spider list error: %s", err.Error())
debug.PrintStack()
return err
}
// 遍历任务列表
for _, s := range spiders {
// 添加到定时任务
if err := g.AddJob(s); err != nil {
log.Errorf("add job error: %s, job: %s, cron: %s", err.Error(), s.Name, s.GitSyncFrequency)
debug.PrintStack()
return err
}
}
return nil
}
func (g *GitCronScheduler) AddJob(s model.Spider) error {
spec := s.GitSyncFrequency
// 添加定时任务
_, err := g.cron.AddFunc(spec, AddGitCronJob(s))
if err != nil {
log.Errorf("add func task error: %s", err.Error())
debug.PrintStack()
return err
}
return nil
}
// 添加Git定时任务
func AddGitCronJob(s model.Spider) func() {
return func() {
if err := SyncSpiderGit(s); err != nil {
@@ -301,6 +335,7 @@ func AddGitCronJob(s model.Spider) func() {
}
}
// 更新Git定时任务
func UpdateGitCron() {
if err := GitCron.Update(); err != nil {
log.Errorf(err.Error())
@@ -308,6 +343,7 @@ func UpdateGitCron() {
}
}
// 获取SSH公钥
func GetGitSshPublicKey() string {
if !utils.Exists(path.Join(os.Getenv("HOME"), ".ssh")) ||
!utils.Exists(path.Join(os.Getenv("HOME"), ".ssh", "id_rsa")) ||
@@ -322,3 +358,119 @@ func GetGitSshPublicKey() string {
}
return string(content)
}
func GetGitBranches(s model.Spider) (branches []GitBranch, err error) {
// 打开 repo
repo, err := git.PlainOpen(s.Src)
if err != nil {
log.Error(err.Error())
debug.PrintStack()
return branches, err
}
iter, err := repo.Branches()
if iter == nil {
return branches, nil
}
if err := iter.ForEach(func(reference *plumbing.Reference) error {
branches = append(branches, GitBranch{
Hash: reference.Hash().String(),
Name: reference.Name().String(),
Label: reference.Name().Short(),
})
return nil
}); err != nil {
return branches, err
}
return branches, nil
}
func GetGitTags(s model.Spider) (tags []GitTag, err error) {
// 打开 repo
repo, err := git.PlainOpen(s.Src)
if err != nil {
log.Error(err.Error())
debug.PrintStack()
return tags, err
}
iter, err := repo.Tags()
if iter == nil {
return tags, nil
}
if err := iter.ForEach(func(reference *plumbing.Reference) error {
tags = append(tags, GitTag{
Hash: reference.Hash().String(),
Name: reference.Name().String(),
Label: reference.Name().Short(),
})
return nil
}); err != nil {
return tags, err
}
return tags, nil
}
func GetHeadHash(repo *git.Repository) string {
head, _ := repo.Head()
return head.Hash().String()
}
func GetGitCommits(s model.Spider) (commits []GitCommit, err error) {
// 打开 repo
repo, err := git.PlainOpen(s.Src)
if err != nil {
log.Error(err.Error())
debug.PrintStack()
return commits, err
}
// 获取分支列表
branches, err := GetGitBranches(s)
branchesDict := make(map[string][]GitBranch)
for _, b := range branches {
branchesDict[b.Hash] = append(branchesDict[b.Hash], b)
}
// 获取标签列表
tags, err := GetGitTags(s)
tagsDict := make(map[string][]GitTag)
for _, t := range tags {
tagsDict[t.Hash] = append(tagsDict[t.Hash], t)
}
// 获取日志遍历器
iter, err := repo.Log(&git.LogOptions{
All: true,
})
if err != nil {
log.Error(err.Error())
debug.PrintStack()
return commits, err
}
// 遍历日志
if err := iter.ForEach(func(commit *object.Commit) error {
gc := GitCommit{
Hash: commit.Hash.String(),
TreeHash: commit.TreeHash.String(),
Message: commit.Message,
Author: commit.Author.Name,
Email: commit.Author.Email,
Ts: commit.Author.When,
IsHead: commit.Hash.String() == GetHeadHash(repo),
Branches: branchesDict[commit.Hash.String()],
Tags: tagsDict[commit.Hash.String()],
}
commits = append(commits, gc)
return nil
}); err != nil {
log.Error(err.Error())
debug.PrintStack()
return commits, err
}
return commits, nil
}

View File

@@ -1,159 +1,217 @@
<template>
<div class="git-settings">
<h3 class="title">{{$t('Git Settings')}}</h3>
<el-form
class="git-settings-form"
label-width="150px"
:model="spiderForm"
ref="git-settings-form"
>
<el-form-item
:label="$t('Git URL')"
prop="git_url"
required
<el-tabs
v-model="activeTabName"
class="git-settings"
>
<el-tab-pane :label="$t('Settings')" name="settings">
<el-form
class="git-settings-form"
label-width="150px"
:model="spiderForm"
ref="git-settings-form"
>
<el-input
v-model="spiderForm.git_url"
:placeholder="$t('Git URL')"
@blur="onGitUrlChange"
<el-form-item
:label="$t('Git URL')"
prop="git_url"
required
>
</el-input>
</el-form-item>
<el-form-item
:label="$t('Has Credential')"
prop="git_has_credential"
>
<el-switch
v-model="spiderForm.git_has_credential"
size="small"
active-color="#67C23A"
/>
</el-form-item>
<el-form-item
v-if="spiderForm.git_has_credential"
:label="$t('Git Username')"
prop="git_username"
>
<el-input
v-model="spiderForm.git_username"
:placeholder="$t('Git Username')"
<el-input
v-model="spiderForm.git_url"
:placeholder="$t('Git URL')"
@blur="onGitUrlChange"
>
</el-input>
</el-form-item>
<el-form-item
:label="$t('Has Credential')"
prop="git_has_credential"
>
</el-input>
</el-form-item>
<el-form-item
v-if="spiderForm.git_has_credential"
:label="$t('Git Password')"
prop="git_password"
>
<el-input
v-model="spiderForm.git_password"
:placeholder="$t('Git Password')"
type="password"
>
</el-input>
</el-form-item>
<el-form-item
:label="$t('Git Branch')"
prop="git_branch"
required
>
<el-select
v-model="spiderForm.git_branch"
:placeholder="$t('Git Branch')"
:disabled="!spiderForm.git_url || isGitBranchesLoading"
>
<el-option
v-for="op in gitBranches"
:key="op"
:value="op"
:label="op"
<el-switch
v-model="spiderForm.git_has_credential"
size="small"
active-color="#67C23A"
/>
</el-select>
</el-form-item>
<el-form-item
:label="$t('Auto Sync')"
prop="git_auto_sync"
>
<el-switch
v-model="spiderForm.git_auto_sync"
size="small"
active-color="#67C23A"
/>
</el-form-item>
<el-form-item
v-if="spiderForm.git_auto_sync"
:label="$t('Sync Frequency')"
prop="git_sync_frequency"
required
>
<el-select
v-model="spiderForm.git_sync_frequency"
:placeholder="$t('Sync Frequency')"
</el-form-item>
<el-form-item
v-if="spiderForm.git_has_credential"
:label="$t('Git Username')"
prop="git_username"
>
<el-option
v-for="op in syncFrequencies"
:key="op.value"
:value="op.value"
:label="op.label"
<el-input
v-model="spiderForm.git_username"
:placeholder="$t('Git Username')"
>
</el-input>
</el-form-item>
<el-form-item
v-if="spiderForm.git_has_credential"
:label="$t('Git Password')"
prop="git_password"
>
<el-input
v-model="spiderForm.git_password"
:placeholder="$t('Git Password')"
type="password"
>
</el-input>
</el-form-item>
<el-form-item
:label="$t('Git Branch')"
prop="git_branch"
required
>
<el-select
v-model="spiderForm.git_branch"
:placeholder="$t('Git Branch')"
:disabled="!spiderForm.git_url || isGitBranchesLoading"
>
<el-option
v-for="op in gitBranches"
:key="op"
:value="op"
:label="op"
/>
</el-select>
</el-form-item>
<el-form-item
:label="$t('Auto Sync')"
prop="git_auto_sync"
>
<el-switch
v-model="spiderForm.git_auto_sync"
size="small"
active-color="#67C23A"
/>
</el-select>
</el-form-item>
<el-form-item
v-if="spiderForm.git_sync_error"
:label="$t('Error Message')"
prop="git_git_sync_error"
>
<el-alert
type="error"
:closable="false"
</el-form-item>
<el-form-item
v-if="spiderForm.git_auto_sync"
:label="$t('Sync Frequency')"
prop="git_sync_frequency"
required
>
{{spiderForm.git_sync_error}}
</el-alert>
</el-form-item>
<el-form-item
v-if="sshPublicKey"
:label="$t('SSH Public Key')"
>
<el-alert
type="info"
:closable="false"
<el-select
v-model="spiderForm.git_sync_frequency"
:placeholder="$t('Sync Frequency')"
>
<el-option
v-for="op in syncFrequencies"
:key="op.value"
:value="op.value"
:label="op.label"
/>
</el-select>
</el-form-item>
<el-form-item
v-if="spiderForm.git_sync_error"
:label="$t('Error Message')"
prop="git_git_sync_error"
>
{{sshPublicKey}}
</el-alert>
<span class="copy" @click="copySshPublicKey">
<el-alert
type="error"
:closable="false"
>
{{spiderForm.git_sync_error}}
</el-alert>
</el-form-item>
<el-form-item
v-if="sshPublicKey"
:label="$t('SSH Public Key')"
>
<el-alert
type="info"
:closable="false"
>
{{sshPublicKey}}
</el-alert>
<span class="copy" @click="copySshPublicKey">
<i class="el-icon-copy-document"></i>
{{$t('Copy')}}
</span>
<input id="ssh-public-key" v-model="sshPublicKey" v-show="true">
</el-form-item>
</el-form>
<div class="action-wrapper">
<el-button
size="small"
type="warning"
:disabled="isGitResetLoading"
:icon="isGitResetLoading ? 'el-icon-loading' : 'el-icon-refresh-left'"
@click="onReset"
<input id="ssh-public-key" v-model="sshPublicKey" v-show="true">
</el-form-item>
</el-form>
<div class="action-wrapper">
<el-button
size="small"
type="warning"
:disabled="isGitResetLoading"
:icon="isGitResetLoading ? 'el-icon-loading' : 'el-icon-refresh-left'"
@click="onReset"
>
{{$t('Reset')}}
</el-button>
<el-button
size="small"
type="danger"
:icon="isGitSyncLoading ? 'el-icon-loading' : 'el-icon-refresh'"
:disabled="!spiderForm.git_url || isGitSyncLoading"
@click="onSync"
>
{{$t('Sync')}}
</el-button>
<el-button size="small" type="success" @click="onSave" icon="el-icon-check">
{{$t('Save')}}
</el-button>
</div>
</el-tab-pane>
<el-tab-pane label="Log" name="log">
<el-timeline
class="log"
>
{{$t('Reset')}}
</el-button>
<el-button
size="small"
type="danger"
:icon="isGitSyncLoading ? 'el-icon-loading' : 'el-icon-refresh'"
:disabled="!spiderForm.git_url || isGitSyncLoading"
@click="onSync"
>
{{$t('Sync')}}
</el-button>
<el-button size="small" type="success" @click="onSave" icon="el-icon-check">
{{$t('Save')}}
</el-button>
</div>
</div>
<el-timeline-item
v-for="c in commits"
:key="c.hash"
:timestamp="c.ts"
:type="c.is_head ? 'primary' : ''"
>
<div class="commit">
<div class="row">
<div class="message">
{{c.message}}
</div>
<div class="author">
{{c.author}} ({{c.email}})
</div>
</div>
<div class="row" style="margin-top: 5px">
<div class="tags">
<el-tag
v-if="c.is_head"
type="primary"
size="mini"
>
<i class="fa fa-tag"></i>
HEAD
</el-tag>
<el-tag
v-for="b in c.branches"
:key="b.name"
:type="b.label === 'master' ? 'danger' : 'warning'"
size="mini"
>
<i class="fa fa-tag"></i>
{{b.label}}
</el-tag>
<el-tag
v-for="t in c.tags"
:key="t.name"
type="success"
size="mini"
>
<i class="fa fa-tag"></i>
{{t.label}}
</el-tag>
</div>
</div>
</div>
</el-timeline-item>
</el-timeline>
</el-tab-pane>
</el-tabs>
</template>
<script>
import dayjs from 'dayjs'
import {
mapState
} from 'vuex'
@@ -176,7 +234,9 @@ export default {
{ label: '12h', value: '0 0 0/12 * * *' },
{ label: '1d', value: '0 0 0 0 * *' }
],
sshPublicKey: ''
sshPublicKey: '',
activeTabName: 'settings',
commits: []
}
},
computed: {
@@ -217,6 +277,7 @@ export default {
}
} finally {
this.isGitSyncLoading = false
this.updateGit()
await this.$store.dispatch('spider/getSpiderData', this.$route.params.id)
}
this.$st.sendEv('爬虫详情', 'Git 设置', '同步')
@@ -240,6 +301,7 @@ export default {
}
} finally {
this.isGitResetLoading = false
this.updateGit()
}
})
this.$st.sendEv('爬虫详情', 'Git 设置', '点击重置')
@@ -255,13 +317,24 @@ export default {
document.execCommand('copy')
this.$message.success(this.$t('SSH Public Key is copied to the clipboard'))
this.$st.sendEv('爬虫详情', 'Git 设置', '拷贝 SSH 公钥')
},
async getCommits () {
const res = await this.$request.get('/git/commits', { spider_id: this.spiderForm._id })
this.commits = res.data.data.map(d => {
d.ts = dayjs(d.ts).format('YYYY-MM-DD HH:mm:ss')
return d
})
},
async updateGit () {
this.getCommits()
}
},
async created () {
if (this.spiderForm.git_url) {
this.onGitUrlChange()
}
await this.getSshPublicKey()
this.getSshPublicKey()
this.getCommits()
}
}
</script>
@@ -318,4 +391,27 @@ export default {
text-align: right;
margin-top: 10px;
}
.git-settings .log {
height: calc(100vh - 280px);
overflow: auto;
}
.git-settings .log .commit {
border-top: 1px solid rgb(244, 244, 245);
padding: 10px 0;
}
.git-settings .log .commit .row {
display: flex;
justify-content: space-between;
}
.git-settings .log .el-timeline-item {
/*cursor: pointer;*/
}
.git-settings .log .commit .row .tags .el-tag {
margin-right: 5px;
}
</style>

View File

@@ -22,7 +22,7 @@
<el-tab-pane :label="$t('Overview')" name="overview">
<spider-overview/>
</el-tab-pane>
<el-tab-pane v-if="isGit" :label="$t('Git Settings')" name="git-settings">
<el-tab-pane v-if="isGit" :label="$t('Git')" name="git-settings">
<git-settings/>
</el-tab-pane>
<el-tab-pane v-if="isScrapy" :label="$t('Scrapy Settings')" name="scrapy-settings">