git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

added spider market

Browse Source
This commit is contained in:
marvzhang
2020-07-13 16:56:11 +08:00
parent c54be2a83e
commit a3d82c40c0
10 changed files with 475 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend/conf/config.yml
View File

@@ -59,4 +59,8 @@ notification:
senderIdentity: ''
smtp:
user: ''
password: ''
password: ''
repo:
apiUrl: "https://center.crawlab.cn/api"
# apiUrl: "http://localhost:8002"
ossUrl: "https://repo.crawlab.cn"

6
backend/main.go
View File

@@ -317,6 +317,12 @@ func main() {
authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥
authGroup.GET("/git/commits", routes.GetGitCommits) // 获取 Git Commits
authGroup.POST("/git/checkout", routes.PostGitCheckout) // 获取 Git Commits
// 爬虫市场 / 仓库
{
authGroup.GET("/repos", routes.GetRepoList) // 获取仓库列表
authGroup.GET("/repos/sub-dir", routes.GetRepoSubDirList) // 获取仓库子目录
authGroup.POST("/repos/download", routes.DownloadRepo) // 下载仓库
}
}
}

8
backend/routes/base.go
View File

@@ -14,3 +14,11 @@ type ListResponse struct {
Data interface{} `json:"data"`
Error string `json:"error"`
}
type ListRequestData struct {
PageNum int `form:"page_num" json:"page_num"`
PageSize int `form:"page_size" json:"page_size"`
SortKey string `form:"sort_key" json:"sort_key"`
Status string `form:"status" json:"status"`
Keyword string `form:"keyword" json:"keyword"`
}

81
backend/routes/repos.go Normal file
View File

@@ -0,0 +1,81 @@
package routes
import (
"crawlab/services"
"fmt"
"github.com/apex/log"
"github.com/gin-gonic/gin"
"github.com/imroc/req"
"github.com/spf13/viper"
"net/http"
"runtime/debug"
)
func GetRepoList(c *gin.Context) {
var data ListRequestData
if err := c.ShouldBindQuery(&data); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
params := req.Param{
"page_num": data.PageNum,
"page_size": data.PageSize,
"keyword": data.Keyword,
"sort_key": data.SortKey,
}
res, err := req.Get(fmt.Sprintf("%s/public/repos", viper.GetString("repo.apiUrl")), params)
if err != nil {
log.Error("get repos error: " + err.Error())
debug.PrintStack()
HandleError(http.StatusInternalServerError, c, err)
return
}
var resJson interface{}
if err := res.ToJSON(&resJson); err != nil {
log.Error("to json error: " + err.Error())
debug.PrintStack()
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, resJson)
}
func GetRepoSubDirList(c *gin.Context) {
params := req.Param{
"full_name": c.Query("full_name"),
}
res, err := req.Get(fmt.Sprintf("%s/public/repos/sub-dir", viper.GetString("repo.apiUrl")), params)
if err != nil {
log.Error("get repo sub-dir error: " + err.Error())
debug.PrintStack()
HandleError(http.StatusInternalServerError, c, err)
return
}
var resJson interface{}
if err := res.ToJSON(&resJson); err != nil {
log.Error("to json error: " + err.Error())
debug.PrintStack()
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, resJson)
}
func DownloadRepo(c *gin.Context) {
type RequestData struct {
FullName string `json:"full_name"`
}
var reqData RequestData
if err := c.ShouldBindJSON(&reqData); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
if err := services.DownloadRepo(reqData.FullName, services.GetCurrentUserId(c)); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}

82
backend/services/repo.go Normal file
View File

@@ -0,0 +1,82 @@
package services
import (
"crawlab/constants"
"crawlab/model"
"crawlab/utils"
"fmt"
"github.com/apex/log"
"github.com/globalsign/mgo/bson"
"github.com/imroc/req"
uuid "github.com/satori/go.uuid"
"github.com/spf13/viper"
"path"
"path/filepath"
"runtime/debug"
"strings"
)
func DownloadRepo(fullName string, userId bson.ObjectId) (err error) {
// 下载 zip 文件
url := fmt.Sprintf("%s/%s.zip", viper.GetString("repo.ossUrl"), fullName)
progress := func(current, total int64) {
fmt.Println(float32(current)/float32(total)*100, "%")
}
res, err := req.Get(url, req.DownloadProgress(progress))
if err != nil {
log.Errorf("download repo error: " + err.Error())
debug.PrintStack()
return err
}
spiderName := strings.Replace(fullName, "/", "_", -1)
randomId := uuid.NewV4()
tmpFilePath := filepath.Join(viper.GetString("other.tmppath"), spiderName+"."+randomId.String()+".zip")
if err := res.ToFile(tmpFilePath); err != nil {
log.Errorf("to file error: " + err.Error())
debug.PrintStack()
return err
}
// 解压 zip 文件
tmpFile := utils.OpenFile(tmpFilePath)
if err := utils.DeCompress(tmpFile, viper.GetString("other.tmppath")); err != nil {
log.Errorf("de-compress error: " + err.Error())
debug.PrintStack()
return err
}
// 拷贝文件
spiderPath := path.Join(viper.GetString("spider.path"), spiderName)
srcDirPath := fmt.Sprintf("%s/data/github.com/%s", viper.GetString("other.tmppath"), fullName)
if err := utils.CopyDir(srcDirPath, spiderPath); err != nil {
log.Errorf("copy error: " + err.Error())
debug.PrintStack()
return err
}
// 创建爬虫
spider := model.Spider{
Id: bson.NewObjectId(),
Name: spiderName,
DisplayName: spiderName,
Type: constants.Customized,
Src: spiderPath,
ProjectId: bson.ObjectIdHex(constants.ObjectIdNull),
FileId: bson.ObjectIdHex(constants.ObjectIdNull),
UserId: userId,
}
if err := spider.Add(); err != nil {
log.Error("add spider error: " + err.Error())
debug.PrintStack()
return err
}
// 上传爬虫
if err := UploadSpiderToGridFsFromMaster(spider); err != nil {
log.Error("upload spider error: " + err.Error())
debug.PrintStack()
return err
}
return nil
}

4
backend/services/spider.go
View File

@@ -239,7 +239,9 @@ func PublishSpider(spider model.Spider) {
}
// 安装依赖
go spiderSync.InstallDeps()
if viper.GetString("setting.autoInstall") == "Y" {
go spiderSync.InstallDeps()
}
//目录不存在，则直接下载
path := filepath.Join(viper.GetString("spider.path"), spider.Name)

12
frontend/src/i18n/zh.js
View File

@@ -13,6 +13,7 @@ export default {
'Sites': '网站',
'Setting': '设置',
'Project': '项目',
'Spider Market': '爬虫市场',
// 标签
'Overview': '概览',
@@ -518,6 +519,15 @@ export default {
'Year': '年',
'Years': '年',
// 爬虫市场
'Search Keyword': '搜索关键词',
'Sort': '排序',
'Default Sort': '默认排序',
'Most Stars': '最多 Stars',
'Most Forks': '最多 Forks',
'Latest Pushed': '最近提交',
'Pushed At': '提交时间',
// 全局
'Related Documentation': '相关文档',
'Click to view related Documentation': '点击查看相关文档',
@@ -647,6 +657,8 @@ export default {
'Are you sure to add an API token?': '确认创建 API Token?',
'Are you sure to delete this API token?': '确认删除该 API Token?',
'Please enter Web Hook URL': '请输入 Web Hook URL',
'Are you sure to download this spider?': '您确定要下载该爬虫?',
'Downloaded successfully': '下载成功',
// 其他
'Star crawlab-team/crawlab on GitHub': '在 GitHub 上为 Crawlab 加星吧'

22
frontend/src/router/index.js
View File

@@ -181,6 +181,25 @@ export const constantRouterMap = [
}
]
},
{
path: '/repos',
component: Layout,
meta: {
title: 'Spider Market',
icon: 'fa fa-cloud'
},
children: [
{
path: '',
name: 'RepoList',
component: () => import('../views/repo/RepoList'),
meta: {
title: 'Spider Market',
icon: 'fa fa-cloud'
}
}
]
},
{
path: '/disclaimer',
component: Layout,
@@ -243,8 +262,7 @@ export const constantRouterMap = [
component: Layout,
meta: {
title: 'User',
icon: 'fa fa-users',
isNew: true
icon: 'fa fa-users'
},
children: [
{

2
frontend/src/utils/request.js
View File

@@ -82,7 +82,7 @@ if (!CRAWLAB_API_ADDRESS.match('CRAWLAB_API_ADDRESS')) {
const service = axios.create({
baseURL: baseUrl, // url = base url + request url
// withCredentials: true, // send cookies when cross-domain requests
timeout: 5000 // request timeout
timeout: 15000 // request timeout
})
// request interceptor
service.interceptors.request.use(

257
frontend/src/views/repo/RepoList.vue Normal file
View File

@@ -0,0 +1,257 @@
<template>
<div class="app-container repo-list">
<el-card>
<div class="filter">
<el-form inline>
<el-form-item :label="$t('Search Keyword')">
<el-input
v-model="keyword"
size="small"
:placeholder="$t('Search Keyword')"
@keyup.enter.native="getRepos"
/>
</el-form-item>
<el-form-item :label="$t('Sort')">
<el-select
v-model="sortKey"
size="small"
>
<el-option :label="$t('Default Sort')" value=""/>
<el-option :label="$t('Most Stars')" value="stars"/>
<el-option :label="$t('Most Forks')" value="forks"/>
<el-option :label="$t('Latest Pushed')" value="pushed_at"/>
</el-select>
</el-form-item>
<el-form-item>
<el-button type="success" size="small" @click="getRepos">{{ $t('Search') }}</el-button>
</el-form-item>
</el-form>
</div>
<el-table
ref="table"
:data="repos"
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
:row-class-name="getRowClassName"
row-key="id"
border
v-loading="isLoading"
@expand-change="onRowExpand"
>
<el-table-column type="expand">
<template slot-scope="scope">
<ul class="sub-dir-list">
<li
v-for="sub in getSubDirList(scope.row)"
:key="sub.full_name"
class="sub-dir-item"
>
<div class="sub-dir-title">
{{ sub.name }}
</div>
<div class="action">
<el-tooltip :content="$t('Download')" placement="top">
<el-button
type="primary"
icon="fa fa-download"
size="mini"
@click="onDownload(scope.row, sub.full_name, $event)"
v-loading="scope.row.loading"
/>
</el-tooltip>
</div>
</li>
</ul>
</template>
</el-table-column>
<el-table-column
:label="$t('Name')"
prop="full_name"
width="300px"
/>
<el-table-column
:label="$t('Description')"
prop="description"
min-width="500px"
/>
<el-table-column
label="Stars"
prop="stars"
width="80px"
align="right"
/>
<el-table-column
label="Forks"
prop="forks"
width="80px"
align="right"
/>
<el-table-column
:label="$t('Pushed At')"
prop="pushed_at"
width="150px"
>
<template slot-scope="scope">
{{ getTime(scope.row.pushed_at) }}
</template>
</el-table-column>
<el-table-column
:label="$t('Action')"
width="120px"
>
<template slot-scope="scope">
<el-tooltip :content="$t('Download')" placement="top">
<el-button
type="primary"
icon="fa fa-download"
size="mini"
@click="onDownload(scope.row, null, $event)"
v-loading="scope.row.loading"
/>
</el-tooltip>
</template>
</el-table-column>
</el-table>
<div class="pagination">
<el-pagination
:current-page.sync="pageNum"
:page-sizes="[10, 20, 50, 100]"
:page-size.sync="pageSize"
layout="sizes, prev, pager, next"
:total="total"
/>
</div>
</el-card>
</div>
</template>
<script>
import dayjs from 'dayjs'
export default {
name: 'RepoList',
data() {
return {
repos: [],
total: 0,
pageNum: 1,
pageSize: 20,
keyword: '',
sortKey: '',
isLoading: false,
subDirCache: {}
}
},
watch: {
pageNum() {
this.getRepos()
},
pageSize() {
this.getRepos()
},
sortKey() {
this.getRepos()
}
},
methods: {
async getRepos() {
this.isLoading = true
try {
const res = await this.$request.get('/repos', {
page_num: this.pageNum,
page_size: this.pageSize,
keyword: this.keyword,
sort_key: this.sortKey
})
this.repos = res.data.data
this.total = res.data.total
} finally {
this.isLoading = false
}
},
getTime(t) {
return dayjs(t).format('YYYY-MM-DD HH:mm:ss')
},
onDownload(row, fullName, ev) {
ev.stopPropagation()
this.$confirm(this.$t('Are you sure to download this spider?'), this.$t('Notification'), {
confirmButtonText: this.$t('Confirm'),
cancelButtonText: this.$t('Cancel'),
type: 'warning'
}).then(async() => {
this.$set(row, 'loading', true)
try {
await this.download(fullName || row.full_name)
this.$message.success('Downloaded successfully')
} finally {
this.$set(row, 'loading', false)
}
})
},
async download(fullName) {
this.$request.post('/repos/download', {
full_name: fullName
})
},
getRowClassName({ row }) {
return row.is_sub_dir ? '' : 'non-expandable'
},
async onRowExpand(row, expandedRows) {
if (!this.subDirCache[row.full_name]) {
const res = await this.$request.get('/repos/sub-dir', {
full_name: row.full_name
})
this.$set(this.subDirCache, row.full_name, res.data.data)
}
},
getSubDirList(row) {
if (!this.subDirCache[row.full_name]) return []
return this.subDirCache[row.full_name].map(n => {
return {
name: n,
full_name: `${row.full_name}/${n}`
}
})
}
},
async created() {
await this.getRepos()
}
}
</script>
<style scoped>
.el-table .el-button {
padding: 7px;
}
.el-table >>> .non-expandable .el-table__expand-icon {
display: none;
}
.el-table .sub-dir-list {
list-style: none;
font-size: 12px;
margin: 0;
padding: 0;
}
.el-table .sub-dir-list .sub-dir-item {
padding: 6px 0 6px 60px;
border-bottom: 1px dashed #EBEEF5;
display: flex;
justify-content: space-between;
}
.el-table .sub-dir-list .sub-dir-item:last-child {
border-bottom: none;
}
.el-table .sub-dir-list .sub-dir-item .sub-dir-title {
line-height: 29px;
}
.el-table .sub-dir-list .sub-dir-item .action {
width: 120px;
padding: 0 12px;
}
</style>