git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #581 from crawlab-team/develop

Browse Source 
Develop
This commit is contained in:
Marvin Zhang
2020-02-22 11:07:30 +08:00
committed by GitHub
parent 319148968d 57feee8268
commit 5de992f557
11 changed files with 941 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
backend/entity/spider.go
View File

@@ -6,7 +6,12 @@ type SpiderType struct {
}
type ScrapySettingParam struct {
Key string
Value interface{}
Type string
Key string `json:"key"`
Value interface{} `json:"value"`
Type string `json:"type"`
}
type ScrapyItem struct {
Name string `json:"name"`
Fields []string `json:"fields"`
}

54
backend/main.go
View File

@@ -155,31 +155,35 @@ func main() {
}
// 爬虫
{
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID）
authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表
authGroup.PUT("/spiders/:id/scrapy/spiders", routes.PutSpiderScrapySpiders) // Scrapy 爬虫创建爬虫
authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings) // Scrapy 爬虫设置
authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings) // Scrapy 爬虫修改设置
authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步
authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID）
authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表
authGroup.PUT("/spiders/:id/scrapy/spiders", routes.PutSpiderScrapySpiders) // Scrapy 爬虫创建爬虫
authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings) // Scrapy 爬虫设置
authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings) // Scrapy 爬虫修改设置
authGroup.GET("/spiders/:id/scrapy/items", routes.GetSpiderScrapyItems) // Scrapy 爬虫 items
authGroup.POST("/spiders/:id/scrapy/items", routes.PostSpiderScrapyItems) // Scrapy 爬虫修改 items
authGroup.GET("/spiders/:id/scrapy/pipelines", routes.GetSpiderScrapyPipelines) // Scrapy 爬虫 pipelines
authGroup.GET("/spiders/:id/scrapy/spider/filepath", routes.GetSpiderScrapySpiderFilepath) // Scrapy 爬虫 pipelines
authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步
authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置
}
// 可配置爬虫
{

143
backend/routes/spider.go
View File

@@ -187,19 +187,33 @@ func PutSpider(c *gin.Context) {
// 将FileId置空
spider.FileId = bson.ObjectIdHex(constants.ObjectIdNull)
// 创建爬虫目录
// 爬虫目录
spiderDir := filepath.Join(viper.GetString("spider.path"), spider.Name)
// 赋值到爬虫实例
spider.Src = spiderDir
// 移除已有爬虫目录
if utils.Exists(spiderDir) {
if err := os.RemoveAll(spiderDir); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
}
// 生成爬虫目录
if err := os.MkdirAll(spiderDir, 0777); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
spider.Src = spiderDir
// 如果为 Scrapy 项目，生成 Scrapy 项目
if spider.IsScrapy {
if err := services.CreateScrapyProject(spider); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
}
// 添加爬虫到数据库
if err := spider.Add(); err != nil {
@@ -960,8 +974,9 @@ func GetSpiderScrapySpiders(c *gin.Context) {
func PutSpiderScrapySpiders(c *gin.Context) {
type ReqBody struct {
Name string `json:"name"`
Domain string `json:"domain"`
Name string `json:"name"`
Domain string `json:"domain"`
Template string `json:"template"`
}
id := c.Param("id")
@@ -983,7 +998,7 @@ func PutSpiderScrapySpiders(c *gin.Context) {
return
}
if err := services.CreateScrapySpider(spider, reqBody.Name, reqBody.Domain); err != nil {
if err := services.CreateScrapySpider(spider, reqBody.Name, reqBody.Domain, reqBody.Template); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
@@ -1052,6 +1067,124 @@ func PostSpiderScrapySettings(c *gin.Context) {
})
}
func GetSpiderScrapyItems(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid")
return
}
spider, err := model.GetSpider(bson.ObjectIdHex(id))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
data, err := services.GetScrapyItems(spider)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: data,
})
}
func PostSpiderScrapyItems(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid")
return
}
var reqData []entity.ScrapyItem
if err := c.ShouldBindJSON(&reqData); err != nil {
HandleErrorF(http.StatusBadRequest, c, "invalid request")
return
}
spider, err := model.GetSpider(bson.ObjectIdHex(id))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
if err := services.SaveScrapyItems(spider, reqData); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}
func GetSpiderScrapyPipelines(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid")
return
}
spider, err := model.GetSpider(bson.ObjectIdHex(id))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
data, err := services.GetScrapyPipelines(spider)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: data,
})
}
func GetSpiderScrapySpiderFilepath(c *gin.Context) {
id := c.Param("id")
spiderName := c.Query("spider_name")
if spiderName == "" {
HandleErrorF(http.StatusBadRequest, c, "spider_name is empty")
return
}
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid")
return
}
spider, err := model.GetSpider(bson.ObjectIdHex(id))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
data, err := services.GetScrapySpiderFilepath(spider, spiderName)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: data,
})
}
func PostSpiderSyncGit(c *gin.Context) {
id := c.Param("id")

135
backend/services/scrapy.go
View File

@@ -135,16 +135,147 @@ func SaveScrapySettings(s model.Spider, settingsData []entity.ScrapySettingParam
return
}
func CreateScrapySpider(s model.Spider, name string, domain string) (err error) {
func GetScrapyItems(s model.Spider) (res []map[string]interface{}, err error) {
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd := exec.Command("scrapy", "genspider", name, domain)
cmd := exec.Command("crawlab", "items")
cmd.Dir = s.Src
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Errorf(err.Error())
log.Errorf(stderr.String())
debug.PrintStack()
return res, err
}
if err := json.Unmarshal([]byte(stdout.String()), &res); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return res, err
}
return res, nil
}
func SaveScrapyItems(s model.Spider, itemsData []entity.ScrapyItem) (err error) {
// 读取 scrapy.cfg
cfg, err := goconfig.LoadConfigFile(path.Join(s.Src, "scrapy.cfg"))
if err != nil {
return
}
modName, err := cfg.GetValue("settings", "default")
if err != nil {
return
}
// 定位到 settings.py 文件
arr := strings.Split(modName, ".")
dirName := arr[0]
fileName := "items"
filePath := fmt.Sprintf("%s/%s/%s.py", s.Src, dirName, fileName)
// 生成文件内容
content := ""
content += "import scrapy\n"
content += "\n\n"
for _, item := range itemsData {
content += fmt.Sprintf("class %s(scrapy.Item):\n", item.Name)
for _, field := range item.Fields {
content += fmt.Sprintf(" %s = scrapy.Field()\n", field)
}
content += "\n\n"
}
// 写到 settings.py
if err := ioutil.WriteFile(filePath, []byte(content), os.ModePerm); err != nil {
return err
}
// 同步到GridFS
if err := UploadSpiderToGridFsFromMaster(s); err != nil {
return err
}
return
}
func GetScrapyPipelines(s model.Spider) (res []string, err error) {
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd := exec.Command("crawlab", "pipelines")
cmd.Dir = s.Src
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Errorf(err.Error())
log.Errorf(stderr.String())
debug.PrintStack()
return res, err
}
if err := json.Unmarshal([]byte(stdout.String()), &res); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return res, err
}
return res, nil
}
func GetScrapySpiderFilepath(s model.Spider, spiderName string) (res string, err error) {
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd := exec.Command("crawlab", "find_spider_filepath", "-n", spiderName)
cmd.Dir = s.Src
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Errorf(err.Error())
log.Errorf(stderr.String())
debug.PrintStack()
return res, err
}
res = strings.Replace(stdout.String(), "\n", "", 1)
return res, nil
}
func CreateScrapySpider(s model.Spider, name string, domain string, template string) (err error) {
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd := exec.Command("scrapy", "genspider", name, domain, "-t", template)
cmd.Dir = s.Src
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Errorf(err.Error())
log.Errorf("stdout: " + stdout.String())
log.Errorf("stderr: " + stderr.String())
debug.PrintStack()
return err
}
return
}
func CreateScrapyProject(s model.Spider) (err error) {
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd := exec.Command("scrapy", "startproject", s.Name, s.Src)
cmd.Dir = s.Src
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Errorf(err.Error())
log.Errorf("stdout: " + stdout.String())
log.Errorf("stderr: " + stderr.String())
debug.PrintStack()
return err
}

24
frontend/src/components/File/FileList.vue
View File

@@ -409,6 +409,30 @@ export default {
this.isShowDelete = false
this.showFile = false
this.$st.sendEv('爬虫详情', '文件', '删除')
},
clickSpider (filepath) {
const node = this.$refs['tree'].getNode(filepath)
const data = node.data
this.onFileClick(data)
node.parent.expanded = true
node.parent.parent.expanded = true
},
clickPipeline () {
const filename = 'pipelines.py'
for (let i = 0; i < this.computedFileTree.length; i++) {
const dataLv1 = this.computedFileTree[i]
const nodeLv1 = this.$refs['tree'].getNode(dataLv1.path)
if (dataLv1.is_dir) {
for (let j = 0; j < dataLv1.children.length; j++) {
const dataLv2 = dataLv1.children[j]
if (dataLv2.path.match(filename)) {
this.onFileClick(dataLv2)
nodeLv1.expanded = true
return
}
}
}
}
}
},
async created () {

681
frontend/src/components/Scrapy/SpiderScrapy.vue
View File

@@ -1,5 +1,6 @@
<template>
<div class="spider-scrapy">
<!--parameter edit-->
<el-dialog
:title="$t('Parameter Edit')"
:visible="dialogVisible"
@@ -12,7 +13,7 @@
type="primary"
size="small"
icon="el-icon-plus"
@click="onActiveParamAdd"
@click="onSettingsActiveParamAdd"
>
{{$t('Add')}}
</el-button>
@@ -57,19 +58,21 @@
size="mini"
icon="el-icon-delete"
circle
@click="onActiveParamRemove(scope.$index)"
@click="onSettingsActiveParamRemove(scope.$index)"
/>
</template>
</el-table-column>
</el-table>
<template slot="footer">
<el-button type="plain" size="small" @click="onCloseDialog">{{$t('Cancel')}}</el-button>
<el-button type="primary" size="small" @click="onConfirm">
<el-button type="primary" size="small" @click="onSettingsConfirm">
{{$t('Confirm')}}
</el-button>
</template>
</el-dialog>
<!--./parameter edit-->
<!--add scrapy spider-->
<el-dialog
:title="$t('Add Scrapy Spider')"
:visible.sync="isAddSpiderVisible"
@@ -87,6 +90,14 @@
<el-form-item :label="$t('Domain')" prop="domain" required>
<el-input v-model="addSpiderForm.domain" :placeholder="$t('Domain')"/>
</el-form-item>
<el-form-item :label="$t('Template')" prop="template" required>
<el-select v-model="addSpiderForm.template" :placeholder="$t('Template')">
<el-option value="basic" label="basic"/>
<el-option value="crawl" label="crawl"/>
<el-option value="csvfeed" label="csvfeed"/>
<el-option value="xmlfeed" label="xmlfeed"/>
</el-select>
</el-form-item>
</el-form>
<template slot="footer">
<el-button type="plain" size="small" @click="isAddSpiderVisible = false">{{$t('Cancel')}}</el-button>
@@ -101,139 +112,255 @@
</el-button>
</template>
</el-dialog>
<!--./add scrapy spider-->
<div class="spiders">
<h3 class="title">{{$t('Scrapy Spiders')}}</h3>
<div class="action-wrapper">
<el-button
type="primary"
size="small"
icon="el-icon-plus"
@click="onAddSpider"
>
{{$t('Add Spider')}}
</el-button>
</div>
<ul class="spider-list">
<li
v-for="s in spiderForm.spider_names"
:key="s"
class="spider-item"
>
<i class="el-icon-caret-right"></i>
{{s}}
</li>
</ul>
</div>
<div class="settings">
<h3 class="title">{{$t('Settings')}}</h3>
<div class="top-action-wrapper">
<el-button
type="primary"
size="small"
icon="el-icon-plus"
@click="onAdd"
>
{{$t('Add')}}
</el-button>
<el-button size="small" type="success" @click="onSave" icon="el-icon-check">
{{$t('Save')}}
</el-button>
</div>
<el-table
:data="spiderScrapySettings"
border
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
max-height="calc(100vh - 240px)"
>
<el-table-column
:label="$t('Variable Name')"
width="240px"
>
<template slot-scope="scope">
<el-autocomplete
v-model="scope.row.key"
size="small"
suffix-icon="el-icon-edit"
:fetch-suggestions="settingsKeysFetchSuggestions"
/>
</template>
</el-table-column>
<el-table-column
:label="$t('Variable Type')"
width="120px"
>
<template slot-scope="scope">
<el-select v-model="scope.row.type" size="small" @change="onParamTypeChange(scope.row)">
<el-option value="string" :label="$t('String')"/>
<el-option value="number" :label="$t('Number')"/>
<el-option value="boolean" :label="$t('Boolean')"/>
<el-option value="array" :label="$t('Array/List')"/>
<el-option value="object" :label="$t('Object/Dict')"/>
</el-select>
</template>
</el-table-column>
<el-table-column
:label="$t('Variable Value')"
width="calc(100% - 150px)"
>
<template slot-scope="scope">
<el-input
v-if="scope.row.type === 'string'"
v-model="scope.row.value"
size="small"
suffix-icon="el-icon-edit"
/>
<el-input
v-else-if="scope.row.type === 'number'"
type="number"
v-model="scope.row.value"
size="small"
suffix-icon="el-icon-edit"
@change="scope.row.value = Number(scope.row.value)"
/>
<div
v-else-if="scope.row.type === 'boolean'"
style="margin-left: 10px"
>
<el-switch
v-model="scope.row.value"
size="small"
active-color="#67C23A"
/>
</div>
<div
v-else
style="margin-left: 10px;font-size: 12px"
>
{{JSON.stringify(scope.row.value)}}
<el-button
type="warning"
size="mini"
icon="el-icon-edit"
style="margin-left: 10px"
@click="onEditParam(scope.row, scope.$index)"
/>
</div>
</template>
</el-table-column>
<el-table-column
:label="$t('Action')"
width="60px"
align="center"
>
<template slot-scope="scope">
<el-tabs
v-model="activeTabName"
>
<!--settings-->
<el-tab-pane :label="$t('Settings')" name="settings">
<div class="settings">
<div class="top-action-wrapper">
<el-button
type="danger"
size="mini"
icon="el-icon-delete"
circle
@click="onRemove(scope.$index)"
/>
</template>
</el-table-column>
</el-table>
</div>
type="primary"
size="small"
icon="el-icon-plus"
@click="onSettingsAdd"
>
{{$t('Add Variable')}}
</el-button>
<el-button size="small" type="success" @click="onSettingsSave" icon="el-icon-check">
{{$t('Save')}}
</el-button>
</div>
<el-table
:data="spiderScrapySettings"
border
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
max-height="calc(100vh - 240px)"
>
<el-table-column
:label="$t('Variable Name')"
width="240px"
>
<template slot-scope="scope">
<el-autocomplete
v-model="scope.row.key"
size="small"
suffix-icon="el-icon-edit"
:fetch-suggestions="settingsKeysFetchSuggestions"
/>
</template>
</el-table-column>
<el-table-column
:label="$t('Variable Type')"
width="120px"
>
<template slot-scope="scope">
<el-select v-model="scope.row.type" size="small" @change="onSettingsParamTypeChange(scope.row)">
<el-option value="string" :label="$t('String')"/>
<el-option value="number" :label="$t('Number')"/>
<el-option value="boolean" :label="$t('Boolean')"/>
<el-option value="array" :label="$t('Array/List')"/>
<el-option value="object" :label="$t('Object/Dict')"/>
</el-select>
</template>
</el-table-column>
<el-table-column
:label="$t('Variable Value')"
width="calc(100% - 150px)"
>
<template slot-scope="scope">
<el-input
v-if="scope.row.type === 'string'"
v-model="scope.row.value"
size="small"
suffix-icon="el-icon-edit"
/>
<el-input
v-else-if="scope.row.type === 'number'"
type="number"
v-model="scope.row.value"
size="small"
suffix-icon="el-icon-edit"
@change="scope.row.value = Number(scope.row.value)"
/>
<div
v-else-if="scope.row.type === 'boolean'"
style="margin-left: 10px"
>
<el-switch
v-model="scope.row.value"
size="small"
active-color="#67C23A"
/>
</div>
<div
v-else
style="margin-left: 10px;font-size: 12px"
>
{{JSON.stringify(scope.row.value)}}
<el-button
type="warning"
size="mini"
icon="el-icon-edit"
style="margin-left: 10px"
@click="onSettingsEditParam(scope.row, scope.$index)"
/>
</div>
</template>
</el-table-column>
<el-table-column
:label="$t('Action')"
width="60px"
align="center"
>
<template slot-scope="scope">
<el-button
type="danger"
size="mini"
icon="el-icon-delete"
circle
@click="onSettingsRemove(scope.$index)"
/>
</template>
</el-table-column>
</el-table>
</div>
</el-tab-pane>
<!--./settings-->
<!--spiders-->
<el-tab-pane :label="$t('Spiders')" name="spiders">
<div class="spiders">
<div class="action-wrapper">
<el-button
type="primary"
size="small"
icon="el-icon-plus"
@click="onAddSpider"
>
{{$t('Add Spider')}}
</el-button>
</div>
<ul class="list">
<li
v-for="s in spiderForm.spider_names"
:key="s"
class="item"
@click="onClickSpider(s)"
>
<i class="el-icon-star-on"></i>
{{s}}
<i v-if="loadingDict[s]" class="el-icon-loading"></i>
</li>
</ul>
</div>
</el-tab-pane>
<!--./spiders-->
<!--items-->
<el-tab-pane label="Items" name="items">
<div class="items">
<div class="action-wrapper">
<el-button
type="primary"
size="small"
icon="el-icon-plus"
@click="onAddItem"
>
{{$t('Add Item')}}
</el-button>
<el-button size="small" type="success" @click="onItemsSave" icon="el-icon-check">
{{$t('Save')}}
</el-button>
</div>
<el-tree
:data="spiderScrapyItems"
default-expand-all
>
<span class="custom-tree-node" :class="`level-${data.level}`" slot-scope="{ node, data }">
<template v-if="data.level === 1">
<span v-if="!node.isEdit" class="label" @click="onItemLabelEdit(node, data, $event)">
<i class="el-icon-star-on"></i>
{{ data.label }}
<i class="el-icon-edit"></i>
</span>
<el-input
v-else
:ref="`el-input-${data.id}`"
:placeholder="$t('Item Name')"
v-model="data.name"
size="mini"
@change="onItemChange(node, data, $event)"
@blur="$set(node, 'isEdit', false)"
/>
<span>
<el-button
type="primary"
size="mini"
icon="el-icon-plus"
@click="onAddItemField(data, $event)">
{{$t('Add Field')}}
</el-button>
<el-button
type="danger"
size="mini"
icon="el-icon-delete"
@click="removeItem(data, $event)">
{{$t('Remove')}}
</el-button>
</span>
</template>
<template v-if="data.level === 2">
<span v-if="!node.isEdit" class="label" @click="onItemLabelEdit(node, data, $event)">
<i class="el-icon-arrow-right"></i>
{{ node.label }}
<i class="el-icon-edit"></i>
</span>
<el-input
v-else
:ref="`el-input-${data.id}`"
:placeholder="$t('Field Name')"
v-model="data.name"
size="mini"
@change="onItemFieldChange(node, data, $event)"
@blur="node.isEdit = false"
/>
<span>
<el-button
type="danger"
size="mini"
icon="el-icon-delete"
@click="onRemoveItemField(node, data, $event)">
{{$t('Remove')}}
</el-button>
</span>
</template>
</span>
</el-tree>
</div>
</el-tab-pane>
<!--./items-->
<!--pipelines-->
<el-tab-pane label="Pipelines" name="pipelines">
<div class="pipelines">
<ul class="list">
<li
v-for="s in spiderScrapyPipelines"
:key="s"
class="item"
@click="$emit('click-pipeline')"
>
<i class="el-icon-star-on"></i>
{{s}}
</li>
</ul>
</div>
</el-tab-pane>
<!--./pipelines-->
</el-tabs>
</div>
</template>
@@ -247,7 +374,9 @@ export default {
computed: {
...mapState('spider', [
'spiderForm',
'spiderScrapySettings'
'spiderScrapySettings',
'spiderScrapyItems',
'spiderScrapyPipelines'
]),
activeParamData () {
if (this.activeParam.type === 'array') {
@@ -273,9 +402,12 @@ export default {
isAddSpiderVisible: false,
addSpiderForm: {
name: '',
domain: ''
domain: '',
template: 'basic'
},
isAddSpiderLoading: false
isAddSpiderLoading: false,
activeTabName: 'settings',
loadingDict: {}
}
},
methods: {
@@ -285,7 +417,7 @@ export default {
onCloseDialog () {
this.dialogVisible = false
},
onConfirm () {
onSettingsConfirm () {
if (this.activeParam.type === 'array') {
this.activeParam.value = this.activeParamData.map(d => d.value)
} else if (this.activeParam.type === 'object') {
@@ -297,22 +429,22 @@ export default {
}
this.$set(this.spiderScrapySettings, this.activeParamIndex, JSON.parse(JSON.stringify(this.activeParam)))
this.dialogVisible = false
this.$st('爬虫详情', 'Scrapy 设置', '确认编辑参数')
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '确认编辑参数')
},
onEditParam (row, index) {
onSettingsEditParam (row, index) {
this.activeParam = JSON.parse(JSON.stringify(row))
this.activeParamIndex = index
this.onOpenDialog()
this.$st('爬虫详情', 'Scrapy 设置', '点击编辑参数')
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '点击编辑参数')
},
async onSave () {
async onSettingsSave () {
const res = await this.$store.dispatch('spider/saveSpiderScrapySettings', this.$route.params.id)
if (!res.data.error) {
this.$message.success(this.$t('Saved successfully'))
}
this.$st('爬虫详情', 'Scrapy 设置', '保存设置')
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '保存设置')
},
onAdd () {
onSettingsAdd () {
const data = JSON.parse(JSON.stringify(this.spiderScrapySettings))
data.push({
key: '',
@@ -320,15 +452,15 @@ export default {
type: 'string'
})
this.$store.commit('spider/SET_SPIDER_SCRAPY_SETTINGS', data)
this.$st('爬虫详情', 'Scrapy 设置', '添加参数')
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '添加参数')
},
onRemove (index) {
onSettingsRemove (index) {
const data = JSON.parse(JSON.stringify(this.spiderScrapySettings))
data.splice(index, 1)
this.$store.commit('spider/SET_SPIDER_SCRAPY_SETTINGS', data)
this.$st('爬虫详情', 'Scrapy 设置', '删除参数')
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '删除参数')
},
onActiveParamAdd () {
onSettingsActiveParamAdd () {
if (this.activeParam.type === 'array') {
this.activeParam.value.push('')
} else if (this.activeParam.type === 'object') {
@@ -337,9 +469,9 @@ export default {
}
this.$set(this.activeParam.value, '', 999)
}
this.$st('爬虫详情', 'Scrapy 设置', '添加参数中参数')
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '添加参数中参数')
},
onActiveParamRemove (index) {
onSettingsActiveParamRemove (index) {
if (this.activeParam.type === 'array') {
this.activeParam.value.splice(index, 1)
} else if (this.activeParam.type === 'object') {
@@ -348,7 +480,7 @@ export default {
delete value[key]
this.$set(this.activeParam, 'value', value)
}
this.$st('爬虫详情', 'Scrapy 设置', '删除参数中参数')
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '删除参数中参数')
},
settingsKeysFetchSuggestions (queryString, cb) {
const data = this.$utils.scrapy.settingParamNames
@@ -367,7 +499,7 @@ export default {
})
cb(data)
},
onParamTypeChange (row) {
onSettingsParamTypeChange (row) {
if (row.type === 'number') {
row.value = Number(row.value)
}
@@ -387,15 +519,138 @@ export default {
this.isAddSpiderLoading = false
await this.$store.dispatch('spider/getSpiderScrapySpiders', this.$route.params.id)
})
this.$st('爬虫详情', 'Scrapy 设置', '确认添加爬虫')
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '确认添加爬虫')
},
onAddSpider () {
this.addSpiderForm = {
name: '',
domain: ''
domain: '',
template: 'basic'
}
this.isAddSpiderVisible = true
this.$st('爬虫详情', 'Scrapy 设置', '添加爬虫')
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '添加爬虫')
},
getMaxItemNodeId () {
let max = 0
this.spiderScrapyItems.forEach(d => {
if (max < d.id) max = d.id
d.children.forEach(f => {
if (max < f.id) max = f.id
})
})
return max
},
onAddItem () {
const maxId = this.getMaxItemNodeId()
this.spiderScrapyItems.push({
id: maxId + 1,
label: `Item_${+new Date()}`,
level: 1,
children: [
{
id: maxId + 2,
level: 2,
label: `field_${+new Date()}`
}
]
})
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '添加Item')
},
removeItem (data, ev) {
ev.stopPropagation()
for (let i = 0; i < this.spiderScrapyItems.length; i++) {
const item = this.spiderScrapyItems[i]
if (item.id === data.id) {
this.spiderScrapyItems.splice(i, 1)
break
}
}
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '删除Item')
},
onAddItemField (data, ev) {
ev.stopPropagation()
for (let i = 0; i < this.spiderScrapyItems.length; i++) {
const item = this.spiderScrapyItems[i]
if (item.id === data.id) {
item.children.push({
id: this.getMaxItemNodeId() + 1,
level: 2,
label: `field_${+new Date()}`
})
break
}
}
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '添加Items字段')
},
onRemoveItemField (node, data, ev) {
ev.stopPropagation()
for (let i = 0; i < this.spiderScrapyItems.length; i++) {
const item = this.spiderScrapyItems[i]
if (item.id === node.parent.data.id) {
for (let j = 0; j < item.children.length; j++) {
const field = item.children[j]
if (field.id === data.id) {
item.children.splice(j, 1)
break
}
}
break
}
}
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '删除Items字段')
},
onItemLabelEdit (node, data, ev) {
ev.stopPropagation()
this.$set(node, 'isEdit', true)
this.$set(data, 'name', node.label)
setTimeout(() => {
this.$refs[`el-input-${data.id}`].focus()
}, 0)
},
onItemChange (node, data, value) {
for (let i = 0; i < this.spiderScrapyItems.length; i++) {
const item = this.spiderScrapyItems[i]
if (item.id === data.id) {
item.label = value
break
}
}
},
onItemFieldChange (node, data, value) {
for (let i = 0; i < this.spiderScrapyItems.length; i++) {
const item = this.spiderScrapyItems[i]
if (item.id === node.parent.data.id) {
for (let j = 0; j < item.children.length; j++) {
const field = item.children[j]
if (field.id === data.id) {
item.children[j].label = value
break
}
}
break
}
}
},
async onItemsSave () {
const res = await this.$store.dispatch('spider/saveSpiderScrapyItems', this.$route.params.id)
if (!res.data.error) {
this.$message.success(this.$t('Saved successfully'))
}
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '保存Items')
},
async onClickSpider (spiderName) {
if (this.loadingDict[spiderName]) return
this.$set(this.loadingDict, spiderName, true)
try {
const res = await this.$store.dispatch('spider/getSpiderScrapySpiderFilepath', {
id: this.$route.params.id,
spiderName
})
this.$emit('click-spider', res.data.data)
} finally {
this.$set(this.loadingDict, spiderName, false)
}
this.$st.sendEv('爬虫详情', 'Scrapy 设置', '点击爬虫')
}
}
}
@@ -407,49 +662,16 @@ export default {
color: #606266;
}
.spiders {
float: left;
display: inline-block;
width: 240px;
height: 100%;
border: 1px solid #DCDFE6;
border-radius: 3px;
padding: 0 10px;
.spider-scrapy >>> .el-tabs__content {
overflow: auto;
}
.spiders .title {
border-bottom: 1px solid #DCDFE6;
padding-bottom: 15px;
}
.spiders .action-wrapper {
margin-bottom: 10px;
text-align: right;
}
.spiders .spider-list {
list-style: none;
padding: 0;
margin: 0;
}
.spiders .spider-list .spider-item {
padding: 10px;
cursor: pointer;
}
.spiders .spider-list .spider-item:hover {
background: #F5F7FA;
.spider-scrapy >>> .el-tab-pane {
height: calc(100vh - 239px);
}
.settings {
margin-left: 20px;
border: 1px solid #DCDFE6;
float: left;
width: calc(100% - 240px - 20px);
height: 100%;
border-radius: 3px;
padding: 0 20px;
width: 100%;
}
.settings .title {
@@ -485,4 +707,77 @@ export default {
.settings >>> .top-action-wrapper .el-button {
margin-left: 10px;
}
.spiders {
width: 100%;
height: auto;
overflow: auto;
}
.spiders .action-wrapper {
text-align: right;
padding-bottom: 10px;
border-bottom: 1px solid #DCDFE6;
}
.pipelines .list,
.spiders .list {
list-style: none;
padding: 0;
margin: 0;
}
.pipelines .list .item,
.spiders .list .item {
font-size: 14px;
padding: 10px;
cursor: pointer;
}
.pipelines .list .item:hover,
.spiders .list .item:hover {
background: #F5F7FA;
}
.items {
width: 100%;
height: auto;
overflow: auto;
}
.items >>> .action-wrapper {
text-align: right;
padding-bottom: 10px;
border-bottom: 1px solid #DCDFE6;
}
.items >>> .custom-tree-node {
flex: 1;
display: flex;
align-items: center;
justify-content: space-between;
font-size: 14px;
padding-right: 8px;
min-height: 36px;
}
.items >>> .el-tree > .el-tree-node {
border-bottom: 1px solid #e6e9f0;
}
.items >>> .el-tree-node__content {
height: auto;
}
.items >>> .custom-tree-node .label i.el-icon-edit {
visibility: hidden;
}
.items >>> .custom-tree-node:hover .label i.el-icon-edit {
visibility: visible;
}
.items >>> .custom-tree-node .el-input {
width: 240px;
}
</style>

3
frontend/src/i18n/zh.js
View File

@@ -217,6 +217,9 @@ export default {
'Long Task': '长任务',
'Running Task Count': '运行中的任务数',
'Running Tasks': '运行中的任务',
'Item Name': 'Item 名称',
'Add Item': '添加 Item',
'Add Variable': '添加变量',
// 爬虫列表
'Name': '名称',

49
frontend/src/store/modules/spider.js
View File

@@ -13,6 +13,12 @@ const state = {
// spider scrapy settings
spiderScrapySettings: [],
// spider scrapy items
spiderScrapyItems: [],
// spider scrapy pipelines
spiderScrapyPipelines: [],
// node to deploy/run
activeNode: {},
@@ -98,6 +104,12 @@ const mutations = {
},
SET_SPIDER_SCRAPY_SETTINGS (state, value) {
state.spiderScrapySettings = value
},
SET_SPIDER_SCRAPY_ITEMS (state, value) {
state.spiderScrapyItems = value
},
SET_SPIDER_SCRAPY_PIPELINES (state, value) {
state.spiderScrapyPipelines = value
}
}
@@ -150,6 +162,43 @@ const actions = {
async saveSpiderScrapySettings ({ state }, id) {
return request.post(`/spiders/${id}/scrapy/settings`, state.spiderScrapySettings)
},
async getSpiderScrapyItems ({ state, commit }, id) {
const res = await request.get(`/spiders/${id}/scrapy/items`)
let nodeId = 0
commit('SET_SPIDER_SCRAPY_ITEMS', res.data.data.map(d => {
d.id = nodeId++
d.label = d.name
d.level = 1
d.isEdit = false
d.children = d.fields.map(f => {
return {
id: nodeId++,
label: f,
level: 2,
isEdit: false
}
})
return d
}))
},
async saveSpiderScrapyItems ({ state }, id) {
return request.post(`/spiders/${id}/scrapy/items`, state.spiderScrapyItems.map(d => {
d.name = d.label
d.fields = d.children.map(f => f.label)
return d
}))
},
async getSpiderScrapyPipelines ({ state, commit }, id) {
const res = await request.get(`/spiders/${id}/scrapy/pipelines`)
commit('SET_SPIDER_SCRAPY_PIPELINES', res.data.data)
},
async saveSpiderScrapyPipelines ({ state }, id) {
return request.post(`/spiders/${id}/scrapy/pipelines`, state.spiderScrapyPipelines)
},
async getSpiderScrapySpiderFilepath ({ state, commit }, payload) {
const { id, spiderName } = payload
return request.get(`/spiders/${id}/scrapy/spider/filepath`, { spider_name: spiderName })
},
addSpiderScrapySpider ({ state }, payload) {
const { id, form } = payload
return request.put(`/spiders/${id}/scrapy/spiders`, form)

1
frontend/src/views/layout/components/Navbar.vue
View File

@@ -180,7 +180,6 @@ docker-compose up -d
},
howToUpgradeHtml () {
if (this.lang === 'zh') {
console.log(this.howToUpgradeHtmlZh)
return this.converter.makeHtml(this.howToUpgradeHtmlZh)
} else if (this.lang === 'en') {
return this.converter.makeHtml(this.howToUpgradeHtmlEn)

48
frontend/src/views/spider/SpiderDetail.vue
View File

@@ -26,13 +26,18 @@
<git-settings/>
</el-tab-pane>
<el-tab-pane v-if="isScrapy" :label="$t('Scrapy Settings')" name="scrapy-settings">
<spider-scrapy/>
<spider-scrapy
@click-spider="onClickScrapySpider"
@click-pipeline="onClickScrapyPipeline"
/>
</el-tab-pane>
<el-tab-pane v-if="isConfigurable" :label="$t('Config')" name="config">
<config-list ref="config"/>
</el-tab-pane>
<el-tab-pane :label="$t('Files')" name="files">
<file-list/>
<file-list
ref="file-list"
/>
</el-tab-pane>
<el-tab-pane :label="$t('Environment')" name="environment">
<environment-list/>
@@ -162,7 +167,8 @@ export default {
}
this.$utils.tour.nextStep('spider-detail', currentStep)
}
}
},
redirectType: ''
}
},
computed: {
@@ -190,7 +196,7 @@ export default {
}
},
methods: {
onTabClick (tab) {
async onTabClick (tab) {
if (this.activeTabName === 'analytics') {
setTimeout(() => {
this.$refs['spider-stats'].update()
@@ -207,12 +213,11 @@ export default {
}, 100)
}
} else if (this.activeTabName === 'scrapy-settings') {
this.$store.dispatch('spider/getSpiderScrapySpiders', this.$route.params.id)
this.$store.dispatch('spider/getSpiderScrapySettings', this.$route.params.id)
await this.getScrapyData()
} else if (this.activeTabName === 'files') {
this.$store.dispatch('spider/getFileTree')
await this.$store.dispatch('spider/getFileTree')
if (this.currentPath) {
this.$store.dispatch('file/getFileContent', { path: this.currentPath })
await this.$store.dispatch('file/getFileContent', { path: this.currentPath })
}
}
this.$st.sendEv('爬虫详情', '切换标签', tab.name)
@@ -220,12 +225,27 @@ export default {
onSpiderChange (id) {
this.$router.push(`/spiders/${id}`)
this.$st.sendEv('爬虫详情', '切换爬虫')
},
async getScrapyData () {
await Promise.all([
this.$store.dispatch('spider/getSpiderScrapySpiders', this.$route.params.id),
this.$store.dispatch('spider/getSpiderScrapyItems', this.$route.params.id),
this.$store.dispatch('spider/getSpiderScrapySettings', this.$route.params.id),
this.$store.dispatch('spider/getSpiderScrapyPipelines', this.$route.params.id)
])
},
async onClickScrapySpider (filepath) {
this.activeTabName = 'files'
await this.$store.dispatch('spider/getFileTree')
this.$refs['file-list'].clickSpider(filepath)
},
async onClickScrapyPipeline () {
this.activeTabName = 'files'
await this.$store.dispatch('spider/getFileTree')
this.$refs['file-list'].clickPipeline()
}
},
async created () {
// get the list of the spiders
// this.$store.dispatch('spider/getSpiderList')
// get spider basic info
await this.$store.dispatch('spider/getSpiderData', this.$route.params.id)
@@ -237,12 +257,6 @@ export default {
// get spider list
await this.$store.dispatch('spider/getSpiderList')
// get scrapy spider names
if (this.spiderForm.is_scrapy) {
await this.$store.dispatch('spider/getSpiderScrapySpiders', this.$route.params.id)
await this.$store.dispatch('spider/getSpiderScrapySettings', this.$route.params.id)
}
},
mounted () {
if (!this.$utils.tour.isFinishedTour('spider-detail')) {

51
frontend/src/views/spider/SpiderList.vue
View File

@@ -45,7 +45,12 @@
</el-select>
</el-form-item>
<el-form-item :label="$t('Execute Command')" prop="cmd" required>
<el-input id="cmd" v-model="spiderForm.cmd" :placeholder="$t('Execute Command')"/>
<el-input
id="cmd"
v-model="spiderForm.cmd"
:placeholder="$t('Execute Command')"
:disabled="spiderForm.is_scrapy"
/>
</el-form-item>
<el-form-item :label="$t('Results')" prop="col" required>
<el-input id="col" v-model="spiderForm.col" :placeholder="$t('Results')"/>
@@ -64,18 +69,33 @@
</el-button>
</el-upload>
</el-form-item>
<el-form-item :label="$t('Is Git')" prop="is_git">
<el-switch
v-model="spiderForm.is_git"
active-color="#13ce66"
/>
</el-form-item>
<el-form-item :label="$t('Is Long Task')" prop="is_long_task">
<el-switch
v-model="spiderForm.is_long_task"
active-color="#13ce66"
/>
</el-form-item>
<el-row>
<el-col :span="8">
<el-form-item :label="$t('Is Scrapy')" prop="is_scrapy">
<el-switch
v-model="spiderForm.is_scrapy"
active-color="#13ce66"
@change="onIsScrapy"
/>
</el-form-item>
</el-col>
<el-col :span="8">
<el-form-item :label="$t('Is Git')" prop="is_git">
<el-switch
v-model="spiderForm.is_git"
active-color="#13ce66"
/>
</el-form-item>
</el-col>
<el-col :span="8">
<el-form-item :label="$t('Is Long Task')" prop="is_long_task">
<el-switch
v-model="spiderForm.is_long_task"
active-color="#13ce66"
/>
</el-form-item>
</el-col>
</el-row>
</el-form>
<el-alert
type="warning"
@@ -1052,6 +1072,11 @@ export default {
this.$message.success(`Task "${row._id}" has been sent signal to stop`)
this.getList()
}
},
onIsScrapy (value) {
if (value) {
this.spiderForm.cmd = 'scrapy crawl'
}
}
},
async created () {