git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

加入scrapy日志选择

Browse Source
This commit is contained in:
marvzhang
2020-02-15 21:25:21 +08:00
parent acdf4d5951
commit 01fefbee88
7 changed files with 115 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
backend/main.go
View File

@@ -154,25 +154,26 @@ func main() {
}
// 爬虫
{
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID）
authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID）
authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表
}
// 可配置爬虫
{

25
backend/model/schedule.go
View File

@@ -12,18 +12,19 @@ import (
)
type Schedule struct {
Id bson.ObjectId `json:"_id" bson:"_id"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
SpiderId bson.ObjectId `json:"spider_id" bson:"spider_id"`
Cron string `json:"cron" bson:"cron"`
EntryId cron.EntryID `json:"entry_id" bson:"entry_id"`
Param string `json:"param" bson:"param"`
RunType string `json:"run_type" bson:"run_type"`
NodeIds []bson.ObjectId `json:"node_ids" bson:"node_ids"`
Status string `json:"status" bson:"status"`
Enabled bool `json:"enabled" bson:"enabled"`
UserId bson.ObjectId `json:"user_id" bson:"user_id"`
Id bson.ObjectId `json:"_id" bson:"_id"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
SpiderId bson.ObjectId `json:"spider_id" bson:"spider_id"`
Cron string `json:"cron" bson:"cron"`
EntryId cron.EntryID `json:"entry_id" bson:"entry_id"`
Param string `json:"param" bson:"param"`
RunType string `json:"run_type" bson:"run_type"`
NodeIds []bson.ObjectId `json:"node_ids" bson:"node_ids"`
Status string `json:"status" bson:"status"`
Enabled bool `json:"enabled" bson:"enabled"`
UserId bson.ObjectId `json:"user_id" bson:"user_id"`
ScrapySpider string `json:"scrapy_spider" bson:"scrapy_spider"`
// 前端展示
SpiderName string `json:"spider_name" bson:"spider_name"`

6
backend/routes/task.go
View File

@@ -118,7 +118,7 @@ func PutTask(c *gin.Context) {
UserId: services.GetCurrentUser(c).Id,
}
id, err := services.AddTask(t);
id, err := services.AddTask(t)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
@@ -133,7 +133,7 @@ func PutTask(c *gin.Context) {
Param: reqBody.Param,
UserId: services.GetCurrentUser(c).Id,
}
id, err := services.AddTask(t);
id, err := services.AddTask(t)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
@@ -149,7 +149,7 @@ func PutTask(c *gin.Context) {
UserId: services.GetCurrentUser(c).Id,
}
id, err := services.AddTask(t);
id, err := services.AddTask(t)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return

28
backend/services/schedule.go
View File

@@ -22,6 +22,28 @@ func AddScheduleTask(s model.Schedule) func() {
// 生成任务ID
id := uuid.NewV4()
// 参数
var param string
// 爬虫
spider, err := model.GetSpider(s.SpiderId)
if err != nil {
return
}
// scrapy 爬虫
if spider.IsScrapy {
if s.ScrapySpider == "" {
log.Errorf("scrapy spider is not set")
debug.PrintStack()
return
}
param = s.ScrapySpider + " " + s.Param
} else {
param = s.Param
}
if s.RunType == constants.RunTypeAllNodes {
// 所有节点
nodes, err := model.GetNodeList(nil)
@@ -33,7 +55,7 @@ func AddScheduleTask(s model.Schedule) func() {
Id: id.String(),
SpiderId: s.SpiderId,
NodeId: node.Id,
Param: s.Param,
Param: param,
UserId: s.UserId,
}
@@ -46,7 +68,7 @@ func AddScheduleTask(s model.Schedule) func() {
t := model.Task{
Id: id.String(),
SpiderId: s.SpiderId,
Param: s.Param,
Param: param,
UserId: s.UserId,
}
if _, err := AddTask(t); err != nil {
@@ -61,7 +83,7 @@ func AddScheduleTask(s model.Schedule) func() {
Id: id.String(),
SpiderId: s.SpiderId,
NodeId: nodeId,
Param: s.Param,
Param: param,
UserId: s.UserId,
}

41
frontend/src/components/Common/CrawlConfirmDialog.vue
View File

@@ -3,11 +3,11 @@
:title="$t('Notification')"
:visible="visible"
class="crawl-confirm-dialog"
width="540px"
width="580px"
:before-close="beforeClose"
>
<div style="margin-bottom: 20px;">{{$t('Are you sure to run this spider?')}}</div>
<el-form label-width="120px" :model="form" ref="form">
<el-form label-width="140px" :model="form" ref="form">
<el-form-item :label="$t('Run Type')" prop="runType" required inline-message>
<el-select v-model="form.runType" :placeholder="$t('Run Type')">
<el-option value="all-nodes" :label="$t('All Nodes')"/>
@@ -36,8 +36,23 @@
/>
</el-select>
</el-form-item>
<el-form-item :label="$t('Parameters')" prop="param" inline-message>
<el-input v-model="form.param" :placeholder="$t('Parameters')"></el-input>
<el-form-item v-if="spiderForm.is_scrapy" :label="$t('Scrapy Log Level')" prop="scrapy_log_level" required
inline-message>
<el-select v-model="form.scrapy_log_level" :placeholder="$t('Scrapy Log Level')">
<el-option value="INFO" label="INFO"/>
<el-option value="DEBUG" label="DEBUG"/>
<el-option value="WARN" label="WARN"/>
<el-option value="ERROR" label="ERROR"/>
</el-select>
</el-form-item>
<el-form-item v-if="spiderForm.type === 'customized'" :label="$t('Parameters')" prop="param" inline-message>
<template v-if="spiderForm.is_scrapy">
<el-input v-model="form.param" :placeholder="$t('Parameters')" class="param-input"/>
<el-button type="primary" icon="el-icon-edit" class="param-btn"/>
</template>
<template v-else>
<el-input v-model="form.param" :placeholder="$t('Parameters')"></el-input>
</template>
</el-form-item>
<el-form-item class="disclaimer-wrapper">
<div>
@@ -84,6 +99,7 @@ export default {
runType: 'random',
nodeIds: undefined,
spider: undefined,
scrapy_log_level: 'INFO',
param: '',
nodeList: []
},
@@ -120,7 +136,7 @@ export default {
const res = await this.$store.dispatch('spider/crawlSpider', {
spiderId: this.spiderId,
nodeIds: this.form.nodeIds,
param: this.form.param + ' ' + this.form.spider,
param: `${this.form.spider} --loglevel=${this.form.scrapy_log_level} ${this.form.param}`,
runType: this.form.runType
})
@@ -177,4 +193,19 @@ export default {
.crawl-confirm-dialog >>> .disclaimer-wrapper a {
color: #409eff;
}
.crawl-confirm-dialog >>> .param-input {
width: calc(100% - 56px);
}
.crawl-confirm-dialog >>> .param-input .el-input__inner {
border-top-right-radius: 0;
border-bottom-right-radius: 0;
border-right: none;
}
.crawl-confirm-dialog >>> .param-btn {
width: 56px;
border-top-left-radius: 0;
border-bottom-left-radius: 0;
}
</style>

1
frontend/src/i18n/zh.js
View File

@@ -184,6 +184,7 @@ export default {
'Template': '模版',
'Is Scrapy': '是否为 Scrapy',
'Scrapy Spider': 'Scrapy 爬虫',
'Scrapy Log Level': 'Scrapy 日志等级',
// 爬虫列表
'Name': '名称',

19
frontend/src/views/schedule/ScheduleList.vue
View File

@@ -94,6 +94,15 @@
/>
</el-select>
</el-form-item>
<el-form-item v-if="spiderForm.is_scrapy" :label="$t('Scrapy Log Level')" prop="scrapy_spider" required
inline-message>
<el-select v-model="scheduleForm.scrapy_log_level" :placeholder="$t('Scrapy Log Level')">
<el-option value="INFO" label="INFO"/>
<el-option value="DEBUG" label="DEBUG"/>
<el-option value="WARN" label="WARN"/>
<el-option value="ERROR" label="ERROR"/>
</el-select>
</el-form-item>
<el-form-item :label="$t('Cron')" prop="cron" required>
<el-popover v-model="isShowCron" trigger="focus">
<template>
@@ -111,7 +120,7 @@
</el-popover>
<!--<el-button size="small" style="width:100px" type="primary" @click="onShowCronDialog">{{$t('schedules.add_cron')}}</el-button>-->
</el-form-item>
<el-form-item :label="$t('Execute Command')" prop="params">
<el-form-item :label="$t('Execute Command')" prop="cmd">
<el-input
id="cmd"
v-model="spiderForm.cmd"
@@ -119,7 +128,7 @@
disabled
/>
</el-form-item>
<el-form-item :label="$t('Parameters')" prop="param">
<el-form-item v-if="spiderForm.type === 'customized'" :label="$t('Parameters')" prop="param">
<el-input
id="param"
v-model="scheduleForm.param"
@@ -445,6 +454,9 @@ export default {
if (res) {
const form = JSON.parse(JSON.stringify(this.scheduleForm))
form.cron = '0 ' + this.scheduleForm.cron
if (this.spiderForm.is_scrapy) {
form.param += form.param + ' --loglevel=' + form.scrapy_log_level
}
if (this.isEdit) {
request.post(`/schedules/${this.scheduleForm._id}`, form).then(response => {
if (response.data.error) {
@@ -479,6 +491,9 @@ export default {
this.$st.sendEv('定时任务', '修改定时任务')
this.isLoading = true
if (!this.scheduleForm.scrapy_log_level) {
this.$set(this.scheduleForm, 'scrapy_log_level', 'INFO')
}
await this.$store.dispatch('spider/getSpiderData', row.spider_id)
if (this.spiderForm.is_scrapy) {
await this.$store.dispatch('spider/getSpiderScrapySpiders', row.spider_id)