diff --git a/backend/main.go b/backend/main.go index ab0d0e7b..19c111d7 100644 --- a/backend/main.go +++ b/backend/main.go @@ -154,25 +154,26 @@ func main() { } // 爬虫 { - authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 - authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 - authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫 - authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫 - authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫 - authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 - authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID） - authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 - authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 - authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取 - authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 - authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改 - authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建 - authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建 - authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除 - authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名 - authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 - authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 - authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务 + authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 + authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 + authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫 + authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫 + authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫 + authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 + authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID） + authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 + authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 + authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取 + authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 + authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改 + authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建 + authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建 + authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除 + authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名 + authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 + authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 + authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务 + authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表 } // 可配置爬虫 { diff --git a/backend/model/schedule.go b/backend/model/schedule.go index d98dabf6..1b988f2a 100644 --- a/backend/model/schedule.go +++ b/backend/model/schedule.go @@ -12,18 +12,19 @@ import ( ) type Schedule struct { - Id bson.ObjectId `json:"_id" bson:"_id"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - SpiderId bson.ObjectId `json:"spider_id" bson:"spider_id"` - Cron string `json:"cron" bson:"cron"` - EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` - Param string `json:"param" bson:"param"` - RunType string `json:"run_type" bson:"run_type"` - NodeIds []bson.ObjectId `json:"node_ids" bson:"node_ids"` - Status string `json:"status" bson:"status"` - Enabled bool `json:"enabled" bson:"enabled"` - UserId bson.ObjectId `json:"user_id" bson:"user_id"` + Id bson.ObjectId `json:"_id" bson:"_id"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + SpiderId bson.ObjectId `json:"spider_id" bson:"spider_id"` + Cron string `json:"cron" bson:"cron"` + EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` + Param string `json:"param" bson:"param"` + RunType string `json:"run_type" bson:"run_type"` + NodeIds []bson.ObjectId `json:"node_ids" bson:"node_ids"` + Status string `json:"status" bson:"status"` + Enabled bool `json:"enabled" bson:"enabled"` + UserId bson.ObjectId `json:"user_id" bson:"user_id"` + ScrapySpider string `json:"scrapy_spider" bson:"scrapy_spider"` // 前端展示 SpiderName string `json:"spider_name" bson:"spider_name"` diff --git a/backend/routes/task.go b/backend/routes/task.go index d1071881..ae1c431c 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -118,7 +118,7 @@ func PutTask(c *gin.Context) { UserId: services.GetCurrentUser(c).Id, } - id, err := services.AddTask(t); + id, err := services.AddTask(t) if err != nil { HandleError(http.StatusInternalServerError, c, err) return @@ -133,7 +133,7 @@ func PutTask(c *gin.Context) { Param: reqBody.Param, UserId: services.GetCurrentUser(c).Id, } - id, err := services.AddTask(t); + id, err := services.AddTask(t) if err != nil { HandleError(http.StatusInternalServerError, c, err) return @@ -149,7 +149,7 @@ func PutTask(c *gin.Context) { UserId: services.GetCurrentUser(c).Id, } - id, err := services.AddTask(t); + id, err := services.AddTask(t) if err != nil { HandleError(http.StatusInternalServerError, c, err) return diff --git a/backend/services/schedule.go b/backend/services/schedule.go index 1bf70e8a..ab47d7e3 100644 --- a/backend/services/schedule.go +++ b/backend/services/schedule.go @@ -22,6 +22,28 @@ func AddScheduleTask(s model.Schedule) func() { // 生成任务ID id := uuid.NewV4() + // 参数 + var param string + + // 爬虫 + spider, err := model.GetSpider(s.SpiderId) + if err != nil { + return + } + + // scrapy 爬虫 + if spider.IsScrapy { + if s.ScrapySpider == "" { + log.Errorf("scrapy spider is not set") + debug.PrintStack() + return + } + + param = s.ScrapySpider + " " + s.Param + } else { + param = s.Param + } + if s.RunType == constants.RunTypeAllNodes { // 所有节点 nodes, err := model.GetNodeList(nil) @@ -33,7 +55,7 @@ func AddScheduleTask(s model.Schedule) func() { Id: id.String(), SpiderId: s.SpiderId, NodeId: node.Id, - Param: s.Param, + Param: param, UserId: s.UserId, } @@ -46,7 +68,7 @@ func AddScheduleTask(s model.Schedule) func() { t := model.Task{ Id: id.String(), SpiderId: s.SpiderId, - Param: s.Param, + Param: param, UserId: s.UserId, } if _, err := AddTask(t); err != nil { @@ -61,7 +83,7 @@ func AddScheduleTask(s model.Schedule) func() { Id: id.String(), SpiderId: s.SpiderId, NodeId: nodeId, - Param: s.Param, + Param: param, UserId: s.UserId, } diff --git a/frontend/src/components/Common/CrawlConfirmDialog.vue b/frontend/src/components/Common/CrawlConfirmDialog.vue index de067295..a10442f3 100644 --- a/frontend/src/components/Common/CrawlConfirmDialog.vue +++ b/frontend/src/components/Common/CrawlConfirmDialog.vue @@ -3,11 +3,11 @@ :title="$t('Notification')" :visible="visible" class="crawl-confirm-dialog" - width="540px" + width="580px" :before-close="beforeClose" >
{{$t('Are you sure to run this spider?')}}
- + @@ -36,8 +36,23 @@ /> - - + + + + + + + + + + +
@@ -84,6 +99,7 @@ export default { runType: 'random', nodeIds: undefined, spider: undefined, + scrapy_log_level: 'INFO', param: '', nodeList: [] }, @@ -120,7 +136,7 @@ export default { const res = await this.$store.dispatch('spider/crawlSpider', { spiderId: this.spiderId, nodeIds: this.form.nodeIds, - param: this.form.param + ' ' + this.form.spider, + param: `${this.form.spider} --loglevel=${this.form.scrapy_log_level} ${this.form.param}`, runType: this.form.runType }) @@ -177,4 +193,19 @@ export default { .crawl-confirm-dialog >>> .disclaimer-wrapper a { color: #409eff; } + + .crawl-confirm-dialog >>> .param-input { + width: calc(100% - 56px); + } + .crawl-confirm-dialog >>> .param-input .el-input__inner { + border-top-right-radius: 0; + border-bottom-right-radius: 0; + border-right: none; + } + + .crawl-confirm-dialog >>> .param-btn { + width: 56px; + border-top-left-radius: 0; + border-bottom-left-radius: 0; + } diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index 67a5437f..c6a8afa7 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -184,6 +184,7 @@ export default { 'Template': '模版', 'Is Scrapy': '是否为 Scrapy', 'Scrapy Spider': 'Scrapy 爬虫', + 'Scrapy Log Level': 'Scrapy 日志等级', // 爬虫列表 'Name': '名称', diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index f987386d..3646f403 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -94,6 +94,15 @@ /> + + + + + + + +