From 8c059c912e612b6667861ad1d8d52a1e9be5e7b2 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Sat, 22 Feb 2020 17:35:51 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E6=89=B9=E9=87=8F=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E7=88=AC=E8=99=AB=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/main.go | 5 +-- backend/model/spider.go | 4 +-- backend/routes/spider.go | 24 ++++++++++++++ backend/routes/task.go | 2 +- backend/routes/version.go | 6 ++-- backend/services/spider.go | 32 +++++++++++++++++++ frontend/src/components/File/FileList.vue | 4 +-- .../components/InfoView/SpiderInfoView.vue | 2 +- frontend/src/i18n/zh.js | 2 ++ frontend/src/store/modules/task.js | 2 +- frontend/src/views/spider/SpiderList.vue | 24 +++++++++++++- 11 files changed, 95 insertions(+), 12 deletions(-) diff --git a/backend/main.go b/backend/main.go index cc7ed140..0d79f7ba 100644 --- a/backend/main.go +++ b/backend/main.go @@ -185,6 +185,7 @@ func main() { authGroup.GET("/spiders/:id/scrapy/spider/filepath", routes.GetSpiderScrapySpiderFilepath) // Scrapy 爬虫 pipelines authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步 authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置 + authGroup.POST("/spiders-cancel", routes.CancelSelectedSpider) // 停止所选爬虫任务 } // 可配置爬虫 { @@ -202,8 +203,8 @@ func main() { authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情 authGroup.PUT("/tasks", routes.PutTask) // 派发任务 authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务 - authGroup.DELETE("/tasks_multiple", routes.DeleteMultipleTask) // 删除多个任务 - authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) //删除指定状态的任务 + authGroup.DELETE("/tasks", routes.DeleteSelectedTask) // 删除多个任务 + authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) // 删除指定状态的任务 authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务 authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志 authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果 diff --git a/backend/model/spider.go b/backend/model/spider.go index a741fc89..49f735a4 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -279,6 +279,8 @@ func RemoveSpider(id bson.ObjectId) error { var result Spider if err := c.FindId(id).One(&result); err != nil { + log.Errorf("find spider error: %s, id:%s", err.Error(), id.Hex()) + debug.PrintStack() return err } @@ -291,12 +293,10 @@ func RemoveSpider(id bson.ObjectId) error { // gf上的文件 s, gf := database.GetGridFs("files") defer s.Close() - if result.FileId.Hex() != constants.ObjectIdNull { if err := gf.RemoveId(result.FileId); err != nil { log.Error("remove file error, id:" + result.FileId.Hex()) debug.PrintStack() - return err } } diff --git a/backend/routes/spider.go b/backend/routes/spider.go index d1d84e54..2f69582a 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -512,6 +512,30 @@ func DeleteSelectedSpider(c *gin.Context) { }) } +func CancelSelectedSpider(c *gin.Context) { + type ReqBody struct { + SpiderIds []string `json:"spider_ids"` + } + + var reqBody ReqBody + if err := c.ShouldBindJSON(&reqBody); err != nil { + HandleErrorF(http.StatusBadRequest, c, "invalid request") + return + } + + for _, spiderId := range reqBody.SpiderIds { + if err := services.CancelSpider(spiderId); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + } + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + func GetSpiderTasks(c *gin.Context) { id := c.Param("id") diff --git a/backend/routes/task.go b/backend/routes/task.go index ae1c431c..2880abb9 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -183,7 +183,7 @@ func DeleteTaskByStatus(c *gin.Context) { } // 删除多个任务 -func DeleteMultipleTask(c *gin.Context) { +func DeleteSelectedTask(c *gin.Context) { ids := make(map[string][]string) if err := c.ShouldBindJSON(&ids); err != nil { HandleError(http.StatusInternalServerError, c, err) diff --git a/backend/routes/version.go b/backend/routes/version.go index ec3b80c7..f62d1387 100644 --- a/backend/routes/version.go +++ b/backend/routes/version.go @@ -2,15 +2,17 @@ package routes import ( "crawlab/services" + "github.com/apex/log" "github.com/gin-gonic/gin" "net/http" + "runtime/debug" ) func GetLatestRelease(c *gin.Context) { latestRelease, err := services.GetLatestRelease() if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return + log.Errorf(err.Error()) + debug.PrintStack() } c.JSON(http.StatusOK, Response{ Status: "ok", diff --git a/backend/services/spider.go b/backend/services/spider.go index 5b50bbf0..6d450ef1 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -261,6 +261,38 @@ func RemoveSpider(id string) error { return nil } +func CancelSpider(id string) error { + // 获取该爬虫 + spider, err := model.GetSpider(bson.ObjectIdHex(id)) + if err != nil { + return err + } + + // 获取该爬虫待定或运行中的任务列表 + query := bson.M{ + "spider_id": spider.Id, + "status": bson.M{ + "$in": []string{ + constants.StatusPending, + constants.StatusRunning, + }, + }, + } + tasks, err := model.GetTaskList(query, 0, constants.Infinite, "-create_ts") + if err != nil { + return err + } + + // 遍历任务列表，依次停止 + for _, task := range tasks { + if err := CancelTask(task.Id); err != nil { + return err + } + } + + return nil +} + // 启动爬虫服务 func InitSpiderService() error { // 构造定时任务执行器 diff --git a/frontend/src/components/File/FileList.vue b/frontend/src/components/File/FileList.vue index 0a8ac7a0..002e26bf 100644 --- a/frontend/src/components/File/FileList.vue +++ b/frontend/src/components/File/FileList.vue @@ -25,8 +25,8 @@ - {{$t('Cancel')}} - {{$t('Confirm')}} + {{$t('Cancel')}} + {{$t('Confirm')}} diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index 02a22818..7a95fede 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -45,7 +45,7 @@ /> - + diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index 21a93801..5f68aa66 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -560,6 +560,8 @@ docker run -d --restart always --name crawlab_worker \\ 'SSH Public Key is copied to the clipboard': 'SSH 公钥已粘贴到剪切板', 'Removed successfully': '已成功删除', 'Are you sure to delete selected items?': '您是否确认删除所选项？', + 'Are you sure to stop selected items?': '您是否确认停止所选项？', + 'Sent signals to cancel selected tasks': '已经向所选任务发送取消任务信号', // 其他 'Star crawlab-team/crawlab on GitHub': '在 GitHub 上为 Crawlab 加星吧' diff --git a/frontend/src/store/modules/task.js b/frontend/src/store/modules/task.js index 85270729..67f6a153 100644 --- a/frontend/src/store/modules/task.js +++ b/frontend/src/store/modules/task.js @@ -136,7 +136,7 @@ const actions = { }) }, deleteTaskMultiple ({ state }, ids) { - return request.delete(`/tasks_multiple`, { + return request.delete(`/tasks`, { ids: ids }) }, diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index f36f5d0f..48c6aecc 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -52,7 +52,7 @@ :disabled="spiderForm.is_scrapy" /> - + @@ -1143,6 +1143,28 @@ export default { } this.$st.sendEv('爬虫列表', '批量删除爬虫') }) + }, + async onStopSelectedSpiders () { + this.$confirm(this.$t('Are you sure to stop selected items?'), this.$t('Notification'), { + confirmButtonText: this.$t('Confirm'), + cancelButtonText: this.$t('Cancel'), + type: 'warning' + }).then(async () => { + this.isStopLoading = true + try { + const res = await this.$request.post('/spiders-cancel', { + spider_ids: this.selectedSpiders.map(d => d._id) + }) + if (!res.data.error) { + this.$message.success('Sent signals to cancel selected tasks') + this.$refs['table'].clearSelection() + await this.getList() + } + } finally { + this.isStopLoading = false + } + this.$st.sendEv('爬虫列表', '批量删除爬虫') + }) } }, async created () {