From 7a75e643eae5f75b382bc2c0e9fdbe478dd6f8ea Mon Sep 17 00:00:00 2001 From: marvzhang Date: Sat, 22 Feb 2020 19:12:24 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E6=89=B9=E9=87=8F=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E7=88=AC=E8=99=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/main.go | 1 + backend/routes/spider.go | 89 ++++++++++++ .../components/Common/CrawlConfirmDialog.vue | 133 ++++++++++++++---- frontend/src/store/modules/spider.js | 8 ++ frontend/src/views/spider/SpiderList.vue | 20 ++- 5 files changed, 223 insertions(+), 28 deletions(-) diff --git a/backend/main.go b/backend/main.go index 0d79f7ba..7ddfdbf8 100644 --- a/backend/main.go +++ b/backend/main.go @@ -186,6 +186,7 @@ func main() { authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步 authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置 authGroup.POST("/spiders-cancel", routes.CancelSelectedSpider) // 停止所选爬虫任务 + authGroup.POST("/spiders-run", routes.RunSelectedSpider) // 运行所选爬虫 } // 可配置爬虫 { diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 2f69582a..9ad77948 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -536,6 +536,95 @@ func CancelSelectedSpider(c *gin.Context) { }) } +func RunSelectedSpider(c *gin.Context) { + type TaskParam struct { + SpiderId bson.ObjectId `json:"spider_id"` + Param string `json:"param"` + } + type ReqBody struct { + RunType string `json:"run_type"` + NodeIds []bson.ObjectId `json:"node_ids"` + TaskParams []TaskParam `json:"task_params"` + } + + var reqBody ReqBody + if err := c.ShouldBindJSON(&reqBody); err != nil { + HandleErrorF(http.StatusBadRequest, c, "invalid request") + return + } + + // 任务ID + var taskIds []string + + // 遍历爬虫 + // TODO: 优化此部分代码，与 routes.PutTask 有重合部分 + for _, taskParam := range reqBody.TaskParams { + if reqBody.RunType == constants.RunTypeAllNodes { + // 所有节点 + nodes, err := model.GetNodeList(nil) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + for _, node := range nodes { + t := model.Task{ + SpiderId: taskParam.SpiderId, + NodeId: node.Id, + Param: taskParam.Param, + UserId: services.GetCurrentUser(c).Id, + } + + id, err := services.AddTask(t) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + taskIds = append(taskIds, id) + } + } else if reqBody.RunType == constants.RunTypeRandom { + // 随机 + t := model.Task{ + SpiderId: taskParam.SpiderId, + Param: taskParam.Param, + UserId: services.GetCurrentUser(c).Id, + } + id, err := services.AddTask(t) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + taskIds = append(taskIds, id) + } else if reqBody.RunType == constants.RunTypeSelectedNodes { + // 指定节点 + for _, nodeId := range reqBody.NodeIds { + t := model.Task{ + SpiderId: taskParam.SpiderId, + NodeId: nodeId, + Param: taskParam.Param, + UserId: services.GetCurrentUser(c).Id, + } + + id, err := services.AddTask(t) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + taskIds = append(taskIds, id) + } + } else { + HandleErrorF(http.StatusInternalServerError, c, "invalid run_type") + return + } + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: taskIds, + }) +} + func GetSpiderTasks(c *gin.Context) { id := c.Param("id") diff --git a/frontend/src/components/Common/CrawlConfirmDialog.vue b/frontend/src/components/Common/CrawlConfirmDialog.vue index 1d5e2d7a..a1d6b1ad 100644 --- a/frontend/src/components/Common/CrawlConfirmDialog.vue +++ b/frontend/src/components/Common/CrawlConfirmDialog.vue @@ -44,8 +44,13 @@ /> - + @@ -68,7 +73,7 @@ 我已阅读并同意 《免责声明》 所有内容 -
+
跳转到任务详情页
@@ -100,9 +105,19 @@ export default { type: String, default: '' }, + spiders: { + type: Array, + default () { + return [] + } + }, visible: { type: Boolean, default: false + }, + multiple: { + type: Boolean, + default: false } }, data () { @@ -118,7 +133,8 @@ export default { isAllowDisclaimer: true, isRedirect: true, isLoading: false, - isParametersVisible: false + isParametersVisible: false, + scrapySpidersNamesDict: {} } }, computed: { @@ -129,6 +145,9 @@ export default { if (this.isLoading) return true if (!this.isAllowDisclaimer) return true return false + }, + scrapySpiders () { + return this.spiders.filter(d => d.type === 'customized' && d.is_scrapy) } }, watch: { @@ -145,30 +164,77 @@ export default { beforeParameterClose () { this.isParametersVisible = false }, + async fetchScrapySpiderName (id) { + const res = await this.$request.get(`/spiders/${id}/scrapy/spiders`) + this.scrapySpidersNamesDict[id] = res.data.data + }, onConfirm () { this.$refs['form'].validate(async valid => { if (!valid) return - let param = this.form.param - if (this.spiderForm.type === 'customized' && this.spiderForm.is_scrapy) { - param = `${this.form.spider} --loglevel=${this.form.scrapy_log_level} ${this.form.param}` + // 请求响应 + let res + + if (!this.multiple) { + // 运行单个爬虫 + + // 参数 + let param = this.form.param + + // Scrapy爬虫特殊处理 + if (this.spiderForm.type === 'customized' && this.spiderForm.is_scrapy) { + param = `${this.form.spider} --loglevel=${this.form.scrapy_log_level} ${this.form.param}` + } + + // 发起请求 + res = await this.$store.dispatch('spider/crawlSpider', { + spiderId: this.spiderId, + nodeIds: this.form.nodeIds, + param, + runType: this.form.runType + }) + } else { + // 运行多个爬虫 + + // 发起请求 + res = await this.$store.dispatch('spider/crawlSelectedSpiders', { + nodeIds: this.form.nodeIds, + runType: this.form.runType, + taskParams: this.spiders.map(d => { + // 参数 + let param = this.form.param + + // Scrapy爬虫特殊处理 + if (d.type === 'customized' && d.is_scrapy) { + param = `${this.scrapySpidersNamesDict[d._id] ? this.scrapySpidersNamesDict[d._id][0] : ''} --loglevel=${this.form.scrapy_log_level} ${this.form.param}` + } + + return { + spider_id: d._id, + param + } + }) + }) } - const res = await this.$store.dispatch('spider/crawlSpider', { - spiderId: this.spiderId, - nodeIds: this.form.nodeIds, - param, - runType: this.form.runType - }) - - const id = res.data.data[0] - + // 消息提示 this.$message.success(this.$t('A task has been scheduled successfully')) this.$emit('close') - this.$st.sendEv('爬虫确认', '确认运行', this.form.runType) + if (this.multiple) { + this.$st.sendEv('爬虫确认', '确认批量运行', this.form.runType) + } else { + this.$st.sendEv('爬虫确认', '确认运行', this.form.runType) + } - if (this.isRedirect && !this.spiderForm.is_long_task) { + // 是否重定向 + if ( + this.isRedirect && + !this.spiderForm.is_long_task && + !this.multiple + ) { + // 返回任务id + const id = res.data.data[0] this.$router.push('/tasks/' + id) this.$st.sendEv('爬虫确认', '跳转到任务详情') } @@ -194,15 +260,32 @@ export default { }) // 爬虫列表 - this.isLoading = true - await this.$store.dispatch('spider/getSpiderData', this.spiderId) - if (this.spiderForm.is_scrapy) { - await this.$store.dispatch('spider/getSpiderScrapySpiders', this.spiderId) - if (this.spiderForm.spider_names && this.spiderForm.spider_names.length > 0) { - this.$set(this.form, 'spider', this.spiderForm.spider_names[0]) + if (!this.multiple) { + // 单个爬虫 + this.isLoading = true + try { + await this.$store.dispatch('spider/getSpiderData', this.spiderId) + if (this.spiderForm.is_scrapy) { + await this.$store.dispatch('spider/getSpiderScrapySpiders', this.spiderId) + if (this.spiderForm.spider_names && this.spiderForm.spider_names.length > 0) { + this.$set(this.form, 'spider', this.spiderForm.spider_names[0]) + } + } + } finally { + this.isLoading = false + } + } else { + // 多个爬虫 + this.isLoading = true + try { + // 遍历 Scrapy 爬虫列表 + await Promise.all(this.scrapySpiders.map(async d => { + return this.fetchScrapySpiderName(d._id) + })) + } finally { + this.isLoading = false } } - this.isLoading = false }, onOpenParameters () { this.isParametersVisible = true diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js index f4e5dd02..fe982ded 100644 --- a/frontend/src/store/modules/spider.js +++ b/frontend/src/store/modules/spider.js @@ -212,6 +212,14 @@ const actions = { param: param }) }, + crawlSelectedSpiders ({ state, dispatch }, payload) { + const { taskParams, runType, nodeIds } = payload + return request.post(`/spiders-run`, { + task_params: taskParams, + run_type: runType, + node_ids: nodeIds + }) + }, getTaskList ({ state, commit }, id) { return request.get(`/spiders/${id}/tasks`) .then(response => { diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 48c6aecc..4783aaab 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -283,7 +283,9 @@ @@ -342,7 +344,7 @@ type="danger" icon="el-icon-video-play" class="btn add" - @click="onRunSelectedSpiders" + @click="onCrawlSelectedSpiders" style="font-weight: bolder" > {{$t('Run')}} @@ -781,7 +783,8 @@ export default { activeSpiderTaskStatus: 'running', selectedSpiders: [], isStopLoading: false, - isRemoveLoading: false + isRemoveLoading: false, + isMultiple: false } }, computed: { @@ -830,6 +833,9 @@ export default { return this.nodeList.filter(d => { return d.status === 'online' }) + }, + activeSpiderIds () { + return this.selectedSpiders.map(d => d._id) } }, methods: { @@ -1165,6 +1171,14 @@ export default { } this.$st.sendEv('爬虫列表', '批量删除爬虫') }) + }, + onCrawlSelectedSpiders () { + this.crawlConfirmDialogVisible = true + this.isMultiple = true + }, + onCrawlConfirmDialogClose () { + this.crawlConfirmDialogVisible = false + this.isMultiple = false } }, async created () {