From acdf4d59512fc0a89aa432d8f779dcead28f109a Mon Sep 17 00:00:00 2001 From: marvzhang Date: Sat, 15 Feb 2020 18:28:37 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5scrapy=E7=88=AC=E8=99=AB?= =?UTF-8?q?=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/spider.go | 4 + backend/routes/spider.go | 32 ++++++- backend/services/scrapy.go | 32 +++++++ backend/services/task.go | 1 - .../components/Common/CrawlConfirmDialog.vue | 83 ++++++++++++++----- .../components/InfoView/SpiderInfoView.vue | 51 +++++++----- frontend/src/components/Status/StatusTag.vue | 4 + frontend/src/i18n/zh.js | 2 + frontend/src/store/modules/spider.js | 5 ++ frontend/src/views/schedule/ScheduleList.vue | 34 +++++++- frontend/src/views/spider/SpiderDetail.vue | 5 ++ frontend/src/views/spider/SpiderList.vue | 18 ++++ 12 files changed, 222 insertions(+), 49 deletions(-) create mode 100644 backend/services/scrapy.go diff --git a/backend/model/spider.go b/backend/model/spider.go index c1544259..95a1c358 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -37,6 +37,10 @@ type Spider struct { // 自定义爬虫 Cmd string `json:"cmd" bson:"cmd"` // 执行命令 + // Scrapy 爬虫（属于自定义爬虫） + IsScrapy bool `json:"is_scrapy" bson:"is_scrapy"` // 是否为 Scrapy 爬虫 + SpiderNames []string `json:"spider_names" bson:"spider_names"` // 爬虫名称列表 + // 可配置爬虫 Template string `json:"template" bson:"template"` // Spiderfile模版 diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 1abb6463..18ed5071 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -88,15 +88,16 @@ func GetSpider(c *gin.Context) { HandleErrorF(http.StatusBadRequest, c, "invalid id") } - result, err := model.GetSpider(bson.ObjectIdHex(id)) + spider, err := model.GetSpider(bson.ObjectIdHex(id)) if err != nil { HandleError(http.StatusInternalServerError, c, err) return } + c.JSON(http.StatusOK, Response{ Status: "ok", Message: "success", - Data: result, + Data: spider, }) } @@ -901,3 +902,30 @@ func GetSpiderSchedules(c *gin.Context) { Data: list, }) } + +func GetSpiderScrapySpiders(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid") + return + } + + spider, err := model.GetSpider(bson.ObjectIdHex(id)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + spiderNames, err := services.GetScrapySpiderNames(spider) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: spiderNames, + }) +} diff --git a/backend/services/scrapy.go b/backend/services/scrapy.go new file mode 100644 index 00000000..2e984e6d --- /dev/null +++ b/backend/services/scrapy.go @@ -0,0 +1,32 @@ +package services + +import ( + "bytes" + "crawlab/model" + "os/exec" + "strings" +) + +func GetScrapySpiderNames(s model.Spider) ([]string, error) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + cmd := exec.Command("scrapy", "list") + cmd.Dir = s.Src + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return []string{}, err + } + + spiderNames := strings.Split(stdout.String(), "\n") + + var res []string + for _, sn := range spiderNames { + if sn != "" { + res = append(res, sn) + } + } + + return res, nil +} diff --git a/backend/services/task.go b/backend/services/task.go index c71d344f..e940b325 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -243,7 +243,6 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e if runtime.GOOS == constants.Windows { cmd = exec.Command("cmd", "/C", cmdStr) } else { - cmd = exec.Command("") cmd = exec.Command("sh", "-c", cmdStr) } diff --git a/frontend/src/components/Common/CrawlConfirmDialog.vue b/frontend/src/components/Common/CrawlConfirmDialog.vue index 6739ae9e..de067295 100644 --- a/frontend/src/components/Common/CrawlConfirmDialog.vue +++ b/frontend/src/components/Common/CrawlConfirmDialog.vue @@ -3,11 +3,11 @@ :title="$t('Notification')" :visible="visible" class="crawl-confirm-dialog" - width="480px" + width="540px" :before-close="beforeClose" >
{{$t('Are you sure to run this spider?')}}
- + @@ -26,6 +26,16 @@ /> + + + + + @@ -44,14 +54,17 @@ diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index 801c0fce..cd9e9f95 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -36,24 +36,19 @@ - - - + - - - - @@ -63,6 +58,13 @@ + + + @@ -159,16 +161,14 @@ export default { this.$st.sendEv('爬虫详情', '概览', '点击运行') }, onSave () { - this.$refs['spiderForm'].validate(res => { - if (res) { - this.$store.dispatch('spider/editSpider') - .then(() => { - this.$message.success(this.$t('Spider info has been saved successfully')) - }) - .catch(error => { - this.$message.error(error) - }) + this.$refs['spiderForm'].validate(async valid => { + if (!valid) return + const res = await this.$store.dispatch('spider/editSpider') + if (!res.data.error) { + this.$message.success(this.$t('Spider info has been saved successfully')) } + await this.$store.dispatch('spider/getSpiderData', this.$route.params.id) + await this.$store.dispatch('spider/getSpiderScrapySpiders', this.$route.params.id) }) this.$st.sendEv('爬虫详情', '概览', '保存') }, @@ -197,6 +197,11 @@ export default { }, onUploadError () { this.uploadLoading = false + }, + onIsScrapyChange (value) { + if (value) { + this.spiderForm.cmd = 'scrapy crawl' + } } }, async created () { diff --git a/frontend/src/components/Status/StatusTag.vue b/frontend/src/components/Status/StatusTag.vue index befe2ab3..29f53fcd 100644 --- a/frontend/src/components/Status/StatusTag.vue +++ b/frontend/src/components/Status/StatusTag.vue @@ -47,6 +47,10 @@ export default { return 'el-icon-loading' } else if (this.status === 'error') { return 'el-icon-error' + } else if (this.status === 'cancelled') { + return 'el-icon-video-pause' + } else if (this.status === 'abnormal') { + return 'el-icon-question' } return '' } diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index 419fcd9d..67a5437f 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -182,6 +182,8 @@ export default { 'Settings': '设置', 'Display Name': '显示名称', 'Template': '模版', + 'Is Scrapy': '是否为 Scrapy', + 'Scrapy Spider': 'Scrapy 爬虫', // 爬虫列表 'Name': '名称', diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js index b6dcf223..dbb54514 100644 --- a/frontend/src/store/modules/spider.js +++ b/frontend/src/store/modules/spider.js @@ -116,6 +116,11 @@ const actions = { commit('SET_SPIDER_FORM', data) }) }, + async getSpiderScrapySpiders ({ state, commit }, id) { + const res = await request.get(`/spiders/${id}/scrapy/spiders`) + state.spiderForm.spider_names = res.data.data + commit('SET_SPIDER_FORM', state.spiderForm) + }, crawlSpider ({ state, dispatch }, payload) { const { spiderId, runType, nodeIds, param } = payload return request.put(`/tasks`, { diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index 3b0d85da..f987386d 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -83,6 +83,17 @@ + + + + + + + +