diff --git a/backend/model/spider.go b/backend/model/spider.go
index c1544259..95a1c358 100644
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -37,6 +37,10 @@ type Spider struct {
// 自定义爬虫
Cmd string `json:"cmd" bson:"cmd"` // 执行命令
+ // Scrapy 爬虫(属于自定义爬虫)
+ IsScrapy bool `json:"is_scrapy" bson:"is_scrapy"` // 是否为 Scrapy 爬虫
+ SpiderNames []string `json:"spider_names" bson:"spider_names"` // 爬虫名称列表
+
// 可配置爬虫
Template string `json:"template" bson:"template"` // Spiderfile模版
diff --git a/backend/routes/spider.go b/backend/routes/spider.go
index 1abb6463..18ed5071 100644
--- a/backend/routes/spider.go
+++ b/backend/routes/spider.go
@@ -88,15 +88,16 @@ func GetSpider(c *gin.Context) {
HandleErrorF(http.StatusBadRequest, c, "invalid id")
}
- result, err := model.GetSpider(bson.ObjectIdHex(id))
+ spider, err := model.GetSpider(bson.ObjectIdHex(id))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
+
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
- Data: result,
+ Data: spider,
})
}
@@ -901,3 +902,30 @@ func GetSpiderSchedules(c *gin.Context) {
Data: list,
})
}
+
+func GetSpiderScrapySpiders(c *gin.Context) {
+ id := c.Param("id")
+
+ if !bson.IsObjectIdHex(id) {
+ HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid")
+ return
+ }
+
+ spider, err := model.GetSpider(bson.ObjectIdHex(id))
+ if err != nil {
+ HandleError(http.StatusInternalServerError, c, err)
+ return
+ }
+
+ spiderNames, err := services.GetScrapySpiderNames(spider)
+ if err != nil {
+ HandleError(http.StatusInternalServerError, c, err)
+ return
+ }
+
+ c.JSON(http.StatusOK, Response{
+ Status: "ok",
+ Message: "success",
+ Data: spiderNames,
+ })
+}
diff --git a/backend/services/scrapy.go b/backend/services/scrapy.go
new file mode 100644
index 00000000..2e984e6d
--- /dev/null
+++ b/backend/services/scrapy.go
@@ -0,0 +1,32 @@
+package services
+
+import (
+ "bytes"
+ "crawlab/model"
+ "os/exec"
+ "strings"
+)
+
+func GetScrapySpiderNames(s model.Spider) ([]string, error) {
+ var stdout bytes.Buffer
+ var stderr bytes.Buffer
+
+ cmd := exec.Command("scrapy", "list")
+ cmd.Dir = s.Src
+ cmd.Stdout = &stdout
+ cmd.Stderr = &stderr
+ if err := cmd.Run(); err != nil {
+ return []string{}, err
+ }
+
+ spiderNames := strings.Split(stdout.String(), "\n")
+
+ var res []string
+ for _, sn := range spiderNames {
+ if sn != "" {
+ res = append(res, sn)
+ }
+ }
+
+ return res, nil
+}
diff --git a/backend/services/task.go b/backend/services/task.go
index c71d344f..e940b325 100644
--- a/backend/services/task.go
+++ b/backend/services/task.go
@@ -243,7 +243,6 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e
if runtime.GOOS == constants.Windows {
cmd = exec.Command("cmd", "/C", cmdStr)
} else {
- cmd = exec.Command("")
cmd = exec.Command("sh", "-c", cmdStr)
}
diff --git a/frontend/src/components/Common/CrawlConfirmDialog.vue b/frontend/src/components/Common/CrawlConfirmDialog.vue
index 6739ae9e..de067295 100644
--- a/frontend/src/components/Common/CrawlConfirmDialog.vue
+++ b/frontend/src/components/Common/CrawlConfirmDialog.vue
@@ -3,11 +3,11 @@
:title="$t('Notification')"
:visible="visible"
class="crawl-confirm-dialog"
- width="480px"
+ width="540px"
:before-close="beforeClose"
>
{{$t('Are you sure to run this spider?')}}
-
+
@@ -26,6 +26,16 @@
/>
+
+
+
+
+
@@ -44,14 +54,17 @@
{{$t('Cancel')}}
- {{$t('Confirm')}}
+
+ {{$t('Confirm')}}
diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue
index 801c0fce..cd9e9f95 100644
--- a/frontend/src/components/InfoView/SpiderInfoView.vue
+++ b/frontend/src/components/InfoView/SpiderInfoView.vue
@@ -36,24 +36,19 @@
-
-
-
+
+
+
+
+
-
-
-
-
@@ -63,6 +58,13 @@
+
+
+
@@ -159,16 +161,14 @@ export default {
this.$st.sendEv('爬虫详情', '概览', '点击运行')
},
onSave () {
- this.$refs['spiderForm'].validate(res => {
- if (res) {
- this.$store.dispatch('spider/editSpider')
- .then(() => {
- this.$message.success(this.$t('Spider info has been saved successfully'))
- })
- .catch(error => {
- this.$message.error(error)
- })
+ this.$refs['spiderForm'].validate(async valid => {
+ if (!valid) return
+ const res = await this.$store.dispatch('spider/editSpider')
+ if (!res.data.error) {
+ this.$message.success(this.$t('Spider info has been saved successfully'))
}
+ await this.$store.dispatch('spider/getSpiderData', this.$route.params.id)
+ await this.$store.dispatch('spider/getSpiderScrapySpiders', this.$route.params.id)
})
this.$st.sendEv('爬虫详情', '概览', '保存')
},
@@ -197,6 +197,11 @@ export default {
},
onUploadError () {
this.uploadLoading = false
+ },
+ onIsScrapyChange (value) {
+ if (value) {
+ this.spiderForm.cmd = 'scrapy crawl'
+ }
}
},
async created () {
diff --git a/frontend/src/components/Status/StatusTag.vue b/frontend/src/components/Status/StatusTag.vue
index befe2ab3..29f53fcd 100644
--- a/frontend/src/components/Status/StatusTag.vue
+++ b/frontend/src/components/Status/StatusTag.vue
@@ -47,6 +47,10 @@ export default {
return 'el-icon-loading'
} else if (this.status === 'error') {
return 'el-icon-error'
+ } else if (this.status === 'cancelled') {
+ return 'el-icon-video-pause'
+ } else if (this.status === 'abnormal') {
+ return 'el-icon-question'
}
return ''
}
diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index 419fcd9d..67a5437f 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -182,6 +182,8 @@ export default {
'Settings': '设置',
'Display Name': '显示名称',
'Template': '模版',
+ 'Is Scrapy': '是否为 Scrapy',
+ 'Scrapy Spider': 'Scrapy 爬虫',
// 爬虫列表
'Name': '名称',
diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js
index b6dcf223..dbb54514 100644
--- a/frontend/src/store/modules/spider.js
+++ b/frontend/src/store/modules/spider.js
@@ -116,6 +116,11 @@ const actions = {
commit('SET_SPIDER_FORM', data)
})
},
+ async getSpiderScrapySpiders ({ state, commit }, id) {
+ const res = await request.get(`/spiders/${id}/scrapy/spiders`)
+ state.spiderForm.spider_names = res.data.data
+ commit('SET_SPIDER_FORM', state.spiderForm)
+ },
crawlSpider ({ state, dispatch }, payload) {
const { spiderId, runType, nodeIds, param } = payload
return request.put(`/tasks`, {
diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue
index 3b0d85da..f987386d 100644
--- a/frontend/src/views/schedule/ScheduleList.vue
+++ b/frontend/src/views/schedule/ScheduleList.vue
@@ -83,6 +83,17 @@
+
+
+
+
+
@@ -103,7 +114,7 @@
@@ -123,7 +134,7 @@
@@ -246,6 +257,7 @@ export default {
{ name: 'run_type', label: 'Run Type', width: '120px' },
{ name: 'node_names', label: 'Node', width: '150px' },
{ name: 'spider_name', label: 'Spider', width: '150px' },
+ { name: 'scrapy_spider', label: 'Scrapy Spider', width: '150px' },
{ name: 'param', label: 'Parameters', width: '150px' },
{ name: 'description', label: 'Description', width: '200px' },
{ name: 'enable', label: 'Enable/Disable', width: '120px' }
@@ -259,6 +271,7 @@ export default {
spiderList: [],
nodeList: [],
isShowCron: false,
+ isLoading: false,
// tutorial
tourSteps: [
@@ -379,6 +392,9 @@ export default {
}
},
computed: {
+ ...mapState('spider', [
+ 'spiderForm'
+ ]),
...mapState('schedule', [
'scheduleList',
'scheduleForm'
@@ -456,11 +472,23 @@ export default {
},
isShowRun (row) {
},
- onEdit (row) {
+ async onEdit (row) {
this.$store.commit('schedule/SET_SCHEDULE_FORM', row)
this.dialogVisible = true
this.isEdit = true
this.$st.sendEv('定时任务', '修改定时任务')
+
+ this.isLoading = true
+ await this.$store.dispatch('spider/getSpiderData', row.spider_id)
+ if (this.spiderForm.is_scrapy) {
+ await this.$store.dispatch('spider/getSpiderScrapySpiders', row.spider_id)
+ if (!this.scheduleForm.scrapy_spider) {
+ if (this.spiderForm.spider_names && this.spiderForm.spider_names.length > 0) {
+ this.$set(this.scheduleForm, 'scrapy_spider', this.spiderForm.spider_names[0])
+ }
+ }
+ }
+ this.isLoading = false
},
onRemove (row) {
this.$confirm(this.$t('Are you sure to delete the schedule task?'), this.$t('Notification'), {
diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue
index 9bc49984..6a7ee5f3 100644
--- a/frontend/src/views/spider/SpiderDetail.vue
+++ b/frontend/src/views/spider/SpiderDetail.vue
@@ -217,6 +217,11 @@ export default {
// get spider list
await this.$store.dispatch('spider/getSpiderList')
+ // get scrapy spider names
+ if (this.spiderForm.is_scrapy) {
+ await this.$store.dispatch('spider/getSpiderScrapySpiders', this.$route.params.id)
+ }
+
// if spider is configurable spider, set to config tab by default
// if (this.spiderForm.type === 'configurable') {
// this.activeTabName = 'config'
diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue
index 3a1d38ca..aae91d72 100644
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -309,6 +309,23 @@
+
+
+
+
+