加入scrapy爬虫支持

2026-01-28 17:50:56 +01:00 · 2020-02-15 18:28:37 +08:00
parent e2b9d285da
commit acdf4d5951
12 changed files with 222 additions and 49 deletions
--- a/frontend/src/components/Common/CrawlConfirmDialog.vue
+++ b/frontend/src/components/Common/CrawlConfirmDialog.vue
@@ -3,11 +3,11 @@
    :title="$t('Notification')"
    :visible="visible"
    class="crawl-confirm-dialog"
-    width="480px"
+    width="540px"
    :before-close="beforeClose"
  >
    <div style="margin-bottom: 20px;">{{$t('Are you sure to run this spider?')}}</div>
-    <el-form label-width="80px" :model="form" ref="form">
+    <el-form label-width="120px" :model="form" ref="form">
      <el-form-item :label="$t('Run Type')" prop="runType" required inline-message>
        <el-select v-model="form.runType" :placeholder="$t('Run Type')">
          <el-option value="all-nodes" :label="$t('All Nodes')"/>
@@ -26,6 +26,16 @@
          />
        </el-select>
      </el-form-item>
+      <el-form-item v-if="spiderForm.is_scrapy" :label="$t('Scrapy Spider')" prop="spider" required inline-message>
+        <el-select v-model="form.spider" :placeholder="$t('Scrapy Spider')" :disabled="isLoading">
+          <el-option
+            v-for="s in spiderForm.spider_names"
+            :key="s"
+            :label="s"
+            :value="s"
+          />
+        </el-select>
+      </el-form-item>
      <el-form-item :label="$t('Parameters')" prop="param" inline-message>
        <el-input v-model="form.param" :placeholder="$t('Parameters')"></el-input>
      </el-form-item>
@@ -44,14 +54,17 @@
    </el-form>
    <template slot="footer">
      <el-button type="plain" size="small" @click="$emit('close')">{{$t('Cancel')}}</el-button>
-      <el-button type="primary" size="small" @click="onConfirm" :disabled="!isAllowDisclaimer">{{$t('Confirm')}}
+      <el-button type="primary" size="small" @click="onConfirm" :disabled="isConfirmDisabled">
+        {{$t('Confirm')}}
      </el-button>
    </template>
  </el-dialog>
 </template>

 <script>
-import request from '../../api/request'
+import {
+  mapState
+} from 'vuex'

 export default {
  name: 'CrawlConfirmDialog',
@@ -70,11 +83,30 @@ export default {
      form: {
        runType: 'random',
        nodeIds: undefined,
+        spider: undefined,
        param: '',
        nodeList: []
      },
      isAllowDisclaimer: true,
-      isRedirect: true
+      isRedirect: true,
+      isLoading: false
+    }
+  },
+  computed: {
+    ...mapState('spider', [
+      'spiderForm'
+    ]),
+    isConfirmDisabled () {
+      if (this.isLoading) return true
+      if (!this.isAllowDisclaimer) return true
+      return false
+    }
+  },
+  watch: {
+    visible (value) {
+      if (value) {
+        this.onOpen()
+      }
    }
  },
  methods: {
@@ -88,7 +120,7 @@ export default {
        const res = await this.$store.dispatch('spider/crawlSpider', {
          spiderId: this.spiderId,
          nodeIds: this.form.nodeIds,
-          param: this.form.param,
+          param: this.form.param + ' ' + this.form.spider,
          runType: this.form.runType
        })

@@ -107,21 +139,32 @@ export default {
    },
    onClickDisclaimer () {
      this.$router.push('/disclaimer')
-    }
-  },
-  created () {
-    // 节点列表
-    request.get('/nodes', {}).then(response => {
-      this.nodeList = response.data.data.map(d => {
-        d.systemInfo = {
-          os: '',
-          arch: '',
-          num_cpu: '',
-          executables: []
-        }
-        return d
+    },
+    async onOpen () {
+      // 节点列表
+      this.$request.get('/nodes', {}).then(response => {
+        this.nodeList = response.data.data.map(d => {
+          d.systemInfo = {
+            os: '',
+            arch: '',
+            num_cpu: '',
+            executables: []
+          }
+          return d
+        })
      })
-    })
+
+      // 爬虫列表
+      this.isLoading = true
+      await this.$store.dispatch('spider/getSpiderData', this.spiderId)
+      if (this.spiderForm.is_scrapy) {
+        await this.$store.dispatch('spider/getSpiderScrapySpiders', this.spiderId)
+        if (this.spiderForm.spider_names && this.spiderForm.spider_names.length > 0) {
+          this.$set(this.form, 'spider', this.spiderForm.spider_names[0])
+        }
+      }
+      this.isLoading = false
+    }
  }
 }
 </script>
--- a/frontend/src/components/InfoView/SpiderInfoView.vue
+++ b/frontend/src/components/InfoView/SpiderInfoView.vue
@@ -36,24 +36,19 @@
        <el-form-item :label="$t('Source Folder')">
          <el-input v-model="spiderForm.src" :placeholder="$t('Source Folder')" disabled></el-input>
        </el-form-item>
-        <el-form-item v-if="spiderForm.type === 'customized'" :label="$t('Execute Command')" prop="cmd" required
-                      :inline-message="true">
-          <el-input v-model="spiderForm.cmd" :placeholder="$t('Execute Command')"
-                    :disabled="isView"></el-input>
-        </el-form-item>
+        <template v-if="spiderForm.type === 'customized'">
+          <el-form-item :label="$t('Execute Command')" prop="cmd" required :inline-message="true">
+            <el-input
+              v-model="spiderForm.cmd"
+              :placeholder="$t('Execute Command')"
+              :disabled="isView || spiderForm.is_scrapy"
+            />
+          </el-form-item>
+        </template>
        <el-form-item :label="$t('Results Collection')" prop="col" required :inline-message="true">
          <el-input v-model="spiderForm.col" :placeholder="$t('Results Collection')"
                    :disabled="isView"></el-input>
        </el-form-item>
-        <el-form-item v-if="false" :label="$t('Site')">
-          <el-autocomplete v-model="spiderForm.site"
-                           :placeholder="$t('Site')"
-                           :fetch-suggestions="fetchSiteSuggestions"
-                           clearable
-                           :disabled="isView"
-                           @select="onSiteSelect">
-          </el-autocomplete>
-        </el-form-item>
        <el-form-item :label="$t('Spider Type')">
          <el-select v-model="spiderForm.type" :placeholder="$t('Spider Type')" :disabled="true" clearable>
            <el-option value="configurable" :label="$t('Configurable')"></el-option>
@@ -63,6 +58,13 @@
        <el-form-item :label="$t('Remark')">
          <el-input type="textarea" v-model="spiderForm.remark" :placeholder="$t('Remark')" :disabled="isView"/>
        </el-form-item>
+        <el-form-item v-if="spiderForm.type === 'customized'" :label="$t('Is Scrapy')" prop="is_scrapy">
+          <el-switch
+            v-model="spiderForm.is_scrapy"
+            active-color="#13ce66"
+            @change="onIsScrapyChange"
+          />
+        </el-form-item>
      </el-form>
    </el-row>
    <el-row class="button-container" v-if="!isView">
@@ -159,16 +161,14 @@ export default {
      this.$st.sendEv('爬虫详情', '概览', '点击运行')
    },
    onSave () {
-      this.$refs['spiderForm'].validate(res => {
-        if (res) {
-          this.$store.dispatch('spider/editSpider')
-            .then(() => {
-              this.$message.success(this.$t('Spider info has been saved successfully'))
-            })
-            .catch(error => {
-              this.$message.error(error)
-            })
+      this.$refs['spiderForm'].validate(async valid => {
+        if (!valid) return
+        const res = await this.$store.dispatch('spider/editSpider')
+        if (!res.data.error) {
+          this.$message.success(this.$t('Spider info has been saved successfully'))
        }
+        await this.$store.dispatch('spider/getSpiderData', this.$route.params.id)
+        await this.$store.dispatch('spider/getSpiderScrapySpiders', this.$route.params.id)
      })
      this.$st.sendEv('爬虫详情', '概览', '保存')
    },
@@ -197,6 +197,11 @@ export default {
    },
    onUploadError () {
      this.uploadLoading = false
+    },
+    onIsScrapyChange (value) {
+      if (value) {
+        this.spiderForm.cmd = 'scrapy crawl'
+      }
    }
  },
  async created () {
--- a/frontend/src/components/Status/StatusTag.vue
+++ b/frontend/src/components/Status/StatusTag.vue
@@ -47,6 +47,10 @@ export default {
        return 'el-icon-loading'
      } else if (this.status === 'error') {
        return 'el-icon-error'
+      } else if (this.status === 'cancelled') {
+        return 'el-icon-video-pause'
+      } else if (this.status === 'abnormal') {
+        return 'el-icon-question'
      }
      return ''
    }