优化爬虫添加逻辑

2026-01-22 17:31:03 +01:00 · 2019-12-25 19:46:35 +08:00
parent fdcf12f866
commit 7d5b313a0b
4 changed files with 114 additions and 74 deletions
--- a/frontend/src/components/InfoView/SpiderInfoView.vue
+++ b/frontend/src/components/InfoView/SpiderInfoView.vue
@@ -21,7 +21,8 @@
        <el-form-item :label="$t('Source Folder')">
          <el-input v-model="spiderForm.src" :placeholder="$t('Source Folder')" disabled></el-input>
        </el-form-item>
-        <el-form-item v-if="spiderForm.type === 'customized'" :label="$t('Execute Command')" prop="cmd" required :inline-message="true">
+        <el-form-item v-if="spiderForm.type === 'customized'" :label="$t('Execute Command')" prop="cmd" required
+                      :inline-message="true">
          <el-input v-model="spiderForm.cmd" :placeholder="$t('Execute Command')"
                    :disabled="isView"></el-input>
        </el-form-item>
@@ -45,20 +46,41 @@
          </el-select>
        </el-form-item>
        <el-form-item :label="$t('Remark')">
-          <el-input v-model="spiderForm.remark"/>
+          <el-input type="textarea" v-model="spiderForm.remark" :placeholder="$t('Remark')"/>
        </el-form-item>
      </el-form>
    </el-row>
    <el-row class="button-container" v-if="!isView">
-      <el-button size="small" v-if="isShowRun" type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
-      <el-button size="small" type="success" @click="onSave">{{$t('Save')}}</el-button>
+      <el-upload
+        v-if="spiderForm.type === 'customized'"
+        :action="$request.baseUrl + `/spiders/${spiderForm._id}/upload`"
+        :headers="{Authorization:token}"
+        :on-progress="() => this.uploadLoading = true"
+        :on-error="onUploadError"
+        :on-success="onUploadSuccess"
+        :file-list="fileList"
+        style="display:inline-block;margin-right:10px"
+      >
+        <el-button size="normal" type="primary" icon="el-icon-upload" v-loading="uploadLoading">
+          {{$t('Upload')}}
+        </el-button>
+      </el-upload>
+      <el-button size="normal" v-if="isShowRun" type="danger" @click="onCrawl"
+                 icon="el-icon-video-play">
+        {{$t('Run')}}
+      </el-button>
+      <el-button size="normal" type="success" @click="onSave"
+                 icon="el-icon-check">
+        {{$t('Save')}}
+      </el-button>
    </el-row>
  </div>
 </template>

 <script>
 import {
-  mapState
+  mapState,
+  mapGetters
 } from 'vuex'
 import CrawlConfirmDialog from '../Common/CrawlConfirmDialog'

@@ -88,6 +110,8 @@ export default {
      callback()
    }
    return {
+      uploadLoading: false,
+      fileList: [],
      crawlConfirmDialogVisible: false,
      cmdRule: [
        { message: 'Execute Command should not be empty', required: true }
@@ -101,6 +125,9 @@ export default {
    ...mapState('spider', [
      'spiderForm'
    ]),
+    ...mapGetters('user', [
+      'token'
+    ]),
    isShowRun () {
      if (this.spiderForm.type === 'customized') {
        return !!this.spiderForm.cmd
@@ -143,6 +170,14 @@ export default {
    },
    onSiteSelect (item) {
      this.spiderForm.site = item._id
+    },
+    onUploadSuccess () {
+      this.$store.dispatch('file/getFileList', this.spiderForm.src)
+
+      this.uploadLoading = false
+    },
+    onUploadError () {
+      this.uploadLoading = false
    }
  }
 }
@@ -162,4 +197,8 @@ export default {
  .el-autocomplete {
    width: 100%;
  }
+
+  .info-view >>> .el-upload-list {
+    display: none;
+  }
 </style>
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -277,7 +277,9 @@ export default {
  'username already exists': '用户名已存在',
  'Deleted successfully': '成功删除',
  'Saved successfully': '成功保存',
-  'Please zip your spider files from the root directory': '爬虫文件请从根目录下开始压缩。',
+  'You can click "Add" to create an empty spider and upload files later.': '您可以点击"添加"按钮创建空的爬虫，之后再上传文件。',
+  'OR, you can also click "Upload" and upload a zip file containing your spider project.': '或者，您也可以点击"上传"按钮并上传一个包含爬虫项目的 zip 文件。',
+  'NOTE: When uploading a zip file, please zip your spider files from the ROOT DIRECTORY.': '注意: 上传 zip 文件时，请从 根目录 下开始压缩爬虫文件。',
  'English': 'English',
  'Are you sure to delete the schedule task?': '确定删除定时任务?',
  'Disclaimer': '免责声明',
--- a/frontend/src/store/modules/spider.js
+++ b/frontend/src/store/modules/spider.js
@@ -174,6 +174,9 @@ const actions = {
  addConfigSpider ({ state }) {
    return request.put(`/config_spiders`, state.spiderForm)
  },
+  addSpider ({ state }) {
+    return request.put(`/spiders`, state.spiderForm)
+  },
  async getTemplateList ({ state, commit }) {
    const res = await request.get(`/config_spiders_templates`)
    commit('SET_TEMPLATE_LIST', res.data.data)
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -34,6 +34,48 @@
               :visible.sync="addDialogVisible"
               :before-close="onAddDialogClose">
      <el-tabs :active-name="spiderType">
+        <el-tab-pane name="customized" :label="$t('Customized')">
+          <el-form :model="spiderForm" ref="addCustomizedForm" inline-message label-width="120px">
+            <el-form-item :label="$t('Spider Name')" prop="name" required>
+              <el-input v-model="spiderForm.name" :placeholder="$t('Spider Name')"/>
+            </el-form-item>
+            <el-form-item :label="$t('Display Name')" prop="display_name" required>
+              <el-input v-model="spiderForm.display_name" :placeholder="$t('Display Name')"/>
+            </el-form-item>
+            <el-form-item :label="$t('Execute Command')" prop="cmd" required>
+              <el-input v-model="spiderForm.cmd" :placeholder="$t('Execute Command')"/>
+            </el-form-item>
+            <el-form-item :label="$t('Results')" prop="col" required>
+              <el-input v-model="spiderForm.col" :placeholder="$t('Results')"/>
+            </el-form-item>
+            <el-form-item :label="$t('Upload Zip File')" label-width="120px" name="site">
+              <el-upload
+                :action="$request.baseUrl + '/spiders'"
+                :headers="{Authorization:token}"
+                :on-success="onUploadSuccess"
+                :file-list="fileList">
+                <el-button size="normal" type="primary" icon="el-icon-upload"
+                           style="width: 160px; font-size: 18px;font-weight: bolder">
+                  {{$t('Upload')}}
+                </el-button>
+              </el-upload>
+            </el-form-item>
+          </el-form>
+          <el-alert
+            type="error"
+            :closable="false"
+            style="margin-bottom: 10px"
+          >
+            <p>{{$t('You can click "Add" to create an empty spider and upload files later.')}}</p>
+            <p>{{$t('OR, you can also click "Upload" and upload a zip file containing your spider project.')}}</p>
+            <p style="font-weight: bolder">
+              <i class="fa fa-exclamation-triangle"></i> {{$t('NOTE: When uploading a zip file, please zip your spider files from the ROOT DIRECTORY.')}}
+            </p>
+          </el-alert>
+          <div class="actions">
+            <el-button type="primary" @click="onAddCustomized">{{$t('Add')}}</el-button>
+          </div>
+        </el-tab-pane>
        <el-tab-pane name="configurable" :label="$t('Configurable')">
          <el-form :model="spiderForm" ref="addConfigurableForm" inline-message label-width="120px">
            <el-form-item :label="$t('Spider Name')" prop="name" required>
@@ -60,63 +102,10 @@
            <el-button type="primary" @click="onAddConfigurable">{{$t('Add')}}</el-button>
          </div>
        </el-tab-pane>
-        <el-tab-pane name="customized" :label="$t('Customized')">
-          <el-form :model="spiderForm" ref="addCustomizedForm" inline-message>
-            <el-form-item :label="$t('Upload Zip File')" label-width="120px" name="site">
-              <el-upload
-                :action="$request.baseUrl + '/spiders'"
-                :headers="{Authorization:token}"
-                :on-change="onUploadChange"
-                :on-success="onUploadSuccess"
-                :file-list="fileList">
-                <el-button size="small" type="primary" icon="el-icon-upload">{{$t('Upload')}}</el-button>
-              </el-upload>
-            </el-form-item>
-          </el-form>
-          <el-alert type="error" :title="$t('Please zip your spider files from the root directory')"
-                    :closable="false"></el-alert>
-        </el-tab-pane>
      </el-tabs>
    </el-dialog>
    <!--./add dialog-->

-    <!--configurable spider dialog-->
-    <el-dialog :title="$t('Add Configurable Spider')"
-               width="40%"
-               :visible.sync="addConfigurableDialogVisible"
-               :before-close="onAddConfigurableDialogClose">
-      <el-form :model="spiderForm" ref="addConfigurableForm" inline-message>
-        <el-form-item :label="$t('Spider Name')" label-width="120px" prop="name" required>
-          <el-input :placeholder="$t('Spider Name')" v-model="spiderForm.name"></el-input>
-        </el-form-item>
-        <el-form-item :label="$t('Results Collection')" label-width="120px" name="col">
-          <el-input :placeholder="$t('Results Collection')" v-model="spiderForm.col"></el-input>
-        </el-form-item>
-        <el-form-item :label="$t('Site')" label-width="120px" name="site">
-          <el-autocomplete v-model="spiderForm.site"
-                           :placeholder="$t('Site')"
-                           :fetch-suggestions="fetchSiteSuggestions"
-                           @select="onAddConfigurableSiteSelect">
-          </el-autocomplete>
-        </el-form-item>
-      </el-form>
-      <span slot="footer" class="dialog-footer">
-        <el-button @click="addConfigurableDialogVisible = false">{{$t('Cancel')}}</el-button>
-        <el-button v-loading="addConfigurableLoading" type="primary"
-                   @click="onAddConfigurableSpider">{{$t('Add')}}</el-button>
-      </span>
-    </el-dialog>
-    <!--./configurable spider dialog-->
-
-    <!--customized spider dialog-->
-    <el-dialog :title="$t('Add Customized Spider')"
-               width="40%"
-               :visible.sync="addCustomizedDialogVisible"
-               :before-close="onAddCustomizedDialogClose">
-
-    </el-dialog>
-    <!--./customized spider dialog-->
-
    <!--crawl confirm dialog-->
    <crawl-confirm-dialog
      :visible="crawlConfirmDialogVisible"
@@ -154,10 +143,14 @@
          <el-button size="small" v-if="false" type="primary" icon="fa fa-download" @click="openImportDialog">
            {{$t('Import Spiders')}}
          </el-button>
-          <el-button size="small" type="success"
-                     icon="el-icon-plus"
-                     class="btn add"
-                     @click="onAdd">
+          <el-button
+            size="normal"
+            type="success"
+            icon="el-icon-plus"
+            class="btn add"
+            @click="onAdd"
+            style="font-weight: bolder"
+          >
            {{$t('Add Spider')}}
          </el-button>

@@ -299,8 +292,6 @@ export default {
      isEditMode: false,
      dialogVisible: false,
      addDialogVisible: false,
-      addConfigurableDialogVisible: false,
-      addCustomizedDialogVisible: false,
      crawlConfirmDialogVisible: false,
      activeSpiderId: undefined,
      filter: {
@@ -320,7 +311,7 @@ export default {
        name: [{ required: true, message: 'Required Field', trigger: 'change' }]
      },
      fileList: [],
-      spiderType: 'configurable'
+      spiderType: 'customized'
    }
  },
  computed: {
@@ -374,9 +365,19 @@ export default {
      })
    },
    onAddCustomized () {
-      this.addDialogVisible = false
-      this.addCustomizedDialogVisible = true
-      this.$st.sendEv('爬虫列表', '添加爬虫', '自定义爬虫')
+      this.$refs['addCustomizedForm'].validate(async res => {
+        if (!res) return
+        let res2
+        try {
+          res2 = await this.$store.dispatch('spider/addSpider')
+        } catch (e) {
+          this.$message.error(this.$t('Something wrong happened'))
+          return
+        }
+        await this.$store.dispatch('spider/getSpiderList')
+        this.$router.push(`/spiders/${res2.data.data._id}`)
+        this.$st.sendEv('爬虫列表', '添加爬虫', '自定义爬虫')
+      })
    },
    onRefresh () {
      this.getList()
@@ -510,8 +511,6 @@ export default {
        }
      })
    },
-    onUploadChange () {
-    },
    onUploadSuccess () {
      // clear fileList
      this.fileList = []
@@ -520,9 +519,6 @@ export default {
      setTimeout(() => {
        this.getList()
      }, 500)
-
-      // close popup
-      this.addCustomizedDialogVisible = false
    },
    getTime (str) {
      if (!str || str.match('^0001')) return 'NA'