updated configurable spider

2026-01-22 17:31:03 +01:00 · 2019-05-26 17:10:04 +08:00
parent 15f9b7ef05
commit 5daebcb39b
14 changed files with 371 additions and 63 deletions
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -20,9 +20,9 @@
    </el-dialog>
    <!--./preview results-->

-    <el-row style="margin-top: 10px;">
+    <el-row>
      <el-col :span="11" offset="1">
-        <el-form label-width="100px">
+        <el-form label-width="150px">
          <el-form-item :label="$t('Crawl Type')">
            <el-button-group>
              <el-button v-for="type in crawlTypeList"
@@ -36,28 +36,54 @@
          <el-form-item :label="$t('Start URL')">
            <el-input v-model="spiderForm.start_url" :placeholder="$t('Start URL')"></el-input>
          </el-form-item>
+          <el-form-item :label="$t('Obey robots.txt')">
+            <el-switch v-model="spiderForm.obey_robots_txt" :placeholder="$t('Obey robots.txt')"></el-switch>
+          </el-form-item>
        </el-form>
      </el-col>
      <el-col :span="11" :offset="1">
        <el-form label-width="150px">
          <el-form-item :label="$t('Item Selector')"
                        v-if="['list','list-detail'].includes(spiderForm.crawl_type)">
-            <el-input v-model="spiderForm.item_selector" :placeholder="$t('Item Selector')"></el-input>
+            <el-select style="width: 35%;margin-right: 10px;"
+                       v-model="spiderForm.item_selector_type"
+                       :placeholder="$t('Item Selector Type')">
+              <el-option value="xpath" :label="$t('XPath')"></el-option>
+              <el-option value="css" :label="$t('CSS')"></el-option>
+            </el-select>
+            <el-input style="width: calc(65% - 10px);"
+                      v-model="spiderForm.item_selector"
+                      :placeholder="$t('Item Selector')">
+            </el-input>
          </el-form-item>
          <el-form-item :label="$t('Pagination Selector')"
                        v-if="['list','list-detail'].includes(spiderForm.crawl_type)">
-            <el-input v-model="spiderForm.pagination_selector" :placeholder="$t('Pagination Selector')"></el-input>
+            <el-select style="width: 35%;margin-right: 10px;"
+                       v-model="spiderForm.pagination_selector_type"
+                       :placeholder="$t('Pagination Selector Type')">
+              <el-option value="xpath" :label="$t('XPath')"></el-option>
+              <el-option value="css" :label="$t('CSS')"></el-option>
+            </el-select>
+            <el-input style="width: calc(65% - 10px);"
+                      v-model="spiderForm.pagination_selector"
+                      :placeholder="$t('Pagination Selector')">
+            </el-input>
          </el-form-item>
        </el-form>
      </el-col>
    </el-row>

    <!--button group-->
-    <el-row>
-      <div class="button-group">
-        <el-button type="primary" @click="addField" icon="el-icon-plus">{{$t('Add Field')}}</el-button>
-        <el-button type="warning" @click="onPreview" v-loading="previewLoading">{{$t('Preview')}}</el-button>
-        <el-button type="success" @click="onSave" v-loading="saveLoading">{{$t('Save')}}</el-button>
+    <el-row style="margin-top: 10px">
+      <div class="button-group-wrapper">
+        <div class="button-group">
+          <el-button type="primary" @click="addField" icon="el-icon-plus">{{$t('Add Field')}}</el-button>
+        </div>
+        <div class="button-group">
+          <el-button type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
+          <el-button type="warning" @click="onPreview" v-loading="previewLoading">{{$t('Preview')}}</el-button>
+          <el-button type="success" @click="onSave" v-loading="saveLoading">{{$t('Save')}}</el-button>
+        </div>
      </div>
    </el-row>
    <!--./button group-->
@@ -200,6 +226,18 @@ export default {
              this.previewLoading = false
            })
        })
+    },
+    onCrawl () {
+      this.$confirm(this.$t('Are you sure to run this spider?'), this.$t('Notification'), {
+        confirmButtonText: this.$t('Confirm'),
+        cancelButtonText: this.$t('Cancel')
+      })
+        .then(() => {
+          this.$store.dispatch('spider/crawlSpider', this.spiderForm._id)
+            .then(() => {
+              this.$message.success(this.$t(`Spider task has been scheduled`))
+            })
+        })
    }
  },
  created () {
@@ -215,13 +253,15 @@ export default {
    }
    if (!this.spiderForm.crawl_type) this.$set(this.spiderForm, 'crawl_type', 'list')
    if (!this.spiderForm.start_url) this.$set(this.spiderForm, 'start_url', 'http://example.com')
+    if (!this.spiderForm.item_selector_type) this.$set(this.spiderForm, 'item_selector_type', 'css')
+    if (!this.spiderForm.pagination_selector_type) this.$set(this.spiderForm, 'pagination_selector_type', 'css')
+    if (!this.spiderForm.obey_robots_txt) this.$set(this.spiderForm, 'obey_robots_txt', true)
  }
 }
 </script>

 <style scoped>
  .el-table {
-    margin-top: 10px;
  }

  .el-table.edit >>> .el-table__body td {
@@ -248,6 +288,11 @@ export default {
    line-height: 36px;
  }

+  .button-group-wrapper {
+    display: flex;
+    justify-content: space-between;
+  }
+
  .button-group {
    text-align: right;
  }
--- a/frontend/src/components/InfoView/SpiderInfoView.vue
+++ b/frontend/src/components/InfoView/SpiderInfoView.vue
@@ -99,13 +99,14 @@ export default {
    ]),
    isShowRun () {
      if (this.isCustomized) {
+        // customized spider
        if (!this.spiderForm.deploy_ts) {
          return false
        }
        return !!this.spiderForm.cmd
      } else {
-        // TODO: has to add rules
-        return false
+        // configurable spider
+        return !!this.spiderForm.fields
      }
    },
    isCustomized () {
@@ -113,7 +114,7 @@ export default {
    }
  },
  methods: {
-    onRun () {
+    onCrawl () {
      const row = this.spiderForm
      this.$refs['spiderForm'].validate(res => {
        if (res) {
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -112,6 +112,22 @@ export default {
  'Customized': '自定义',
  'Text': '文本',
  'Attribute': '属性',
+  'Field Name': '字段名称',
+  'Query Type': '查询类别',
+  'Query': '查询',
+  'Extract Type': '提取类别',
+  'CSS Selector': 'CSS选择器',
+  'Crawl Type': '抓取类别',
+  'List Only': '仅列表',
+  'Detail Only': '仅详情页',
+  'List + Detail': '列表＋详情页',
+  'Start URL': '开始URL',
+  'Item Selector': '列表项选择器',
+  'Item Selector Type': '列表项选择器类别',
+  'Pagination Selector': '分页选择器',
+  'Pagination Selector Type': '分页项选择器类别',
+  'Preview Results': '预览结果',
+  'Obey robots.txt': '遵守Robots协议',

  // 爬虫列表
  'Name': '名称',
--- a/frontend/src/store/modules/spider.js
+++ b/frontend/src/store/modules/spider.js
@@ -106,7 +106,10 @@ const actions = {
      crawl_type: state.spiderForm.crawl_type,
      start_url: state.spiderForm.start_url,
      item_selector: state.spiderForm.item_selector,
-      pagination_selector: state.spiderForm.pagination_selector
+      item_selector_type: state.spiderForm.item_selector_type,
+      pagination_selector: state.spiderForm.pagination_selector,
+      pagination_selector_type: state.spiderForm.pagination_selector_type,
+      obey_robots_txt: state.spiderForm.obey_robots_txt
    })
      .then(() => {
        dispatch('getSpiderList')
--- a/frontend/src/views/schedule/ScheduleList.vue
+++ b/frontend/src/views/schedule/ScheduleList.vue
@@ -196,7 +196,7 @@ export default {
          }, 100)
        })
    },
-    onRun () {
+    onCrawl () {
    }
  },
  created () {
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -426,13 +426,19 @@ export default {
        })
    },
    isShowRun (row) {
-      if (!row.deploy_ts) {
-        return false
+      if (this.isCustomized(row)) {
+        // customized spider
+        if (!row.deploy_ts) {
+          return false
+        }
+        return !!row.cmd
+      } else {
+        // configurable spider
+        return !!row.fields
      }
-      if (!row.cmd) {
-        return false
-      }
-      return true
+    },
+    isCustomized (row) {
+      return row.type === 'customized'
    },
    fetchSiteSuggestions (keyword, callback) {
      this.$request.get('/sites', {
--- a/frontend/src/views/task/TaskDetail.vue
+++ b/frontend/src/views/task/TaskDetail.vue
@@ -6,11 +6,13 @@
        <task-overview/>
      </el-tab-pane>
      <el-tab-pane :label="$t('Log')" name="log">
-        <div class="log-view">
-          <pre>
-            {{taskLog}}
-          </pre>
-        </div>
+        <el-card>
+          <div class="log-view">
+            <pre>
+              {{taskLog}}
+            </pre>
+          </div>
+        </el-card>
      </el-tab-pane>
      <el-tab-pane :label="$t('Results')" name="results">
        <general-table-view :data="taskResultsData"
@@ -101,4 +103,15 @@ export default {
  .selector .el-select {
    padding-left: 10px;
  }
+
+  .log-view {
+    margin: 20px;
+    height: 640px;
+  }
+
+  .log-view pre {
+    height: 100%;
+    overflow-x: auto;
+    overflow-y: auto;
+  }
 </style>