更新可配置爬虫

backend/template/scrapy/config_spider/settings.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import os
+import re
 
 # Scrapy settings for config_spider project
 #
@@ -17,7 +18,7 @@ NEWSPIDER_MODULE = 'config_spider.spiders'
 
 
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
-#USER_AGENT = 'config_spider (+http://www.yourdomain.com)'
+USER_AGENT = 'Crawlab Spider'
 
 # Obey robots.txt rules
 ROBOTSTXT_OBEY = True
@@ -90,8 +91,18 @@ ITEM_PIPELINES = {
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
 
-for setting_env_name in [x for x in os.environ.keys() if x.startswith('CRAWALAB_SETTING_')]:
+for setting_env_name in [x for x in os.environ.keys() if x.startswith('CRAWLAB_SETTING_')]:
     setting_name = setting_env_name.replace('CRAWLAB_SETTING_', '')
-    setting_value = os.environ.get('setting_env_name')
+    setting_value = os.environ.get(setting_env_name)
+    if setting_value.lower() == 'true':
+        setting_value = True
+    elif setting_value.lower() == 'false':
+        setting_value = False
+    elif re.search(r'^\d+$', setting_value) is not None:
+        setting_value = int(setting_value)
+    else:
+        pass
+    print(setting_name)
+    print(setting_value)
     locals()[setting_name] = setting_value
 

frontend/src/components/TableView/SettingFieldsTableView.vue

@@ -19,13 +19,18 @@
               v-model="scope.row.name"
               :placeholder="$t('Name')"
               suffix-icon="el-icon-edit"
-              @change="onNameChange(scope.row)"
+              @change="onChange(scope.row)"
             />
           </template>
         </el-table-column>
         <el-table-column :label="$t('Value')" width="auto" min-width="120px">
           <template slot-scope="scope">
-            <el-input v-model="scope.row.value" :placeholder="$t('Value')" suffix-icon="el-icon-edit"/>
+            <el-input
+              v-model="scope.row.value"
+              :placeholder="$t('Value')"
+              suffix-icon="el-icon-edit"
+              @change="onChange(scope.row)"
+            />
           </template>
         </el-table-column>
       </el-table>
@@ -83,10 +88,11 @@ export default {
       this.list.splice(index, 1)
       this.$st.sendEv('爬虫详情-配置', '删除字段')
     },
-    onNameChange (row) {
+    onChange (row) {
       if (this.list.filter(d => d.name === row.name).length > 1) {
         this.$message.error(this.$t(`Duplicated field names for ${row.name}`))
       }
+      this.$store.commit('spider/SET_SPIDER_FORM_CONFIG_SETTINGS', this.list)
       this.$st.sendEv('爬虫详情-配置', '更改字段')
     },
     onCheck (row) {
@@ -115,41 +121,26 @@ export default {
         if (!row.attr) this.$set(row, 'attr', 'href')
       }
     },
-    onCopyField (row) {
-      for (let i = 0; i < this.list.length; i++) {
-        if (row.name === this.list[i].name) {
-          this.list.splice(i, 0, JSON.parse(JSON.stringify(row)))
-          break
+    onRemoveField (row) {
+      const list = JSON.parse(JSON.stringify(this.list))
+      for (let i = 0; i < list.length; i++) {
+        if (row.name === list[i].name) {
+          list.splice(i, 0, 1)
         }
       }
-    },
-    onRemoveField (row) {
-      console.log(row)
-      this.$store.commit(
-        'spider/UNSET_SPIDER_FORM_CONFIG_SETTING_ITEM',
-        row.name
-      )
-      if (this.list.length === 0) {
-        this.$store.commit(
-          'spider/SET_SPIDER_FORM_CONFIG_SETTING_ITEM',
-          'VARIABLE_NAME_' + Math.floor(new Date().getTime()),
-          'VARIABLE_VALUE_' + Math.floor(new Date().getTime())
-        )
-      }
+      this.$store.commit('spider/SET_SPIDER_FORM_CONFIG_SETTINGS', list)
     },
     onAddField (row) {
-      for (let i = 0; i < this.list.length; i++) {
-        if (row.name === this.list[i].name) {
-          this.$store.commit(
-            'spider/SET_SPIDER_FORM_CONFIG_SETTING_ITEM',
-            {
-              name: 'VARIABLE_NAME_' + Math.floor(new Date().getTime()),
-              value: 'VARIABLE_VALUE_' + Math.floor(new Date().getTime())
-            }
-          )
+      const list = JSON.parse(JSON.stringify(this.list))
+      for (let i = 0; i < list.length; i++) {
+        if (row.name === list[i].name) {
+          const name = 'VARIABLE_NAME_' + Math.floor(new Date().getTime())
+          const value = 'VARIABLE_VALUE_' + Math.floor(new Date().getTime())
+          list.push({ name, value })
           break
         }
       }
+      this.$store.commit('spider/SET_SPIDER_FORM_CONFIG_SETTINGS', list)
     },
     getCellClassStyle ({ row, columnIndex }) {
       if (columnIndex === 1) {

frontend/src/store/modules/spider.js

@@ -74,13 +74,11 @@ const mutations = {
   SET_PREVIEW_CRAWL_DATA (state, value) {
     state.previewCrawlData = value
   },
-  SET_SPIDER_FORM_CONFIG_SETTING_ITEM (state, payload) {
-    const { name, value } = payload
-    Vue.set(state.spiderForm.config.settings, name, value)
-  },
-  UNSET_SPIDER_FORM_CONFIG_SETTING_ITEM (state, name) {
-    const settings = JSON.parse(JSON.stringify(state.spiderForm.config.settings))
-    delete settings[name]
+  SET_SPIDER_FORM_CONFIG_SETTINGS (state, payload) {
+    const settings = {}
+    payload.forEach(row => {
+      settings[row.name] = row.value
+    })
     Vue.set(state.spiderForm.config, 'settings', settings)
   }
 }