添加Scrapy设置

frontend/src/components/Common/ParametersDialog.vue

@@ -18,6 +18,7 @@
     <el-table
       :data="paramData"
       border
+      :header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
     >
       <el-table-column
         :label="$t('Parameter Type')"

frontend/src/components/Scrapy/SpiderScrapy.vue

@@ -0,0 +1,366 @@
+<template>
+  <div class="spider-scrapy">
+    <el-dialog
+      :title="$t('Parameter Edit')"
+      :visible="dialogVisible"
+      class="setting-param-dialog"
+      width="600px"
+      :before-close="onCloseDialog"
+    >
+      <div class="action-wrapper" style="margin-bottom: 10px;text-align: right">
+        <el-button
+          type="primary"
+          size="small"
+          icon="el-icon-plus"
+          @click="onActiveParamAdd"
+        >
+          {{$t('Add')}}
+        </el-button>
+      </div>
+      <el-table
+        :data="activeParamData"
+      >
+        <el-table-column
+          v-if="activeParam.type === 'object'"
+          :label="$t('Key')"
+        >
+          <template slot-scope="scope">
+            <el-input v-model="scope.row.key" size="small"/>
+          </template>
+        </el-table-column>
+        <el-table-column
+          :label="$t('Value')"
+        >
+          <template slot-scope="scope">
+            <el-input
+              v-model="scope.row.value"
+              size="small"
+              type="number"
+              @change="() => scope.row.value = Number(scope.row.value)"
+            />
+          </template>
+        </el-table-column>
+        <el-table-column
+          :label="$t('Action')"
+          width="60px"
+          align="center"
+        >
+          <template slot-scope="scope">
+            <el-button
+              type="danger"
+              size="mini"
+              icon="el-icon-delete"
+              circle
+              @click="onActiveParamRemove(scope.$index)"
+            />
+          </template>
+        </el-table-column>
+      </el-table>
+      <template slot="footer">
+        <el-button type="plain" size="small" @click="onCloseDialog">{{$t('Cancel')}}</el-button>
+        <el-button type="primary" size="small" @click="onConfirm">
+          {{$t('Confirm')}}
+        </el-button>
+      </template>
+    </el-dialog>
+    <div class="spiders">
+      <h3 class="title">{{$t('Scrapy Spiders')}}</h3>
+      <ul class="spider-list">
+        <li
+          v-for="s in spiderForm.spider_names"
+          :key="s"
+          class="spider-item"
+        >
+          <i class="el-icon-caret-right"></i>
+          {{s}}
+        </li>
+      </ul>
+    </div>
+    <div class="settings">
+      <h3 class="title">{{$t('Settings')}}</h3>
+      <div class="top-action-wrapper">
+        <el-button
+          type="primary"
+          size="small"
+          icon="el-icon-plus"
+          @click="onAdd"
+        >
+          {{$t('Add')}}
+        </el-button>
+        <el-button size="small" type="success" @click="onSave" icon="el-icon-check">
+          {{$t('Save')}}
+        </el-button>
+      </div>
+      <el-table
+        :data="spiderScrapySettings"
+        border
+        :header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
+      >
+        <el-table-column
+          :label="$t('Variable Name')"
+          width="240px"
+        >
+          <template slot-scope="scope">
+            <el-input
+              v-model="scope.row.key"
+              size="small"
+              suffix-icon="el-icon-edit"
+            />
+          </template>
+        </el-table-column>
+        <el-table-column
+          :label="$t('Variable Type')"
+          width="120px"
+        >
+          <template slot-scope="scope">
+            <el-select v-model="scope.row.type" size="small">
+              <el-option value="string" :label="$t('String')"/>
+              <el-option value="number" :label="$t('Number')"/>
+              <el-option value="boolean" :label="$t('Boolean')"/>
+              <el-option value="array" :label="$t('Array/List')"/>
+              <el-option value="object" :label="$t('Object/Dict')"/>
+            </el-select>
+          </template>
+        </el-table-column>
+        <el-table-column
+          :label="$t('Variable Value')"
+          width="calc(100% - 150px)"
+        >
+          <template slot-scope="scope">
+            <el-input
+              v-if="scope.row.type === 'string'"
+              v-model="scope.row.value"
+              size="small"
+              suffix-icon="el-icon-edit"
+            />
+            <el-input
+              v-else-if="scope.row.type === 'number'"
+              type="number"
+              v-model="scope.row.value"
+              size="small"
+              suffix-icon="el-icon-edit"
+            />
+            <div
+              v-else-if="scope.row.type === 'boolean'"
+              style="margin-left: 10px"
+            >
+              <el-switch
+                v-model="scope.row.value"
+                size="small"
+              />
+            </div>
+            <div
+              v-else
+              style="margin-left: 10px;font-size: 12px"
+            >
+              {{JSON.stringify(scope.row.value)}}
+              <el-button
+                type="warning"
+                size="mini"
+                icon="el-icon-edit"
+                style="margin-left: 10px"
+                @click="onEditParam(scope.row, scope.$index)"
+              />
+            </div>
+          </template>
+        </el-table-column>
+        <el-table-column
+          :label="$t('Action')"
+          width="60px"
+          align="center"
+        >
+          <template slot-scope="scope">
+            <el-button
+              type="danger"
+              size="mini"
+              icon="el-icon-delete"
+              circle
+              @click="onRemove(scope.$index)"
+            />
+          </template>
+        </el-table-column>
+      </el-table>
+    </div>
+  </div>
+</template>
+
+<script>
+import {
+  mapState
+} from 'vuex'
+
+export default {
+  name: 'SpiderScrapy',
+  computed: {
+    ...mapState('spider', [
+      'spiderForm',
+      'spiderScrapySettings'
+    ]),
+    activeParamData () {
+      if (this.activeParam.type === 'array') {
+        return this.activeParam.value.map(s => {
+          return { value: s }
+        })
+      } else if (this.activeParam.type === 'object') {
+        return Object.keys(this.activeParam.value).map(key => {
+          return {
+            key,
+            value: this.activeParam.value[key]
+          }
+        })
+      }
+      return []
+    }
+  },
+  data () {
+    return {
+      dialogVisible: false,
+      activeParam: {},
+      activeParamIndex: undefined
+    }
+  },
+  methods: {
+    onOpenDialog () {
+      this.dialogVisible = true
+    },
+    onCloseDialog () {
+      this.dialogVisible = false
+    },
+    onConfirm () {
+      if (this.activeParam.type === 'array') {
+        this.activeParam.value = this.activeParamData.map(d => d.value)
+      } else if (this.activeParam.type === 'object') {
+        const dict = {}
+        this.activeParamData.forEach(d => {
+          dict[d.key] = d.value
+        })
+        this.activeParam.value = dict
+      }
+      this.$set(this.spiderScrapySettings, this.activeParamIndex, JSON.parse(JSON.stringify(this.activeParam)))
+      this.dialogVisible = false
+    },
+    onEditParam (row, index) {
+      this.activeParam = JSON.parse(JSON.stringify(row))
+      this.activeParamIndex = index
+      this.onOpenDialog()
+    },
+    onSave () {
+    },
+    onAdd () {
+      const data = JSON.parse(JSON.stringify(this.spiderScrapySettings))
+      data.push({
+        key: '',
+        value: '',
+        type: 'string'
+      })
+      this.$store.commit('spider/SET_SPIDER_SCRAPY_SETTINGS', data)
+    },
+    onRemove (index) {
+      const data = JSON.parse(JSON.stringify(this.spiderScrapySettings))
+      data.splice(index, 1)
+      this.$store.commit('spider/SET_SPIDER_SCRAPY_SETTINGS', data)
+    },
+    onActiveParamAdd () {
+      if (this.activeParam.type === 'array') {
+        this.activeParam.value.push('')
+      } else if (this.activeParam.type === 'object') {
+        if (!this.activeParam.value) {
+          this.activeParam.value = {}
+        }
+        this.$set(this.activeParam.value, '', 999)
+      }
+    },
+    onActiveParamRemove (index) {
+      if (this.activeParam.type === 'array') {
+        this.activeParam.value.splice(index, 1)
+      } else if (this.activeParam.type === 'object') {
+        const key = this.activeParamData[index].key
+        const value = JSON.parse(JSON.stringify(this.activeParam.value))
+        delete value[key]
+        this.$set(this.activeParam, 'value', value)
+      }
+    }
+  }
+}
+</script>
+
+<style scoped>
+  .spider-scrapy {
+    height: calc(100vh - 200px);
+    color: #606266;
+  }
+
+  .spiders {
+    float: left;
+    display: inline-block;
+    width: 240px;
+    height: 100%;
+    border: 1px solid #DCDFE6;
+    border-radius: 3px;
+    padding: 0 10px;
+  }
+
+  .spiders .title {
+    border-bottom: 1px solid #DCDFE6;
+    padding-bottom: 15px;
+  }
+
+  .spiders .spider-list {
+    list-style: none;
+    padding: 0;
+    margin: 0;
+  }
+
+  .spiders .spider-list .spider-item {
+    padding: 10px;
+    cursor: pointer;
+  }
+
+  .spiders .spider-list .spider-item:hover {
+    background: #F5F7FA;
+  }
+
+  .settings {
+    margin-left: 20px;
+    border: 1px solid #DCDFE6;
+    float: left;
+    width: calc(100% - 240px - 20px);
+    height: 100%;
+    border-radius: 3px;
+    padding: 0 20px;
+  }
+
+  .settings .title {
+    border-bottom: 1px solid #DCDFE6;
+    padding-bottom: 15px;
+  }
+
+  .settings >>> .el-table td,
+  .settings >>> .el-table td .cell {
+    padding: 0;
+    margin: 0;
+  }
+
+  .settings >>> .el-table td .cell .el-autocomplete {
+    width: 100%;
+  }
+
+  .settings >>> .el-table td .cell .el-input__inner {
+    border: none;
+    font-size: 12px;
+  }
+
+  .settings >>> .action-wrapper {
+    margin-top: 10px;
+    text-align: right;
+  }
+
+  .settings >>> .top-action-wrapper {
+    margin-bottom: 10px;
+    text-align: right;
+  }
+
+  .settings >>> .top-action-wrapper .el-button {
+    margin-left: 10px;
+  }
+</style>

frontend/src/components/TableView/TaskTableView.vue

@@ -4,7 +4,13 @@
       <h5 class="title">{{title}}</h5>
       <el-button type="success" plain class="small-btn" size="mini" icon="fa fa-refresh" @click="onRefresh"></el-button>
     </el-row>
-    <el-table border height="480px" :data="taskList" @row-click="onClickTask">
+    <el-table
+      :data="taskList"
+      border
+      :header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
+      height="480px"
+      @row-click="onClickTask"
+    >
       <el-table-column property="node" :label="$t('Node')" width="120" align="left">
         <template slot-scope="scope">
           <a class="a-tag" @click="onClickNode(scope.row)">{{scope.row.node_name}}</a>

frontend/src/i18n/zh.js

@@ -184,11 +184,16 @@ export default {
   'Template': '模版',
   'Is Scrapy': '是否为 Scrapy',
   'Scrapy Spider': 'Scrapy 爬虫',
+  'Scrapy Spiders': 'Scrapy 爬虫',
   'Scrapy Log Level': 'Scrapy 日志等级',
   'Parameter Name': '参数名',
   'Parameter Value': '参数值',
   'Parameter Type': '参数类别',
   'Other': '其他',
+  'Scrapy Config': 'Scrapy 配置',
+  'Variable Name': '变量名',
+  'Variable Type': '变量类型',
+  'Variable Value': '变量值',
 
   // 爬虫列表
   'Name': '名称',
@@ -291,6 +296,13 @@ export default {
   Logout: '退出登录',
   Documentation: '文档',
 
+  // 变量类型
+  'String': '字符串',
+  'Number': '数字',
+  'Boolean': '布尔值',
+  'Array/List': '数组/列表',
+  'Object/Dict': '对象/字典',
+
   // 选择
   'Yes': '是',
   'No': '否',

frontend/src/store/modules/spider.js

@@ -10,6 +10,9 @@ const state = {
   // active spider data
   spiderForm: {},
 
+  // spider scrapy settings
+  spiderScrapySettings: [],
+
   // node to deploy/run
   activeNode: {},
 
@@ -92,6 +95,9 @@ const mutations = {
   },
   SET_FILE_TREE (state, value) {
     state.fileTree = value
+  },
+  SET_SPIDER_SCRAPY_SETTINGS (state, value) {
+    state.spiderScrapySettings = value
   }
 }
 
@@ -121,6 +127,26 @@ const actions = {
     state.spiderForm.spider_names = res.data.data
     commit('SET_SPIDER_FORM', state.spiderForm)
   },
+  async getSpiderScrapySettings ({ state, commit }, id) {
+    const res = await request.get(`/spiders/${id}/scrapy/settings`)
+    commit('SET_SPIDER_SCRAPY_SETTINGS', res.data.data.map(d => {
+      const key = d.key
+      const value = d.value
+      let type = typeof value
+      if (type === 'object') {
+        if (Array.isArray(value)) {
+          type = 'array'
+        } else {
+          type = 'object'
+        }
+      }
+      return {
+        key,
+        value,
+        type
+      }
+    }))
+  },
   crawlSpider ({ state, dispatch }, payload) {
     const { spiderId, runType, nodeIds, param } = payload
     return request.put(`/tasks`, {

frontend/src/utils/scrapy.js

@@ -1,4 +1,32 @@
 export default {
+  importantSettingParamNames: {
+    BOT_NAME: String,
+    SPIDER_MODULES: Array,
+    NEWSPIDER_MODULE: String,
+    USER_AGENT: String,
+    ROBOTSTXT_OBEY: Boolean,
+    CONCURRENT_REQUESTS: Number,
+    DOWNLOAD_DELAY: Number,
+    CONCURRENT_REQUESTS_PER_DOMAIN: Number,
+    CONCURRENT_REQUESTS_PER_IP: Number,
+    COOKIES_ENABLED: Boolean,
+    TELNETCONSOLE_ENABLED: Boolean,
+    DEFAULT_REQUEST_HEADERS: Object,
+    SPIDER_MIDDLEWARES: Object,
+    DOWNLOADER_MIDDLEWARES: Object,
+    EXTENSIONS: Object,
+    ITEM_PIPELINES: Object,
+    AUTOTHROTTLE_ENABLED: Boolean,
+    AUTOTHROTTLE_START_DELAY: Number,
+    AUTOTHROTTLE_MAX_DELAY: Number,
+    AUTOTHROTTLE_TARGET_CONCURRENCY: Number,
+    AUTOTHROTTLE_DEBUG: Boolean,
+    HTTPCACHE_ENABLED: Boolean,
+    HTTPCACHE_EXPIRATION_SECS: Number,
+    HTTPCACHE_DIR: String,
+    HTTPCACHE_IGNORE_HTTP_CODES: Array,
+    HTTPCACHE_STORAGE: String
+  },
   settingParamNames: [
     'AWS_ACCESS_KEY_ID',
     'AWS_SECRET_ACCESS_KEY',

frontend/src/views/spider/SpiderDetail.vue

@@ -22,6 +22,9 @@
       <el-tab-pane :label="$t('Overview')" name="overview">
         <spider-overview/>
       </el-tab-pane>
+      <el-tab-pane v-if="isScrapy" :label="$t('Scrapy Config')" name="scrapy-config">
+        <spider-scrapy/>
+      </el-tab-pane>
       <el-tab-pane v-if="isConfigurable" :label="$t('Config')" name="config">
         <config-list ref="config"/>
       </el-tab-pane>
@@ -51,10 +54,12 @@ import EnvironmentList from '../../components/Environment/EnvironmentList'
 import SpiderStats from '../../components/Stats/SpiderStats'
 import ConfigList from '../../components/Config/ConfigList'
 import SpiderSchedules from './SpiderSchedules'
+import SpiderScrapy from '../../components/Scrapy/SpiderScrapy'
 
 export default {
   name: 'SpiderDetail',
   components: {
+    SpiderScrapy,
     SpiderSchedules,
     ConfigList,
     SpiderStats,
@@ -174,6 +179,9 @@ export default {
     },
     isConfigurable () {
       return this.spiderForm.type === 'configurable'
+    },
+    isScrapy () {
+      return this.isCustomized && this.spiderForm.is_scrapy
     }
   },
   methods: {
@@ -193,6 +201,8 @@ export default {
             this.$st.sendEv('教程', '开始', 'spider-detail-config')
           }, 100)
         }
+      } else if (this.activeTabName === 'scrapy-config') {
+        this.$store.dispatch('spider/getSpiderScrapySpiders', this.$route.params.id)
       }
       this.$st.sendEv('爬虫详情', '切换标签', tab.name)
     },
@@ -220,12 +230,8 @@ export default {
     // get scrapy spider names
     if (this.spiderForm.is_scrapy) {
       await this.$store.dispatch('spider/getSpiderScrapySpiders', this.$route.params.id)
+      await this.$store.dispatch('spider/getSpiderScrapySettings', this.$route.params.id)
     }
-
-    // if spider is configurable spider, set to config tab by default
-    // if (this.spiderForm.type === 'configurable') {
-    // this.activeTabName = 'config'
-    // }
   },
   mounted () {
     if (!this.$utils.tour.isFinishedTour('spider-detail')) {