From c3bc20127dd183df6aa01ff08b08a9a49538a582 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 2 Dec 2019 22:35:45 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=8F=AF=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E7=88=AC=E8=99=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../template/scrapy/config_spider/settings.py | 17 ++++-- .../TableView/SettingFieldsTableView.vue | 53 ++++++++----------- frontend/src/store/modules/spider.js | 12 ++--- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/backend/template/scrapy/config_spider/settings.py b/backend/template/scrapy/config_spider/settings.py index c5cb4a1e..fab0080f 100644 --- a/backend/template/scrapy/config_spider/settings.py +++ b/backend/template/scrapy/config_spider/settings.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import os +import re # Scrapy settings for config_spider project # @@ -17,7 +18,7 @@ NEWSPIDER_MODULE = 'config_spider.spiders' # Crawl responsibly by identifying yourself (and your website) on the user-agent -#USER_AGENT = 'config_spider (+http://www.yourdomain.com)' +USER_AGENT = 'Crawlab Spider' # Obey robots.txt rules ROBOTSTXT_OBEY = True @@ -90,8 +91,18 @@ ITEM_PIPELINES = { #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' -for setting_env_name in [x for x in os.environ.keys() if x.startswith('CRAWALAB_SETTING_')]: +for setting_env_name in [x for x in os.environ.keys() if x.startswith('CRAWLAB_SETTING_')]: setting_name = setting_env_name.replace('CRAWLAB_SETTING_', '') - setting_value = os.environ.get('setting_env_name') + setting_value = os.environ.get(setting_env_name) + if setting_value.lower() == 'true': + setting_value = True + elif setting_value.lower() == 'false': + setting_value = False + elif re.search(r'^\d+$', setting_value) is not None: + setting_value = int(setting_value) + else: + pass + print(setting_name) + print(setting_value) locals()[setting_name] = setting_value diff --git a/frontend/src/components/TableView/SettingFieldsTableView.vue b/frontend/src/components/TableView/SettingFieldsTableView.vue index 2a4ce331..701f8ef5 100644 --- a/frontend/src/components/TableView/SettingFieldsTableView.vue +++ b/frontend/src/components/TableView/SettingFieldsTableView.vue @@ -19,13 +19,18 @@ v-model="scope.row.name" :placeholder="$t('Name')" suffix-icon="el-icon-edit" - @change="onNameChange(scope.row)" + @change="onChange(scope.row)" /> @@ -83,10 +88,11 @@ export default { this.list.splice(index, 1) this.$st.sendEv('爬虫详情-配置', '删除字段') }, - onNameChange (row) { + onChange (row) { if (this.list.filter(d => d.name === row.name).length > 1) { this.$message.error(this.$t(`Duplicated field names for ${row.name}`)) } + this.$store.commit('spider/SET_SPIDER_FORM_CONFIG_SETTINGS', this.list) this.$st.sendEv('爬虫详情-配置', '更改字段') }, onCheck (row) { @@ -115,41 +121,26 @@ export default { if (!row.attr) this.$set(row, 'attr', 'href') } }, - onCopyField (row) { - for (let i = 0; i < this.list.length; i++) { - if (row.name === this.list[i].name) { - this.list.splice(i, 0, JSON.parse(JSON.stringify(row))) - break + onRemoveField (row) { + const list = JSON.parse(JSON.stringify(this.list)) + for (let i = 0; i < list.length; i++) { + if (row.name === list[i].name) { + list.splice(i, 0, 1) } } - }, - onRemoveField (row) { - console.log(row) - this.$store.commit( - 'spider/UNSET_SPIDER_FORM_CONFIG_SETTING_ITEM', - row.name - ) - if (this.list.length === 0) { - this.$store.commit( - 'spider/SET_SPIDER_FORM_CONFIG_SETTING_ITEM', - 'VARIABLE_NAME_' + Math.floor(new Date().getTime()), - 'VARIABLE_VALUE_' + Math.floor(new Date().getTime()) - ) - } + this.$store.commit('spider/SET_SPIDER_FORM_CONFIG_SETTINGS', list) }, onAddField (row) { - for (let i = 0; i < this.list.length; i++) { - if (row.name === this.list[i].name) { - this.$store.commit( - 'spider/SET_SPIDER_FORM_CONFIG_SETTING_ITEM', - { - name: 'VARIABLE_NAME_' + Math.floor(new Date().getTime()), - value: 'VARIABLE_VALUE_' + Math.floor(new Date().getTime()) - } - ) + const list = JSON.parse(JSON.stringify(this.list)) + for (let i = 0; i < list.length; i++) { + if (row.name === list[i].name) { + const name = 'VARIABLE_NAME_' + Math.floor(new Date().getTime()) + const value = 'VARIABLE_VALUE_' + Math.floor(new Date().getTime()) + list.push({ name, value }) break } } + this.$store.commit('spider/SET_SPIDER_FORM_CONFIG_SETTINGS', list) }, getCellClassStyle ({ row, columnIndex }) { if (columnIndex === 1) { diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js index 161057a5..10c4657e 100644 --- a/frontend/src/store/modules/spider.js +++ b/frontend/src/store/modules/spider.js @@ -74,13 +74,11 @@ const mutations = { SET_PREVIEW_CRAWL_DATA (state, value) { state.previewCrawlData = value }, - SET_SPIDER_FORM_CONFIG_SETTING_ITEM (state, payload) { - const { name, value } = payload - Vue.set(state.spiderForm.config.settings, name, value) - }, - UNSET_SPIDER_FORM_CONFIG_SETTING_ITEM (state, name) { - const settings = JSON.parse(JSON.stringify(state.spiderForm.config.settings)) - delete settings[name] + SET_SPIDER_FORM_CONFIG_SETTINGS (state, payload) { + const settings = {} + payload.forEach(row => { + settings[row.name] = row.value + }) Vue.set(state.spiderForm.config, 'settings', settings) } }