From 0edda7b5585c28aab84bace26e2d98fa5fbed69d Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 27 Nov 2019 12:18:51 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E5=89=8D=E7=AB=AF=E5=8F=AF?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E7=BC=96=E8=BE=91=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/entity/config_spider.go | 1 + frontend/package.json | 2 +- frontend/src/components/Config/ConfigList.vue | 281 +++++++++++++----- .../components/TableView/FieldsTableView.vue | 224 +++++++++++--- frontend/src/views/spider/SpiderList.vue | 38 +-- frontend/yarn.lock | 7 +- 6 files changed, 406 insertions(+), 147 deletions(-) diff --git a/backend/entity/config_spider.go b/backend/entity/config_spider.go index d46e092e..095a4328 100644 --- a/backend/entity/config_spider.go +++ b/backend/entity/config_spider.go @@ -6,6 +6,7 @@ type Field struct { Xpath string `yaml:"xpath" json:"xpath"` Attr string `yaml:"attr" json:"attr"` NextStage string `yaml:"next_stage" json:"next_stage"` + Remark string `yaml:"remark" json:"remark"` } type Stage struct { diff --git a/frontend/package.json b/frontend/package.json index 5f19fd7b..724b5e36 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -23,7 +23,7 @@ "cross-env": "^5.2.0", "dayjs": "^1.8.6", "echarts": "^4.1.0", - "element-ui": "2.4.6", + "element-ui": "2.13.0", "font-awesome": "^4.7.0", "js-cookie": "2.2.0", "normalize.css": "7.0.0", diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue index 8cab5294..4ba92f3f 100644 --- a/frontend/src/components/Config/ConfigList.vue +++ b/frontend/src/components/Config/ConfigList.vue @@ -42,97 +42,159 @@ - + - - - - {{$t(type.label)}} - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - -
- {{$t('Run')}} - {{$t('Extract Fields')}} - - {{$t('Preview')}} - {{$t('Save')}} -
- - + + + + + + CSS + + + + + XPath + + + + - - - + + + +
+ {{$t('Run')}} + {{$t('Extract Fields')}} + + {{$t('Preview')}} + {{$t('Save')}} +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + + + + + @@ -146,9 +208,13 @@ import CrawlConfirmDialog from '../Common/CrawlConfirmDialog' export default { name: 'ConfigList', - components: { CrawlConfirmDialog, FieldsTableView }, + components: { + CrawlConfirmDialog, + FieldsTableView + }, data () { return { + activeNames: [], crawlTypeList: [ { value: 'list', label: 'List Only' }, { value: 'detail', label: 'Detail Only' }, @@ -159,7 +225,15 @@ export default { saveLoading: false, dialogVisible: false, crawlConfirmDialogVisible: false, - columnsDict: {} + columnsDict: {}, + fieldColumns: [ + { name: 'name', label: 'Name' }, + { name: 'selector_type', label: 'Selector Type' }, + { name: 'selector', label: 'Selector' }, + { name: 'is_attr', label: 'Is Attribute' }, + { name: 'attr', label: 'Attribute' }, + { name: 'next_stage', label: 'Next Stage' } + ] } }, computed: { @@ -177,6 +251,24 @@ export default { } else { return [] } + }, + isCss () { + let i = 0 + Object.values(this.spiderForm.config.stages).forEach(stage => { + stage.fields.forEach(field => { + if (!field.css) i++ + }) + }) + return i === 0 + }, + isXpath () { + let i = 0 + Object.values(this.spiderForm.config.stages).forEach(stage => { + stage.fields.forEach(field => { + if (!field.xpath) i++ + }) + }) + return i === 0 } }, methods: { @@ -294,7 +386,20 @@ export default { value = value.trim() if (value.length > 20) return value.substr(0, 20) + '...' return value - } + }, + onClickSelectorType (selectorType) { + Object.values(this.spiderForm.config.stages).forEach(stage => { + stage.fields.forEach(field => { + if (selectorType === 'css') { + if (field.xpath) field.xpath = '' + if (!field.css) field.css = 'body' + } else { + if (field.css) field.css = '' + if (!field.xpath) field.xpath = '//body' + } + }) + }) + }, }, created () { // fields for list page @@ -327,6 +432,9 @@ export default { if (!this.spiderForm.pagination_selector_type) this.$set(this.spiderForm, 'pagination_selector_type', 'css') if (this.spiderForm.obey_robots_txt == null) this.$set(this.spiderForm, 'obey_robots_txt', true) if (this.spiderForm.item_threshold == null) this.$set(this.spiderForm, 'item_threshold', 10) + }, + mounted () { + this.activeNames = Object.keys(this.spiderForm.config.stages) } } @@ -335,7 +443,7 @@ export default { .button-group-container { margin-top: 10px; - border-bottom: 1px dashed #dcdfe6; + /*border-bottom: 1px dashed #dcdfe6;*/ padding-bottom: 20px; } @@ -345,7 +453,7 @@ export default { .list-fields-container { margin-top: 20px; - border-bottom: 1px dashed #dcdfe6; + /*border-bottom: 1px dashed #dcdfe6;*/ padding-bottom: 20px; } @@ -369,4 +477,17 @@ export default { .el-table.table-header >>> .el-input .el-input__inner { border-radius: 0; } + + .selector-type-item { + margin: 0 5px; + cursor: pointer; + } + + .selector-type-item > .el-tag.active { + font-weight: bolder; + } + + .selector-type-item > .el-tag.inactive { + opacity: 0.5; + } diff --git a/frontend/src/components/TableView/FieldsTableView.vue b/frontend/src/components/TableView/FieldsTableView.vue index 836a8f49..7b1ef9f5 100644 --- a/frontend/src/components/TableView/FieldsTableView.vue +++ b/frontend/src/components/TableView/FieldsTableView.vue @@ -1,73 +1,104 @@