diff --git a/backend/entity/config_spider.go b/backend/entity/config_spider.go index d46e092e..095a4328 100644 --- a/backend/entity/config_spider.go +++ b/backend/entity/config_spider.go @@ -6,6 +6,7 @@ type Field struct { Xpath string `yaml:"xpath" json:"xpath"` Attr string `yaml:"attr" json:"attr"` NextStage string `yaml:"next_stage" json:"next_stage"` + Remark string `yaml:"remark" json:"remark"` } type Stage struct { diff --git a/frontend/package.json b/frontend/package.json index 5f19fd7b..724b5e36 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -23,7 +23,7 @@ "cross-env": "^5.2.0", "dayjs": "^1.8.6", "echarts": "^4.1.0", - "element-ui": "2.4.6", + "element-ui": "2.13.0", "font-awesome": "^4.7.0", "js-cookie": "2.2.0", "normalize.css": "7.0.0", diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue index 8cab5294..4ba92f3f 100644 --- a/frontend/src/components/Config/ConfigList.vue +++ b/frontend/src/components/Config/ConfigList.vue @@ -42,97 +42,159 @@ - + - - - - {{$t(type.label)}} - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - {{$t('Run')}} - {{$t('Extract Fields')}} - - {{$t('Preview')}} - {{$t('Save')}} - - - + + + + + + CSS + + + + + XPath + + + + - - - + + + + + {{$t('Run')}} + {{$t('Extract Fields')}} + + {{$t('Preview')}} + {{$t('Save')}} + + + + + + + + + {{stageName}} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + + + + + @@ -146,9 +208,13 @@ import CrawlConfirmDialog from '../Common/CrawlConfirmDialog' export default { name: 'ConfigList', - components: { CrawlConfirmDialog, FieldsTableView }, + components: { + CrawlConfirmDialog, + FieldsTableView + }, data () { return { + activeNames: [], crawlTypeList: [ { value: 'list', label: 'List Only' }, { value: 'detail', label: 'Detail Only' }, @@ -159,7 +225,15 @@ export default { saveLoading: false, dialogVisible: false, crawlConfirmDialogVisible: false, - columnsDict: {} + columnsDict: {}, + fieldColumns: [ + { name: 'name', label: 'Name' }, + { name: 'selector_type', label: 'Selector Type' }, + { name: 'selector', label: 'Selector' }, + { name: 'is_attr', label: 'Is Attribute' }, + { name: 'attr', label: 'Attribute' }, + { name: 'next_stage', label: 'Next Stage' } + ] } }, computed: { @@ -177,6 +251,24 @@ export default { } else { return [] } + }, + isCss () { + let i = 0 + Object.values(this.spiderForm.config.stages).forEach(stage => { + stage.fields.forEach(field => { + if (!field.css) i++ + }) + }) + return i === 0 + }, + isXpath () { + let i = 0 + Object.values(this.spiderForm.config.stages).forEach(stage => { + stage.fields.forEach(field => { + if (!field.xpath) i++ + }) + }) + return i === 0 } }, methods: { @@ -294,7 +386,20 @@ export default { value = value.trim() if (value.length > 20) return value.substr(0, 20) + '...' return value - } + }, + onClickSelectorType (selectorType) { + Object.values(this.spiderForm.config.stages).forEach(stage => { + stage.fields.forEach(field => { + if (selectorType === 'css') { + if (field.xpath) field.xpath = '' + if (!field.css) field.css = 'body' + } else { + if (field.css) field.css = '' + if (!field.xpath) field.xpath = '//body' + } + }) + }) + }, }, created () { // fields for list page @@ -327,6 +432,9 @@ export default { if (!this.spiderForm.pagination_selector_type) this.$set(this.spiderForm, 'pagination_selector_type', 'css') if (this.spiderForm.obey_robots_txt == null) this.$set(this.spiderForm, 'obey_robots_txt', true) if (this.spiderForm.item_threshold == null) this.$set(this.spiderForm, 'item_threshold', 10) + }, + mounted () { + this.activeNames = Object.keys(this.spiderForm.config.stages) } } @@ -335,7 +443,7 @@ export default { .button-group-container { margin-top: 10px; - border-bottom: 1px dashed #dcdfe6; + /*border-bottom: 1px dashed #dcdfe6;*/ padding-bottom: 20px; } @@ -345,7 +453,7 @@ export default { .list-fields-container { margin-top: 20px; - border-bottom: 1px dashed #dcdfe6; + /*border-bottom: 1px dashed #dcdfe6;*/ padding-bottom: 20px; } @@ -369,4 +477,17 @@ export default { .el-table.table-header >>> .el-input .el-input__inner { border-radius: 0; } + + .selector-type-item { + margin: 0 5px; + cursor: pointer; + } + + .selector-type-item > .el-tag.active { + font-weight: bolder; + } + + .selector-type-item > .el-tag.inactive { + opacity: 0.5; + } diff --git a/frontend/src/components/TableView/FieldsTableView.vue b/frontend/src/components/TableView/FieldsTableView.vue index 836a8f49..7b1ef9f5 100644 --- a/frontend/src/components/TableView/FieldsTableView.vue +++ b/frontend/src/components/TableView/FieldsTableView.vue @@ -1,73 +1,104 @@ - - {{$t(this.title)}} - - {{$t('Add Field')}} - - + + + + + + - + > + - - + + + - + - + - - - - + + + CSS + + + + + XPath + + - + - - - - - - - - - - - - - - - - + + + - + - - - - + + + {{$t('Text')}} + + + + + {{$t('Attribute')}} + + + + + + + + + + + + N/A + + + + + + + + + + + + + + + @@ -91,6 +122,12 @@ export default { type: String, default: '' }, + stageNames: { + type: Array, + default () { + return [] + } + }, fields: { type: Array, default () { @@ -128,6 +165,57 @@ export default { } }) this.$st.sendEv('爬虫详情-配置', '设置详情页URL') + }, + onClickSelectorType (row, selectorType) { + if (selectorType === 'css') { + if (row.xpath) row.xpath = '' + if (!row.css) row.css = 'body' + } else { + if (row.css) row.css = '' + if (!row.xpath) row.xpath = '//body' + } + }, + onClickIsAttribute (row, isAttribute) { + if (!isAttribute) { + // 文本 + if (row.attr) row.attr = '' + } else { + // 属性 + if (!row.attr) row.attr = 'href' + } + }, + onCopyField (row) { + for (let i = 0; i < this.fields.length; i++) { + if (row.name === this.fields[i].name) { + this.fields.splice(i, 0, JSON.parse(JSON.stringify(row))) + break + } + } + }, + onRemoveField (row) { + for (let i = 0; i < this.fields.length; i++) { + if (row.name === this.fields[i].name) { + this.fields.splice(i, 1) + break + } + } + if (this.fields.length === 0) { + this.fields.push({ + css: 'body', + next_stage: '' + }) + } + }, + onAddField (row) { + for (let i = 0; i < this.fields.length; i++) { + if (row.name === this.fields[i].name) { + this.fields.splice(i + 1, 0, { + css: 'body', + next_stage: '' + }) + break + } + } } } } @@ -158,6 +246,50 @@ export default { line-height: 36px; } + .el-table.edit >>> .button-selector-item { + cursor: pointer; + margin: 0 5px; + } + + .el-table.edit >>> .el-tag.inactive { + opacity: 0.5; + } + + .el-table.edit >>> .action { + background: none !important; + border: none; + } + + .el-table.edit >>> tr { + border: none; + } + + .el-table.edit >>> tr th { + border-right: 1px solid rgb(220, 223, 230); + } + + .el-table.edit >>> tr td:nth-child(2) { + border-left: 1px solid rgb(220, 223, 230); + } + + .el-table.edit >>> tr td { + border-right: 1px solid rgb(220, 223, 230); + } + + .el-table.edit::before { + background: none; + } + + .el-table.edit >>> .action-item { + font-size: 14px; + margin-right: 5px; + cursor: pointer; + } + + .el-table.edit >>> .action-item:last-child { + margin-right: 10px; + } + .button-group-container { /*display: inline-block;*/ /*width: 100%;*/ diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 37ca04de..22563796 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -111,17 +111,24 @@ - - - - - + + + + + + + + + {{$t('Search')}} + + @@ -134,18 +141,13 @@ @click="onAdd"> {{$t('Add Spider')}} - - {{$t('Refresh')}} - + - + @@ -282,10 +284,9 @@ export default { activeSpiderId: undefined, filter: { keyword: '', - type: '' + type: 'all' }, types: [], - // tableData, columns: [ { name: 'display_name', label: 'Name', width: '160', align: 'left' }, { name: 'type', label: 'Spider Type', width: '120' }, @@ -297,8 +298,7 @@ export default { spiderFormRules: { name: [{ required: true, message: 'Required Field', trigger: 'change' }] }, - fileList: [], - activeTab: 'all' + fileList: [] } }, computed: { @@ -499,6 +499,10 @@ export default { this.onView(row) } }, + onClickTab (tab) { + this.filter.type = tab.name + this.getList() + }, getList () { let params = { pageNum: this.pagination.pageNum, diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 8697e8e2..ef361b2b 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -2954,9 +2954,10 @@ electron-to-chromium@^1.3.103: version "1.3.113" resolved "http://registry.npm.taobao.org/electron-to-chromium/download/electron-to-chromium-1.3.113.tgz#b1ccf619df7295aea17bc6951dc689632629e4a9" -element-ui@2.4.6: - version "2.4.6" - resolved "https://registry.yarnpkg.com/element-ui/-/element-ui-2.4.6.tgz#524d3d4cac0b68745dda87311ef0d8fe541b5fc4" +element-ui@2.13.0: + version "2.13.0" + resolved "https://registry.npm.taobao.org/element-ui/download/element-ui-2.13.0.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Felement-ui%2Fdownload%2Felement-ui-2.13.0.tgz#f6bb04e5b0a76ea5f62466044b774407ba4ebd2d" + integrity sha1-9rsE5bCnbqX2JGYES3dEB7pOvS0= dependencies: async-validator "~1.8.1" babel-helper-vue-jsx-merge-props "^2.0.0"