git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-29 18:00:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

prepared for configurable spiders

Browse Source
This commit is contained in:
Marvin Zhang
2019-05-24 17:28:25 +08:00
parent 9ba5623d62
commit 04789febc8
16 changed files with 513 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
frontend/src/components/Config/ConfigList.vue Normal file
View File

@@ -0,0 +1,53 @@
<template>
<div class="config-list">
<el-row>
<div class="button-group">
<el-button type="primary" @click="addEnv" icon="el-icon-plus">{{$t('Add Environment Variables')}}</el-button>
<el-button type="success" @click="save">{{$t('Save')}}</el-button>
</div>
</el-row>
<el-row>
<el-table :data="spiderForm.fields">
<el-table-column :label="$t('Field Name')">
<template slot-scope="scope">
<el-input v-model="scope.row.name" :placeholder="$t('Variable')"></el-input>
</template>
</el-table-column>
<el-table-column :label="$t('Extract Type')">
<template slot-scope="scope">
<el-input v-model="scope.row.type" :placeholder="$t('Value')"></el-input>
</template>
</el-table-column>
<el-table-column :label="$t('Query')">
<template slot-scope="scope">
<el-input v-model="scope.row.query" :placeholder="$t('Value')"></el-input>
</template>
</el-table-column>
<el-table-column :label="$t('Action')">
<template slot-scope="scope">
<el-button size="mini" icon="el-icon-delete" type="danger" @click="deleteEnv(scope.$index)"></el-button>
</template>
</el-table-column>
</el-table>
</el-row>
</div>
</template>
<script>
import {
mapState
} from 'vuex'
export default {
name: 'ConfigList',
computed: {
...mapState('spider', [
'spiderForm'
])
}
}
</script>
<style scoped>
</style>

30
frontend/src/components/InfoView/SpiderInfoView.vue
View File

@@ -12,10 +12,10 @@
<el-form-item :label="$t('Spider Name')">
<el-input v-model="spiderForm.name" :placeholder="$t('Spider Name')" :disabled="isView"></el-input>
</el-form-item>
<el-form-item :label="$t('Source Folder')">
<el-form-item v-if="isCustomized" :label="$t('Source Folder')">
<el-input v-model="spiderForm.src" :placeholder="$t('Source Folder')" disabled></el-input>
</el-form-item>
<el-form-item :label="$t('Execute Command')" prop="cmd" required :inline-message="true">
<el-form-item v-if="isCustomized" :label="$t('Execute Command')" prop="cmd" required :inline-message="true">
<el-input v-model="spiderForm.cmd" :placeholder="$t('Execute Command')"
:disabled="isView"></el-input>
</el-form-item>
@@ -32,13 +32,12 @@
</el-autocomplete>
</el-form-item>
<el-form-item :label="$t('Spider Type')">
<el-select v-model="spiderForm.type" :placeholder="$t('Spider Type')" :disabled="isView" clearable>
<el-option value="scrapy" label="Scrapy"></el-option>
<el-option value="pyspider" label="PySpider"></el-option>
<el-option value="webmagic" label="WebMagic"></el-option>
<el-select v-model="spiderForm.type" :placeholder="$t('Spider Type')" :disabled="true" clearable>
<el-option value="configurable" :label="$t('Configurable')"></el-option>
<el-option value="customized" :label="$t('Customized')"></el-option>
</el-select>
</el-form-item>
<el-form-item :label="$t('Language')">
<el-form-item v-if="isCustomized" :label="$t('Language')">
<el-select v-model="spiderForm.lang" :placeholder="$t('Language')" :disabled="isView" clearable>
<el-option value="python" label="Python"></el-option>
<el-option value="javascript" label="JavaScript"></el-option>
@@ -50,7 +49,7 @@
</el-row>
<el-row class="button-container" v-if="!isView">
<el-button v-if="isShowRun" type="danger" @click="onRun">{{$t('Run')}}</el-button>
<el-button type="primary" @click="onDeploy">{{$t('Deploy')}}</el-button>
<el-button v-if="isCustomized" type="primary" @click="onDeploy">{{$t('Deploy')}}</el-button>
<el-button type="success" @click="onSave">{{$t('Save')}}</el-button>
</el-row>
</div>
@@ -99,13 +98,18 @@ export default {
'spiderForm'
]),
isShowRun () {
if (!this.spiderForm.deploy_ts) {
if (this.isCustomized) {
if (!this.spiderForm.deploy_ts) {
return false
}
return !!this.spiderForm.cmd
} else {
// TODO: has to add rules
return false
}
if (!this.spiderForm.cmd) {
return false
}
return true
},
isCustomized () {
return this.spiderForm.type === 'customized'
}
},
methods: {

16
frontend/src/i18n/zh.js
View File

@@ -13,16 +13,18 @@ export default {
'Sites': '网站',
// 标签
Overview: '概览',
Files: '文件',
'Overview': '概览',
'Files': '文件',
'Deployed Spiders': '已部署爬虫',
'Log': '日志',
'Results': '结果',
'Environment': '环境',
'Analytics': '分析',
'Rules': '规则',
'Config': '配置',
// 选择
Spider: '爬虫',
'Spider': '爬虫',
// 块标题
'Latest Tasks': '最近任务',
@@ -37,6 +39,7 @@ export default {
REVOKED: '已取消',
// 操作
Add: '添加',
Run: '运行',
Deploy: '部署',
Save: '保存',
@@ -88,6 +91,9 @@ export default {
'Variable': '变量',
'Value': '值',
'Add Environment Variables': '添加环境变量',
'Add Spider': '添加爬虫',
'Add Configurable Spider': '添加可配置爬虫',
'Add Customized Spider': '添加自定义爬虫',
'Last 7-Day Tasks': '最近7天任务数',
'Last 5-Run Errors': '最近5次运行错误数',
'30-Day Tasks': '最近30天任务数',
@@ -98,6 +104,10 @@ export default {
'Tasks by Node': '分节点任务数',
'Daily Tasks': '每日任务数',
'Daily Avg Duration (sec)': '每日平均运行时长(秒)',
'Configurable Spider': '可配置爬虫',
'Customized Spider': '自定义爬虫',
'Configurable': '可配置',
'Customized': '自定义',
// 爬虫列表
'Name': '名称',

9
frontend/src/store/modules/spider.js
View File

@@ -79,13 +79,8 @@ const actions = {
addSpider ({ state, dispatch }) {
return request.put('/spiders', {
name: state.spiderForm.name,
src: state.spiderForm.src,
cmd: state.spiderForm.cmd,
type: state.spiderForm.type,
lang: state.spiderForm.lang,
col: state.spiderForm.col,
cron: state.spiderForm.cron,
cron_enabled: state.spiderForm.cron_enabled ? 1 : 0,
type: 'configurable',
site: state.spiderForm.site
})
.then(() => {
@@ -100,8 +95,6 @@ const actions = {
type: state.spiderForm.type,
lang: state.spiderForm.lang,
col: state.spiderForm.col,
cron: state.spiderForm.cron,
cron_enabled: state.spiderForm.cron_enabled ? 1 : 0,
site: state.spiderForm.site
})
.then(() => {

15
frontend/src/views/spider/SpiderDetail.vue
View File

@@ -13,7 +13,10 @@
<el-tab-pane :label="$t('Overview')" name="overview">
<spider-overview/>
</el-tab-pane>
<el-tab-pane :label="$t('Files')" name="files">
<el-tab-pane v-if="isConfigurable" :label="$t('Config')" name="配置">
<config-list/>
</el-tab-pane>
<el-tab-pane v-if="isCustomized" :label="$t('Files')" name="files">
<file-list/>
</el-tab-pane>
<el-tab-pane :label="$t('Environment')" name="environment">
@@ -34,10 +37,12 @@ import FileList from '../../components/FileList/FileList'
import SpiderOverview from '../../components/Overview/SpiderOverview'
import EnvironmentList from '../../components/Environment/EnvironmentList'
import SpiderStats from '../../components/Stats/SpiderStats'
import ConfigList from '../../components/Config/ConfigList'
export default {
name: 'NodeDetail',
components: {
ConfigList,
SpiderStats,
EnvironmentList,
FileList,
@@ -58,7 +63,13 @@ export default {
]),
...mapState('deploy', [
'deployList'
])
]),
isCustomized () {
return this.spiderForm.type === 'customized'
},
isConfigurable () {
return this.spiderForm.type === 'configurable'
}
},
methods: {
onTabClick () {

154
frontend/src/views/spider/SpiderList.vue
View File

@@ -1,6 +1,6 @@
<template>
<div class="app-container">
<!--add popup-->
<!--import popup-->
<el-dialog
:title="$t('Import Spider')"
:visible.sync="dialogVisible"
@@ -26,6 +26,66 @@
<el-button v-loading="importLoading" type="primary" @click="onImport">{{$t('Import')}}</el-button>
</span>
</el-dialog>
<!--./import popup-->
<!--add dialog-->
<el-dialog :title="$t('Add Spider')"
width="40%"
:visible.sync="addDialogVisible"
:before-close="onAddDialogClose">
<div class="add-spider-wrapper">
<div @click="onAddConfigurable">
<el-card shadow="hover" class="add-spider-item success">
{{$t('Configurable Spider')}}
</el-card>
</div>
<div @click="onAddCustomized">
<el-card shadow="hover" class="add-spider-item primary">
{{$t('Customized Spider')}}
</el-card>
</div>
</div>
</el-dialog>
<!--./add dialog-->
<!--configurable spider dialog-->
<el-dialog :title="$t('Add Configurable Spider')"
width="40%"
:visible.sync="addConfigurableDialogVisible"
:before-close="onAddConfigurableDialogClose">
<el-form :model="spiderForm" ref="addConfigurableForm" inline-message>
<el-form-item :label="$t('Spider Name')" label-width="120px" prop="name" required>
<el-input :placeholder="$t('Spider Name')" v-model="spiderForm.name"></el-input>
</el-form-item>
<el-form-item :label="$t('Results Collection')" label-width="120px" name="col">
<el-input :placeholder="$t('Results Collection')" v-model="spiderForm.col"></el-input>
</el-form-item>
<el-form-item :label="$t('Site')" label-width="120px" name="site">
<el-autocomplete v-model="spiderForm.site"
:placeholder="$t('Site')"
:fetch-suggestions="fetchSiteSuggestions"
@select="onAddConfigurableSiteSelect">
</el-autocomplete>
</el-form-item>
</el-form>
<span slot="footer" class="dialog-footer">
<el-button @click="addConfigurableDialogVisible = false">{{$t('Cancel')}}</el-button>
<el-button v-loading="addConfigurableLoading" type="primary"
@click="onAddConfigurableSpider">{{$t('Add')}}</el-button>
</span>
</el-dialog>
<!--./configurable spider dialog-->
<!--customized spider dialog-->
<el-dialog :title="$t('Add Customized Spider')"
width="40%"
:visible.sync="addCustomizedDialogVisible"
:before-close="onAddCustomizedDialogClose">
<p>
{{$t('Please go to the source folder of your spiders, create a sub-folder and add your spider codes into it')}}
</p>
</el-dialog>
<!--./customized spider dialog-->
<!--filter-->
<div class="filter">
@@ -50,6 +110,12 @@
<el-button type="primary" icon="fa fa-download" @click="openImportDialog">
{{$t('Import Spiders')}}
</el-button>
<el-button type="success"
icon="el-icon-plus"
class="btn add"
@click="onAdd">
{{$t('Add Spider')}}
</el-button>
<el-button type="success"
icon="el-icon-refresh"
class="btn refresh"
@@ -68,14 +134,11 @@
<el-table-column v-if="col.name === 'type'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
align="center"
:width="col.width">
<template slot-scope="scope">
<el-tag v-if="scope.row.type === 'scrapy'">Scrapy</el-tag>
<el-tag type="warning" v-else-if="scope.row.type === 'pyspider'">PySpider</el-tag>
<el-tag type="info" v-else-if="scope.row.type === 'webmagic'">WebMagic</el-tag>
<el-tag type="success" v-else-if="scope.row.type">{{scope.row.type}}</el-tag>
<el-tag type="success" v-if="scope.row.type === 'configurable'">{{$t('Configurable')}}</el-tag>
<el-tag type="primary" v-else-if="scope.row.type === 'customized'">{{$t('Customized')}}</el-tag>
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'lang'"
@@ -123,7 +186,7 @@
<el-tooltip :content="$t('Remove')" placement="top">
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
</el-tooltip>
<el-tooltip :content="$t('Deploy')" placement="top">
<el-tooltip v-if="scope.row.type === 'customized'" :content="$t('Deploy')" placement="top">
<el-button type="primary" icon="fa fa-cloud" size="mini" @click="onDeploy(scope.row)"></el-button>
</el-tooltip>
<el-tooltip v-if="isShowRun(scope.row)" :content="$t('Run')" placement="top">
@@ -160,8 +223,12 @@ export default {
pageSize: 10
},
importLoading: false,
addConfigurableLoading: false,
isEditMode: false,
dialogVisible: false,
addDialogVisible: false,
addConfigurableDialogVisible: false,
addCustomizedDialogVisible: false,
filter: {
keyword: ''
},
@@ -169,7 +236,7 @@ export default {
columns: [
{ name: 'name', label: 'Name', width: 'auto' },
{ name: 'site_name', label: 'Site', width: '120' },
{ name: 'type', label: 'Spider Type', width: '120', sortable: true },
{ name: 'type', label: 'Spider Type', width: '120' },
{ name: 'lang', label: 'Language', width: '120', sortable: true },
{ name: 'task_ts', label: 'Last Run', width: '160' },
{ name: 'last_7d_tasks', label: 'Last 7-Day Tasks', width: '80' },
@@ -219,9 +286,16 @@ export default {
console.log(value)
},
onAdd () {
this.addDialogVisible = true
},
onAddConfigurable () {
this.$store.commit('spider/SET_SPIDER_FORM', {})
this.isEditMode = false
this.dialogVisible = true
this.addDialogVisible = false
this.addConfigurableDialogVisible = true
},
onAddCustomized () {
this.addDialogVisible = false
this.addCustomizedDialogVisible = true
},
onRefresh () {
this.$store.dispatch('spider/getSpiderList')
@@ -246,10 +320,22 @@ export default {
this.$store.commit('spider/SET_SPIDER_FORM', {})
this.dialogVisible = false
},
onAddCancel () {
this.addDialogVisible = false
},
onDialogClose () {
this.$store.commit('spider/SET_SPIDER_FORM', {})
this.dialogVisible = false
},
onAddDialogClose () {
this.addDialogVisible = false
},
onAddCustomizedDialogClose () {
this.addCustomizedDialogVisible = false
},
onAddConfigurableDialogClose () {
this.addConfigurableDialogVisible = false
},
onEdit (row) {
this.isEditMode = true
this.$store.commit('spider/SET_SPIDER_FORM', row)
@@ -363,6 +449,21 @@ export default {
},
onSiteSelect (item) {
this.$store.commit('spider/SET_FILTER_SITE', item._id)
},
onAddConfigurableSiteSelect (item) {
this.spiderForm.site = item._id
},
onAddConfigurableSpider () {
this.$refs['addConfigurableForm'].validate(res => {
if (res) {
this.addConfigurableLoading = true
this.$store.dispatch('spider/addSpider')
.finally(() => {
this.addConfigurableLoading = false
this.addConfigurableDialogVisible = false
})
}
})
}
},
created () {
@@ -412,4 +513,37 @@ export default {
background-color: red;
}
.add-spider-wrapper {
display: flex;
justify-content: center;
.add-spider-item {
cursor: pointer;
width: 180px;
font-size: 18px;
height: 120px;
margin: 0 20px;
flex-basis: 40%;
display: flex;
align-items: center;
justify-content: center;
}
.add-spider-item.primary {
color: #409eff;
background: rgba(64, 158, 255, .1);
border: 1px solid rgba(64, 158, 255, .1);
}
.add-spider-item.success {
color: #67c23a;
background: rgba(103, 194, 58, .1);
border: 1px solid rgba(103, 194, 58, .1);
}
}
.el-autocomplete {
width: 100%;
}
</style>