git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

adapt changes to golang api

Browse Source
This commit is contained in:
Marvin Zhang
2019-07-21 16:48:58 +08:00
parent 5150a6a760
commit 67c299e5d9
40 changed files with 263 additions and 2004 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
frontend/.env.development
View File

@@ -1,2 +1,2 @@
NODE_ENV='development'
VUE_APP_BASE_URL=http://localhost:8000/api
VUE_APP_BASE_URL=http://localhost:8000

4
frontend/package.json
View File

@@ -3,8 +3,8 @@
"version": "0.2.3",
"private": true,
"scripts": {
"serve": "cross-env NODE_ENV=development vue-cli-service serve --ip=0.0.0.0",
"serve:prod": "cross-env NODE_ENV=production vue-cli-service serve --mode=production --ip=0.0.0.0",
"serve": "vue-cli-service serve --ip=0.0.0.0",
"serve:prod": "vue-cli-service serve --mode=production --ip=0.0.0.0",
"config": "vue ui",
"build:prod": "vue-cli-service build --mode production",
"lint": "vue-cli-service lint",

157
frontend/src/components/DndList/index.vue
View File

@@ -1,157 +0,0 @@
<template>
<div class="dndList">
<div :style="{width:width1}" class="dndList-list">
<h3>{{ list1Title }}</h3>
<draggable :list="list1" :options="{group:'article'}" class="dragArea">
<div v-for="element in list1" :key="element.id" class="list-complete-item">
<div class="list-complete-item-handle">{{ element.id }}[{{ element.author }}] {{ element.title }}</div>
<div style="position:absolute;right:0px;">
<span style="float: right ;margin-top: -20px;margin-right:5px;" @click="deleteEle(element)">
<i style="color:#ff4949" class="el-icon-delete"/>
</span>
</div>
</div>
</draggable>
</div>
<div :style="{width:width2}" class="dndList-list">
<h3>{{ list2Title }}</h3>
<draggable :list="list2" :options="{group:'article'}" class="dragArea">
<div v-for="element in list2" :key="element.id" class="list-complete-item">
<div class="list-complete-item-handle2" @click="pushEle(element)">{{ element.id }} [{{ element.author }}] {{ element.title }}</div>
</div>
</draggable>
</div>
</div>
</template>
<script>
import draggable from 'vuedraggable'
export default {
name: 'DndList',
components: { draggable },
props: {
list1: {
type: Array,
default() {
return []
}
},
list2: {
type: Array,
default() {
return []
}
},
list1Title: {
type: String,
default: 'list1'
},
list2Title: {
type: String,
default: 'list2'
},
width1: {
type: String,
default: '48%'
},
width2: {
type: String,
default: '48%'
}
},
methods: {
isNotInList1(v) {
return this.list1.every(k => v.id !== k.id)
},
isNotInList2(v) {
return this.list2.every(k => v.id !== k.id)
},
deleteEle(ele) {
for (const item of this.list1) {
if (item.id === ele.id) {
const index = this.list1.indexOf(item)
this.list1.splice(index, 1)
break
}
}
if (this.isNotInList2(ele)) {
this.list2.unshift(ele)
}
},
pushEle(ele) {
for (const item of this.list2) {
if (item.id === ele.id) {
const index = this.list2.indexOf(item)
this.list2.splice(index, 1)
break
}
}
if (this.isNotInList1(ele)) {
this.list1.push(ele)
}
}
}
}
</script>
<style rel="stylesheet/scss" lang="scss" scoped>
.dndList {
background: #fff;
padding-bottom: 40px;
&:after {
content: "";
display: table;
clear: both;
}
.dndList-list {
float: left;
padding-bottom: 30px;
&:first-of-type {
margin-right: 2%;
}
.dragArea {
margin-top: 15px;
min-height: 50px;
padding-bottom: 30px;
}
}
}
.list-complete-item {
cursor: pointer;
position: relative;
font-size: 14px;
padding: 5px 12px;
margin-top: 4px;
border: 1px solid #bfcbd9;
transition: all 1s;
}
.list-complete-item-handle {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
margin-right: 50px;
}
.list-complete-item-handle2 {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
margin-right: 20px;
}
.list-complete-item.sortable-chosen {
background: #4AB7BD;
}
.list-complete-item.sortable-ghost {
background: #30B08F;
}
.list-complete-enter,
.list-complete-leave-active {
opacity: 0;
}
</style>

63
frontend/src/components/ErrorLog/index.vue
View File

@@ -1,63 +0,0 @@
<template>
<div v-if="errorLogs.length>0">
<el-badge :is-dot="true" style="line-height: 25px;margin-top: -5px;" @click.native="dialogTableVisible=true">
<el-button style="padding: 8px 10px;" size="small" type="danger">
<svg-icon icon-class="bug" />
</el-button>
</el-badge>
<el-dialog :visible.sync="dialogTableVisible" title="Error Log" width="80%">
<el-table :data="errorLogs" border>
<el-table-column label="Message">
<template slot-scope="scope">
<div>
<span class="message-title">Msg:</span>
<el-tag type="danger">{{ scope.row.err.message }}</el-tag>
</div>
<br>
<div>
<span class="message-title" style="padding-right: 10px;">Info: </span>
<el-tag type="warning">{{ scope.row.vm.$vnode.tag }} error in {{ scope.row.info }}</el-tag>
</div>
<br>
<div>
<span class="message-title" style="padding-right: 16px;">Url: </span>
<el-tag type="success">{{ scope.row.url }}</el-tag>
</div>
</template>
</el-table-column>
<el-table-column label="Stack">
<template slot-scope="scope">
{{ scope.row.err.stack }}
</template>
</el-table-column>
</el-table>
</el-dialog>
</div>
</template>
<script>
export default {
name: 'ErrorLog',
data() {
return {
dialogTableVisible: false
}
},
computed: {
errorLogs() {
return this.$store.getters.errorLogs
}
}
}
</script>
<style scoped>
.message-title {
font-size: 16px;
color: #333;
font-weight: bold;
padding-right: 8px;
}
</style>

51
frontend/src/components/GithubCorner/index.vue
View File

@@ -1,51 +0,0 @@
<template>
<a href="https://github.com/PanJiaChen/vue-element-admin" target="_blank" class="github-corner" aria-label="View source on Github">
<svg
width="80"
height="80"
viewBox="0 0 250 250"
style="fill:#40c9c6; color:#fff;"
aria-hidden="true">
<path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"/>
<path
d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2"
fill="currentColor"
style="transform-origin: 130px 106px;"
class="octo-arm"/>
<path
d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z"
fill="currentColor"
class="octo-body"/>
</svg>
</a>
</template>
<style scoped>
.github-corner:hover .octo-arm {
animation: octocat-wave 560ms ease-in-out
}
@keyframes octocat-wave {
0%,
100% {
transform: rotate(0)
}
20%,
60% {
transform: rotate(-25deg)
}
40%,
80% {
transform: rotate(10deg)
}
}
@media (max-width:500px) {
.github-corner:hover .octo-arm {
animation: none
}
.github-corner .octo-arm {
animation: octocat-wave 560ms ease-in-out
}
}
</style>

187
frontend/src/components/HeaderSearch/index.vue
View File

@@ -1,187 +0,0 @@
<template>
<div :class="{'show':show}" class="header-search">
<svg-icon class-name="search-icon" icon-class="search" @click="click" />
<el-select
ref="headerSearchSelect"
v-model="search"
:remote-method="querySearch"
filterable
default-first-option
remote
placeholder="Search"
class="header-search-select"
@change="change">
<el-option v-for="item in options" :key="item.path" :value="item" :label="item.title.join(' > ')"/>
</el-select>
</div>
</template>
<script>
import Fuse from 'fuse.js'
import path from 'path'
import i18n from '@/lang'
export default {
name: 'HeaderSearch',
data() {
return {
search: '',
options: [],
searchPool: [],
show: false,
fuse: undefined
}
},
computed: {
routers() {
return this.$store.getters.permission_routers
},
lang() {
return this.$store.getters.language
}
},
watch: {
lang() {
this.searchPool = this.generateRouters(this.routers)
},
routers() {
this.searchPool = this.generateRouters(this.routers)
},
searchPool(list) {
this.initFuse(list)
},
show(value) {
if (value) {
document.body.addEventListener('click', this.close)
} else {
document.body.removeEventListener('click', this.close)
}
}
},
mounted() {
this.searchPool = this.generateRouters(this.routers)
},
methods: {
click() {
this.show = !this.show
if (this.show) {
this.$refs.headerSearchSelect && this.$refs.headerSearchSelect.focus()
}
},
close() {
this.$refs.headerSearchSelect && this.$refs.headerSearchSelect.blur()
this.options = []
this.show = false
},
change(val) {
this.$router.push(val.path)
this.search = ''
this.options = []
this.$nextTick(() => {
this.show = false
})
},
initFuse(list) {
this.fuse = new Fuse(list, {
shouldSort: true,
threshold: 0.4,
location: 0,
distance: 100,
maxPatternLength: 32,
minMatchCharLength: 1,
keys: [{
name: 'title',
weight: 0.7
}, {
name: 'path',
weight: 0.3
}]
})
},
// Filter out the routes that can be displayed in the sidebar
// And generate the internationalized title
generateRouters(routers, basePath = '/', prefixTitle = []) {
let res = []
for (const router of routers) {
// skip hidden router
if (router.hidden) { continue }
const data = {
path: path.resolve(basePath, router.path),
title: [...prefixTitle]
}
if (router.meta && router.meta.title) {
// generate internationalized title
const i18ntitle = i18n.t(`route.${router.meta.title}`)
data.title = [...data.title, i18ntitle]
if (router.redirect !== 'noredirect') {
// only push the routes with title
// special case: need to exclude parent router without redirect
res.push(data)
}
}
// recursive child routers
if (router.children) {
const tempRouters = this.generateRouters(router.children, data.path, data.title)
if (tempRouters.length >= 1) {
res = [...res, ...tempRouters]
}
}
}
return res
},
querySearch(query) {
if (query !== '') {
this.options = this.fuse.search(query)
} else {
this.options = []
}
}
}
}
</script>
<style lang="scss" scoped>
.header-search {
font-size: 0 !important;
.search-icon {
cursor: pointer;
font-size: 18px;
vertical-align: middle;
}
.header-search-select {
font-size: 18px;
transition: width 0.2s;
width: 0;
overflow: hidden;
background: transparent;
border-radius: 0;
display: inline-block;
vertical-align: middle;
/deep/ .el-input__inner {
border-radius: 0;
border: 0;
padding-left: 0;
padding-right: 0;
box-shadow: none !important;
border-bottom: 1px solid #d9d9d9;
vertical-align: middle;
}
}
&.show {
.header-search-select {
width: 210px;
margin-left: 10px;
}
}
}
</style>

8
frontend/src/components/InfoView/NodeInfoView.vue
View File

@@ -7,13 +7,13 @@
class="node-form"
label-position="right">
<el-form-item :label="$t('Node Name')">
<el-input v-model="nodeForm.name" :placeholder="$t('Node Name')" disabled></el-input>
<el-input v-model="nodeForm.name" :placeholder="$t('Node Name')" :disabled="isView"></el-input>
</el-form-item>
<el-form-item :label="$t('Node IP')" prop="ip" required>
<el-input v-model="nodeForm.ip" :placeholder="$t('Node IP')" :disabled="isView"></el-input>
<el-input v-model="nodeForm.ip" :placeholder="$t('Node IP')" disabled></el-input>
</el-form-item>
<el-form-item :label="$t('Node Port')" prop="port" required>
<el-input v-model="nodeForm.port" :placeholder="$t('Node Port')" :disabled="isView"></el-input>
<el-form-item :label="$t('Node MAC')" prop="ip" required>
<el-input v-model="nodeForm.mac" :placeholder="$t('Node MAC')" disabled></el-input>
</el-form-item>
<el-form-item :label="$t('Description')">
<el-input type="textarea" v-model="nodeForm.description" :placeholder="$t('Description')" :disabled="isView">

30
frontend/src/components/InfoView/SpiderInfoView.vue
View File

@@ -10,7 +10,7 @@
<el-input v-model="spiderForm._id" :placeholder="$t('Spider ID')" disabled></el-input>
</el-form-item>
<el-form-item :label="$t('Spider Name')">
<el-input v-model="spiderForm.name" :placeholder="$t('Spider Name')" :disabled="isView"></el-input>
<el-input v-model="spiderForm.display_name" :placeholder="$t('Spider Name')" :disabled="isView"></el-input>
</el-form-item>
<el-form-item v-if="isCustomized" :label="$t('Source Folder')">
<el-input v-model="spiderForm.src" :placeholder="$t('Source Folder')" disabled></el-input>
@@ -28,6 +28,7 @@
:placeholder="$t('Site')"
:fetch-suggestions="fetchSiteSuggestions"
clearable
:disabled="isView"
@select="onSiteSelect">
</el-autocomplete>
</el-form-item>
@@ -49,7 +50,6 @@
</el-row>
<el-row class="button-container" v-if="!isView">
<el-button v-if="isShowRun" type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
<el-button v-if="isCustomized" type="primary" @click="onDeploy">{{$t('Deploy')}}</el-button>
<el-button type="success" @click="onSave">{{$t('Save')}}</el-button>
</el-row>
</div>
@@ -100,9 +100,6 @@ export default {
isShowRun () {
if (this.isCustomized) {
// customized spider
if (!this.spiderForm.deploy_ts) {
return false
}
return !!this.spiderForm.cmd
} else {
// configurable spider
@@ -132,29 +129,6 @@ export default {
}
})
},
onDeploy () {
const row = this.spiderForm
// save spider
this.$store.dispatch('spider/editSpider', row._id)
// validate fields
this.$refs['spiderForm'].validate(res => {
if (res) {
this.$confirm(this.$t('Are you sure to deploy this spider?'), this.$t('Notification'), {
confirmButtonText: this.$t('Confirm'),
cancelButtonText: this.$t('Cancel')
})
.then(() => {
this.$store.dispatch('spider/deploySpider', row._id)
.then(() => {
this.$message.success(this.$t(`Spider has been deployed`))
})
this.$st.sendEv('爬虫详情-概览', '部署')
})
}
})
},
onSave () {
this.$refs['spiderForm'].validate(res => {
if (res) {

18
frontend/src/components/InfoView/TaskInfoView.vue
View File

@@ -10,16 +10,10 @@
<el-input v-model="taskForm._id" placeholder="Task ID" disabled></el-input>
</el-form-item>
<el-form-item :label="$t('Status')">
<el-tag type="success" v-if="taskForm.status === 'SUCCESS'">{{$t('SUCCESS')}}</el-tag>
<el-tag type="warning" v-else-if="taskForm.status === 'STARTED'">{{$t('STARTED')}}</el-tag>
<el-tag type="danger" v-else-if="taskForm.status === 'FAILURE'">{{$t('FAILURE')}}</el-tag>
<el-tag type="info" v-else>{{$t(taskForm.status)}}</el-tag>
<status-tag :status="taskForm.status"/>
</el-form-item>
<!--<el-form-item label="Spider Version">-->
<!--<el-input v-model="taskForm.spider_version" placeholder="Spider Version" disabled></el-input>-->
<!--</el-form-item>-->
<el-form-item :label="$t('Log File Path')">
<el-input v-model="taskForm.log_file_path" placeholder="Log File Path" disabled></el-input>
<el-input v-model="taskForm.log_stdout_path" placeholder="Log File Path" disabled></el-input>
</el-form-item>
<el-form-item :label="$t('Create Timestamp')">
<el-input v-model="taskForm.create_ts" placeholder="Create Timestamp" disabled></el-input>
@@ -37,9 +31,9 @@
<el-input v-model="taskForm.avg_num_results" placeholder="Average Results Count per Second" disabled>
</el-input>
</el-form-item>
<el-form-item :label="$t('Error Message')" v-if="taskForm.status === 'FAILURE'">
<el-form-item :label="$t('Error Message')" v-if="taskForm.status === 'error'">
<div class="error-message">
{{taskForm.log}}
{{ taskForm.error }}
</div>
</el-form-item>
</el-form>
@@ -55,15 +49,17 @@
import {
mapState
} from 'vuex'
import StatusTag from '../Status/StatusTag'
export default {
name: 'NodeInfoView',
components: { StatusTag },
computed: {
...mapState('task', [
'taskForm'
]),
isRunning () {
return !['SUCCESS', 'FAILURE'].includes(this.taskForm.status)
return ['pending', 'running'].includes(this.taskForm.status)
}
},
methods: {

72
frontend/src/components/JsonEditor/index.vue
View File

@@ -1,72 +0,0 @@
<template>
<div class="json-editor">
<textarea ref="textarea"/>
</div>
</template>
<script>
import CodeMirror from 'codemirror'
import 'codemirror/addon/lint/lint.css'
import 'codemirror/lib/codemirror.css'
import 'codemirror/theme/rubyblue.css'
require('script-loader!jsonlint')
import 'codemirror/mode/javascript/javascript'
import 'codemirror/addon/lint/lint'
import 'codemirror/addon/lint/json-lint'
export default {
name: 'JsonEditor',
/* eslint-disable vue/require-prop-types */
props: ['value'],
data() {
return {
jsonEditor: false
}
},
watch: {
value(value) {
const editor_value = this.jsonEditor.getValue()
if (value !== editor_value) {
this.jsonEditor.setValue(JSON.stringify(this.value, null, 2))
}
}
},
mounted() {
this.jsonEditor = CodeMirror.fromTextArea(this.$refs.textarea, {
lineNumbers: true,
mode: 'application/json',
gutters: ['CodeMirror-lint-markers'],
theme: 'rubyblue',
lint: true
})
this.jsonEditor.setValue(JSON.stringify(this.value, null, 2))
this.jsonEditor.on('change', cm => {
this.$emit('changed', cm.getValue())
this.$emit('input', cm.getValue())
})
},
methods: {
getValue() {
return this.jsonEditor.getValue()
}
}
}
</script>
<style scoped>
.json-editor{
height: 100%;
position: relative;
}
.json-editor >>> .CodeMirror {
height: auto;
min-height: 300px;
}
.json-editor >>> .CodeMirror-scroll{
min-height: 300px;
}
.json-editor >>> .cm-s-rubyblue span.cm-string {
color: #F08047;
}
</style>

89
frontend/src/components/Kanban/index.vue
View File

@@ -1,89 +0,0 @@
<template>
<div class="board-column">
<div class="board-column-header">
{{ headerText }}
</div>
<draggable
:list="list"
:options="options"
class="board-column-content">
<div v-for="element in list" :key="element.id" class="board-item">
{{ element.name }} {{ element.id }}
</div>
</draggable>
</div>
</template>
<script>
import draggable from 'vuedraggable'
export default {
name: 'DragKanbanDemo',
components: {
draggable
},
props: {
headerText: {
type: String,
default: 'Header'
},
options: {
type: Object,
default() {
return {}
}
},
list: {
type: Array,
default() {
return []
}
}
}
}
</script>
<style lang="scss" scoped>
.board-column {
min-width: 300px;
min-height: 100px;
height: auto;
overflow: hidden;
background: #f0f0f0;
border-radius: 3px;
.board-column-header {
height: 50px;
line-height: 50px;
overflow: hidden;
padding: 0 20px;
text-align: center;
background: #333;
color: #fff;
border-radius: 3px 3px 0 0;
}
.board-column-content {
height: auto;
overflow: hidden;
border: 10px solid transparent;
min-height: 60px;
display: flex;
justify-content: flex-start;
flex-direction: column;
align-items: center;
.board-item {
cursor: pointer;
width: 100%;
height: 64px;
margin: 5px 0;
background-color: #fff;
text-align: left;
line-height: 54px;
padding: 5px 10px;
box-sizing: border-box;
box-shadow: 0px 1px 3px 0 rgba(0, 0, 0, 0.2);
}
}
}
</style>

32
frontend/src/components/LangSelect/index.vue
View File

@@ -1,32 +0,0 @@
<template>
<el-dropdown trigger="click" class="international" @command="handleSetLanguage">
<div>
<svg-icon class-name="international-icon" icon-class="language" />
</div>
<el-dropdown-menu slot="dropdown">
<el-dropdown-item :disabled="language==='zh'" command="zh">中文</el-dropdown-item>
<el-dropdown-item :disabled="language==='en'" command="en">English</el-dropdown-item>
<el-dropdown-item :disabled="language==='es'" command="es">Español</el-dropdown-item>
</el-dropdown-menu>
</el-dropdown>
</template>
<script>
export default {
computed: {
language() {
return this.$store.getters.language
}
},
methods: {
handleSetLanguage(lang) {
this.$i18n.locale = lang
this.$store.dispatch('setLanguage', lang)
this.$message({
message: 'Switch Language Success',
type: 'success'
})
}
}
}
</script>

354
frontend/src/components/MDinput/index.vue
View File

@@ -1,354 +0,0 @@
<template>
<div :class="computedClasses" class="material-input__component">
<div :class="{iconClass:icon}">
<i v-if="icon" :class="['el-icon-' + icon]" class="el-input__icon material-input__icon"/>
<input
v-if="type === 'email'"
:name="name"
:placeholder="fillPlaceHolder"
v-model="currentValue"
:readonly="readonly"
:disabled="disabled"
:autoComplete="autoComplete"
:required="required"
type="email"
class="material-input"
@focus="handleMdFocus"
@blur="handleMdBlur"
@input="handleModelInput">
<input
v-if="type === 'url'"
:name="name"
:placeholder="fillPlaceHolder"
v-model="currentValue"
:readonly="readonly"
:disabled="disabled"
:autoComplete="autoComplete"
:required="required"
type="url"
class="material-input"
@focus="handleMdFocus"
@blur="handleMdBlur"
@input="handleModelInput">
<input
v-if="type === 'number'"
:name="name"
:placeholder="fillPlaceHolder"
v-model="currentValue"
:step="step"
:readonly="readonly"
:disabled="disabled"
:autoComplete="autoComplete"
:max="max"
:min="min"
:minlength="minlength"
:maxlength="maxlength"
:required="required"
type="number"
class="material-input"
@focus="handleMdFocus"
@blur="handleMdBlur"
@input="handleModelInput">
<input
v-if="type === 'password'"
:name="name"
:placeholder="fillPlaceHolder"
v-model="currentValue"
:readonly="readonly"
:disabled="disabled"
:autoComplete="autoComplete"
:max="max"
:min="min"
:required="required"
type="password"
class="material-input"
@focus="handleMdFocus"
@blur="handleMdBlur"
@input="handleModelInput">
<input
v-if="type === 'tel'"
:name="name"
:placeholder="fillPlaceHolder"
v-model="currentValue"
:readonly="readonly"
:disabled="disabled"
:autoComplete="autoComplete"
:required="required"
type="tel"
class="material-input"
@focus="handleMdFocus"
@blur="handleMdBlur"
@input="handleModelInput">
<input
v-if="type === 'text'"
:name="name"
:placeholder="fillPlaceHolder"
v-model="currentValue"
:readonly="readonly"
:disabled="disabled"
:autoComplete="autoComplete"
:minlength="minlength"
:maxlength="maxlength"
:required="required"
type="text"
class="material-input"
@focus="handleMdFocus"
@blur="handleMdBlur"
@input="handleModelInput">
<span class="material-input-bar"/>
<label class="material-label">
<slot/>
</label>
</div>
</div>
</template>
<script>
// source:https://github.com/wemake-services/vue-material-input/blob/master/src/components/MaterialInput.vue
export default {
name: 'MdInput',
props: {
/* eslint-disable */
icon: String,
name: String,
type: {
type: String,
default: 'text'
},
value: [String, Number],
placeholder: String,
readonly: Boolean,
disabled: Boolean,
min: String,
max: String,
step: String,
minlength: Number,
maxlength: Number,
required: {
type: Boolean,
default: true
},
autoComplete: {
type: String,
default: 'off'
},
validateEvent: {
type: Boolean,
default: true
}
},
data() {
return {
currentValue: this.value,
focus: false,
fillPlaceHolder: null
}
},
computed: {
computedClasses() {
return {
'material--active': this.focus,
'material--disabled': this.disabled,
'material--raised': Boolean(this.focus || this.currentValue) // has value
}
}
},
watch: {
value(newValue) {
this.currentValue = newValue
}
},
methods: {
handleModelInput(event) {
const value = event.target.value
this.$emit('input', value)
if (this.$parent.$options.componentName === 'ElFormItem') {
if (this.validateEvent) {
this.$parent.$emit('el.form.change', [value])
}
}
this.$emit('change', value)
},
handleMdFocus(event) {
this.focus = true
this.$emit('focus', event)
if (this.placeholder && this.placeholder !== '') {
this.fillPlaceHolder = this.placeholder
}
},
handleMdBlur(event) {
this.focus = false
this.$emit('blur', event)
this.fillPlaceHolder = null
if (this.$parent.$options.componentName === 'ElFormItem') {
if (this.validateEvent) {
this.$parent.$emit('el.form.blur', [this.currentValue])
}
}
}
}
}
</script>
<style rel="stylesheet/scss" lang="scss" scoped>
// Fonts:
$font-size-base: 16px;
$font-size-small: 18px;
$font-size-smallest: 12px;
$font-weight-normal: normal;
$font-weight-bold: bold;
$apixel: 1px;
// Utils
$spacer: 12px;
$transition: 0.2s ease all;
$index: 0px;
$index-has-icon: 30px;
// Theme:
$color-white: white;
$color-grey: #9E9E9E;
$color-grey-light: #E0E0E0;
$color-blue: #2196F3;
$color-red: #F44336;
$color-black: black;
// Base clases:
%base-bar-pseudo {
content: '';
height: 1px;
width: 0;
bottom: 0;
position: absolute;
transition: $transition;
}
// Mixins:
@mixin slided-top() {
top: - ($font-size-base + $spacer);
left: 0;
font-size: $font-size-base;
font-weight: $font-weight-bold;
}
// Component:
.material-input__component {
margin-top: 36px;
position: relative;
* {
box-sizing: border-box;
}
.iconClass {
.material-input__icon {
position: absolute;
left: 0;
line-height: $font-size-base;
color: $color-blue;
top: $spacer;
width: $index-has-icon;
height: $font-size-base;
font-size: $font-size-base;
font-weight: $font-weight-normal;
pointer-events: none;
}
.material-label {
left: $index-has-icon;
}
.material-input {
text-indent: $index-has-icon;
}
}
.material-input {
font-size: $font-size-base;
padding: $spacer $spacer $spacer - $apixel * 10 $spacer / 2;
display: block;
width: 100%;
border: none;
line-height: 1;
border-radius: 0;
&:focus {
outline: none;
border: none;
border-bottom: 1px solid transparent; // fixes the height issue
}
}
.material-label {
font-weight: $font-weight-normal;
position: absolute;
pointer-events: none;
left: $index;
top: 0;
transition: $transition;
font-size: $font-size-small;
}
.material-input-bar {
position: relative;
display: block;
width: 100%;
&:before {
@extend %base-bar-pseudo;
left: 50%;
}
&:after {
@extend %base-bar-pseudo;
right: 50%;
}
}
// Disabled state:
&.material--disabled {
.material-input {
border-bottom-style: dashed;
}
}
// Raised state:
&.material--raised {
.material-label {
@include slided-top();
}
}
// Active state:
&.material--active {
.material-input-bar {
&:before,
&:after {
width: 50%;
}
}
}
}
.material-input__component {
background: $color-white;
.material-input {
background: none;
color: $color-black;
text-indent: $index;
border-bottom: 1px solid $color-grey-light;
}
.material-label {
color: $color-grey;
}
.material-input-bar {
&:before,
&:after {
background: $color-blue;
}
}
// Active state:
&.material--active {
.material-label {
color: $color-blue;
}
}
// Errors:
&.material--has-errors {
&.material--active .material-label {
color: $color-red;
}
.material-input-bar {
&:before,
&:after {
background: transparent;
}
}
}
}
</style>

31
frontend/src/components/MarkdownEditor/defaultOptions.js
View File

@@ -1,31 +0,0 @@
// doc: https://nhnent.github.io/tui.editor/api/latest/ToastUIEditor.html#ToastUIEditor
export default {
minHeight: '200px',
previewStyle: 'vertical',
useCommandShortcut: true,
useDefaultHTMLSanitizer: true,
usageStatistics: false,
hideModeSwitch: false,
toolbarItems: [
'heading',
'bold',
'italic',
'strike',
'divider',
'hr',
'quote',
'divider',
'ul',
'ol',
'task',
'indent',
'outdent',
'divider',
'table',
'image',
'link',
'divider',
'code',
'codeblock'
]
}

118
frontend/src/components/MarkdownEditor/index.vue
View File

@@ -1,118 +0,0 @@
<template>
<div :id="id"/>
</template>
<script>
// deps for editor
import 'codemirror/lib/codemirror.css' // codemirror
import 'tui-editor/dist/tui-editor.css' // editor ui
import 'tui-editor/dist/tui-editor-contents.css' // editor content
import Editor from 'tui-editor'
import defaultOptions from './defaultOptions'
export default {
name: 'MarddownEditor',
props: {
value: {
type: String,
default: ''
},
id: {
type: String,
required: false,
default() {
return 'markdown-editor-' + +new Date() + ((Math.random() * 1000).toFixed(0) + '')
}
},
options: {
type: Object,
default() {
return defaultOptions
}
},
mode: {
type: String,
default: 'markdown'
},
height: {
type: String,
required: false,
default: '300px'
},
language: {
type: String,
required: false,
default: 'en_US' // https://github.com/nhnent/tui.editor/tree/master/src/js/langs
}
},
data() {
return {
editor: null
}
},
computed: {
editorOptions() {
const options = Object.assign({}, defaultOptions, this.options)
options.initialEditType = this.mode
options.height = this.height
options.language = this.language
return options
}
},
watch: {
value(newValue, preValue) {
if (newValue !== preValue && newValue !== this.editor.getValue()) {
this.editor.setValue(newValue)
}
},
language(val) {
this.destroyEditor()
this.initEditor()
},
height(newValue) {
this.editor.height(newValue)
},
mode(newValue) {
this.editor.changeMode(newValue)
}
},
mounted() {
this.initEditor()
},
destroyed() {
this.destroyEditor()
},
methods: {
initEditor() {
this.editor = new Editor({
el: document.getElementById(this.id),
...this.editorOptions
})
if (this.value) {
this.editor.setValue(this.value)
}
this.editor.on('change', () => {
this.$emit('input', this.editor.getValue())
})
},
destroyEditor() {
if (!this.editor) return
this.editor.off('change')
this.editor.remove()
},
setValue(value) {
this.editor.setValue(value)
},
getValue() {
return this.editor.getValue()
},
setHtml(value) {
this.editor.setHtml(value)
},
getHtml() {
return this.editor.getHtml()
}
}
}
</script>

7
frontend/src/components/Overview/NodeOverview.vue
View File

@@ -5,11 +5,6 @@
<el-row>
<task-table-view :title="$t('Latest Tasks')"/>
</el-row>
<!--last deploys-->
<el-row v-if="false">
<deploy-table-view :title="$t('Latest Deploys')"/>
</el-row>
</el-col>
<el-col :span="12">
@@ -23,7 +18,6 @@
import {
mapState
} from 'vuex'
import DeployTableView from '../TableView/DeployTableView'
import TaskTableView from '../TableView/TaskTableView'
import NodeInfoView from '../InfoView/NodeInfoView'
@@ -31,7 +25,6 @@ export default {
name: 'NodeOverview',
components: {
NodeInfoView,
DeployTableView,
TaskTableView
},
computed: {

7
frontend/src/components/Overview/SpiderOverview.vue
View File

@@ -5,11 +5,6 @@
<el-row>
<task-table-view :title="$t('Latest Tasks')"/>
</el-row>
<!--last deploys-->
<el-row v-if="false">
<deploy-table-view :title="$t('Latest Deploys')"/>
</el-row>
</el-col>
<el-col :span="12">
@@ -23,7 +18,6 @@
import {
mapState
} from 'vuex'
import DeployTableView from '../TableView/DeployTableView'
import TaskTableView from '../TableView/TaskTableView'
import SpiderInfoView from '../InfoView/SpiderInfoView'
@@ -31,7 +25,6 @@ export default {
name: 'SpiderOverview',
components: {
SpiderInfoView,
DeployTableView,
TaskTableView
},
data () {

48
frontend/src/components/Status/StatusTag.vue Normal file
View File

@@ -0,0 +1,48 @@
<template>
<el-tag :type="type">
{{$t(label)}}
</el-tag>
</template>
<script>
export default {
name: 'StatusTag',
props: {
status: {
type: String,
default: ''
}
},
data () {
return {
statusDict: {
pending: { label: 'Pending', type: 'primary' },
running: { label: 'Running', type: 'warning' },
finished: { label: 'Finished', type: 'success' },
error: { label: 'Error', type: 'danger' },
cancelled: { label: 'Cancelled', type: 'info' }
}
}
},
computed: {
type () {
const s = this.statusDict[this.status]
if (s) {
return s.type
}
return ''
},
label () {
const s = this.statusDict[this.status]
if (s) {
return s.label
}
return 'NA'
}
}
}
</script>
<style scoped>
</style>

25
frontend/src/components/TableView/TaskTableView.vue
View File

@@ -5,30 +5,29 @@
<el-button type="success" plain class="small-btn" size="mini" icon="fa fa-refresh" @click="onRefresh"></el-button>
</el-row>
<el-table border height="480px" :data="taskList">
<el-table-column property="node" :label="$t('Node')" width="220" align="center">
<el-table-column property="node" :label="$t('Node')" width="120" align="left">
<template slot-scope="scope">
<a class="a-tag" @click="onClickNode(scope.row)">{{scope.row.node_id}}</a>
<a class="a-tag" @click="onClickNode(scope.row)">{{scope.row.node_name}}</a>
</template>
</el-table-column>
<el-table-column property="spider_name" :label="$t('Spider')" width="80" align="center">
<el-table-column property="spider_name" :label="$t('Spider')" width="120" align="left">
<template slot-scope="scope">
<a class="a-tag" @click="onClickSpider(scope.row)">{{scope.row.spider_name}}</a>
</template>
</el-table-column>
<el-table-column :label="$t('Status')"
align="center"
align="left"
width="100">
<template slot-scope="scope">
<el-tag type="success" v-if="scope.row.status === 'SUCCESS'">{{$t('SUCCESS')}}</el-tag>
<el-tag type="warning" v-else-if="scope.row.status === 'STARTED'">{{$t('STARTED')}}</el-tag>
<el-tag type="danger" v-else-if="scope.row.status === 'FAILURE'">{{$t('FAILURE')}}</el-tag>
<el-tag type="info" v-else>{{$t(scope.row['status'])}}</el-tag>
<status-tag :status="scope.row.status"/>
</template>
</el-table-column>
<!--<el-table-column property="create_ts" label="Create Time" width="auto" align="center"></el-table-column>-->
<el-table-column property="create_ts" :label="$t('Create Time')" width="auto" align="center">
<el-table-column property="create_ts" :label="$t('Create Time')" width="auto" align="left">
<template slot-scope="scope">
<a href="javascript:" class="a-tag" @click="onClickTask(scope.row)">{{scope.row.create_ts}}</a>
<a href="javascript:" class="a-tag" @click="onClickTask(scope.row)">
{{getTime(scope.row.create_ts).format('YYYY-MM-DD HH:mm:ss')}}
</a>
</template>
</el-table-column>
</el-table>
@@ -40,9 +39,12 @@
import {
mapState
} from 'vuex'
import dayjs from 'dayjs'
import StatusTag from '../Status/StatusTag'
export default {
name: 'TaskTableView',
components: { StatusTag },
data () {
return {
// setInterval handle
@@ -76,6 +78,9 @@ export default {
} else if (this.$route.path.split('/')[1] === 'nodes') {
this.$store.dispatch('node/getTaskList', this.$route.params.id)
}
},
getTime (str) {
return dayjs(str)
}
},
mounted () {

113
frontend/src/components/TextHoverEffect/Mallki.vue
View File

@@ -1,113 +0,0 @@
<template>
<a :class="className" class="link--mallki" href="#">
{{ text }}
<span :data-letters="text"/>
<span :data-letters="text"/>
</a>
</template>
<script>
export default {
props: {
className: {
type: String,
default: ''
},
text: {
type: String,
default: 'vue-element-admin'
}
}
}
</script>
<style>
/* Mallki */
.link--mallki {
font-weight: 800;
color: #4dd9d5;
font-family: 'Dosis', sans-serif;
-webkit-transition: color 0.5s 0.25s;
transition: color 0.5s 0.25s;
overflow: hidden;
position: relative;
display: inline-block;
line-height: 1;
outline: none;
text-decoration: none;
}
.link--mallki:hover {
-webkit-transition: none;
transition: none;
color: transparent;
}
.link--mallki::before {
content: '';
width: 100%;
height: 6px;
margin: -3px 0 0 0;
background: #3888fa;
position: absolute;
left: 0;
top: 50%;
-webkit-transform: translate3d(-100%, 0, 0);
transform: translate3d(-100%, 0, 0);
-webkit-transition: -webkit-transform 0.4s;
transition: transform 0.4s;
-webkit-transition-timing-function: cubic-bezier(0.7, 0, 0.3, 1);
transition-timing-function: cubic-bezier(0.7, 0, 0.3, 1);
}
.link--mallki:hover::before {
-webkit-transform: translate3d(100%, 0, 0);
transform: translate3d(100%, 0, 0);
}
.link--mallki span {
position: absolute;
height: 50%;
width: 100%;
left: 0;
top: 0;
overflow: hidden;
}
.link--mallki span::before {
content: attr(data-letters);
color: red;
position: absolute;
left: 0;
width: 100%;
color: #3888fa;
-webkit-transition: -webkit-transform 0.5s;
transition: transform 0.5s;
}
.link--mallki span:nth-child(2) {
top: 50%;
}
.link--mallki span:first-child::before {
top: 0;
-webkit-transform: translate3d(0, 100%, 0);
transform: translate3d(0, 100%, 0);
}
.link--mallki span:nth-child(2)::before {
bottom: 0;
-webkit-transform: translate3d(0, -100%, 0);
transform: translate3d(0, -100%, 0);
}
.link--mallki:hover span::before {
-webkit-transition-delay: 0.3s;
transition-delay: 0.3s;
-webkit-transform: translate3d(0, 0, 0);
transform: translate3d(0, 0, 0);
-webkit-transition-timing-function: cubic-bezier(0.2, 1, 0.3, 1);
transition-timing-function: cubic-bezier(0.2, 1, 0.3, 1);
}
</style>

29
frontend/src/components/TreeTable/eval.js
View File

@@ -1,29 +0,0 @@
/**
* @Author: jianglei
* @Date: 2017-10-12 12:06:49
*/
'use strict'
import Vue from 'vue'
export default function treeToArray(data, expandAll, parent = null, level = null) {
let tmp = []
Array.from(data).forEach(function(record) {
if (record._expanded === undefined) {
Vue.set(record, '_expanded', expandAll)
}
let _level = 1
if (level !== undefined && level !== null) {
_level = level + 1
}
Vue.set(record, '_level', _level)
// 如果有父元素
if (parent) {
Vue.set(record, 'parent', parent)
}
tmp.push(record)
if (record.children && record.children.length > 0) {
const children = treeToArray(record.children, expandAll, record, _level)
tmp = tmp.concat(children)
}
})
return tmp
}

127
frontend/src/components/TreeTable/index.vue
View File

@@ -1,127 +0,0 @@
<template>
<el-table :data="formatData" :row-style="showRow" v-bind="$attrs">
<el-table-column v-if="columns.length===0" width="150">
<template slot-scope="scope">
<span v-for="space in scope.row._level" :key="space" class="ms-tree-space"/>
<span v-if="iconShow(0,scope.row)" class="tree-ctrl" @click="toggleExpanded(scope.$index)">
<i v-if="!scope.row._expanded" class="el-icon-plus"/>
<i v-else class="el-icon-minus"/>
</span>
{{ scope.$index }}
</template>
</el-table-column>
<el-table-column v-for="(column, index) in columns" v-else :key="column.value" :label="column.text" :width="column.width">
<template slot-scope="scope">
<!-- Todo -->
<!-- eslint-disable-next-line vue/no-confusing-v-for-v-if -->
<span v-for="space in scope.row._level" v-if="index === 0" :key="space" class="ms-tree-space"/>
<span v-if="iconShow(index,scope.row)" class="tree-ctrl" @click="toggleExpanded(scope.$index)">
<i v-if="!scope.row._expanded" class="el-icon-plus"/>
<i v-else class="el-icon-minus"/>
</span>
{{ scope.row[column.value] }}
</template>
</el-table-column>
<slot/>
</el-table>
</template>
<script>
/**
Auth: Lei.j1ang
Created: 2018/1/19-13:59
*/
import treeToArray from './eval'
export default {
name: 'TreeTable',
props: {
/* eslint-disable */
data: {
type: [Array, Object],
required: true
},
columns: {
type: Array,
default: () => []
},
evalFunc: Function,
evalArgs: Array,
expandAll: {
type: Boolean,
default: false
}
},
computed: {
// 格式化数据源
formatData: function() {
let tmp
if (!Array.isArray(this.data)) {
tmp = [this.data]
} else {
tmp = this.data
}
const func = this.evalFunc || treeToArray
const args = this.evalArgs ? Array.concat([tmp, this.expandAll], this.evalArgs) : [tmp, this.expandAll]
return func.apply(null, args)
}
},
methods: {
showRow: function(row) {
const show = (row.row.parent ? (row.row.parent._expanded && row.row.parent._show) : true)
row.row._show = show
return show ? 'animation:treeTableShow 1s;-webkit-animation:treeTableShow 1s;' : 'display:none;'
},
// 切换下级是否展开
toggleExpanded: function(trIndex) {
const record = this.formatData[trIndex]
record._expanded = !record._expanded
},
// 图标显示
iconShow(index, record) {
return (index === 0 && record.children && record.children.length > 0)
}
}
}
</script>
<style rel="stylesheet/css">
@keyframes treeTableShow {
from {opacity: 0;}
to {opacity: 1;}
}
@-webkit-keyframes treeTableShow {
from {opacity: 0;}
to {opacity: 1;}
}
</style>
<style lang="scss" rel="stylesheet/scss" scoped>
$color-blue: #2196F3;
$space-width: 18px;
.ms-tree-space {
position: relative;
top: 1px;
display: inline-block;
font-style: normal;
font-weight: 400;
line-height: 1;
width: $space-width;
height: 14px;
&::before {
content: ""
}
}
.processContainer{
width: 100%;
height: 100%;
}
table td {
line-height: 26px;
}
.tree-ctrl{
position: relative;
cursor: pointer;
color: $color-blue;
margin-left: -$space-width;
}
</style>

89
frontend/src/components/TreeTable/readme.md
View File

@@ -1,89 +0,0 @@
## 写在前面
此组件仅提供一个创建TreeTable的解决思路
## prop说明
#### *data*
**必填**
原始数据,要求是一个数组或者对象
```javascript
[{
key1: value1,
key2: value2,
children: [{
key1: value1
},
{
key1: value1
}]
},
{
key1: value1
}]
```
或者
```javascript
{
key1: value1,
key2: value2,
children: [{
key1: value1
},
{
key1: value1
}]
}
```
#### columns
列属性,要求是一个数组
1. text: 显示在表头的文字
2. value: 对应data的key。treeTable将显示相应的value
3. width: 每列的宽度，为一个数字(可选)
如果你想要每个字段都有自定义的样式或者嵌套其他组件，columns可不提供，直接像在el-table一样写即可，如果没有自定义内容，提供columns将更加的便捷方便
如果你有几个字段是需要自定义的，几个不需要，那么可以将不需要自定义的字段放入columns，将需要自定义的内容放入到slot中，详情见后文
```javascript
[{
value:string,
text:string,
width:number
},{
value:string,
text:string,
width:number
}]
```
#### expandAll
是否默认全部展开，boolean值，默认为false
#### evalFunc
解析函数，function，非必须
如果不提供，将使用默认的[evalFunc](./eval.js)
如果提供了evalFunc,那么会用提供的evalFunc去解析data，并返回treeTable渲染所需要的值。如何编写一个evalFunc，请参考[*eval.js*](https://github.com/PanJiaChen/vue-element-admin/blob/master/src/components/TreeTable/eval.js)或[*customEval.js*](https://github.com/PanJiaChen/vue-element-admin/blob/master/src/views/table/treeTable/customEval.js)
#### evalArgs
解析函数的参数，是一个数组
**请注意，自定义的解析函数参数第一个为this.data，第二个参数为， this.expandAll,你不需要在evalArgs填写。一定记住，这两个参数是强制性的，并且位置不可颠倒** *this.data为需要解析的数据，this.expandAll为是否默认展开*
如你的解析函数需要的参数为`(this.data, this.expandAll,1,2,3,4)`，那么你只需要将`[1,2,3,4]`赋值给`evalArgs`就可以了
如果你的解析函数参数只有`(this.data, this.expandAll)`,那么就可以不用填写evalArgs了
具体可参考[*customEval.js*](https://github.com/PanJiaChen/vue-element-admin/blob/master/src/views/table/treeTable/customEval.js)的函数参数和[customTreeTable](https://github.com/PanJiaChen/vue-element-admin/blob/master/src/views/table/treeTable/customTreeTable.vue)的`evalArgs`属性值
## slot
这是一个自定义列的插槽。
默认情况下，treeTable只有一行行展示数据的功能。但是一般情况下，我们会要给行加上一个操作按钮或者根据当行数据展示不同的样式，这时我们就需要自定义列了。请参考[customTreeTable](https://github.com/PanJiaChen/vue-element-admin/blob/master/src/views/table/treeTable/customTreeTable.vue)，[实例效果](https://panjiachen.github.io/vue-element-admin/#/table/tree-table)
`slot`和`columns属性`可同时存在,columns里面的数据列会在slot自定义列的左边展示
## 其他
如果有其他的需求，请参考[el-table](http://element-cn.eleme.io/#/en-US/component/table)的api自行修改index.vue

132
frontend/src/components/Upload/singleImage.vue
View File

@@ -1,132 +0,0 @@
<template>
<div class="upload-container">
<el-upload
:data="dataObj"
:multiple="false"
:show-file-list="false"
:on-success="handleImageSuccess"
class="image-uploader"
drag
action="https://httpbin.org/post">
<i class="el-icon-upload"/>
<div class="el-upload__text">将文件拖到此处，或<em>点击上传</em></div>
</el-upload>
<div class="image-preview">
<div v-show="imageUrl.length>1" class="image-preview-wrapper">
<img :src="imageUrl+'?imageView2/1/w/200/h/200'">
<div class="image-preview-action">
<i class="el-icon-delete" @click="rmImage"/>
</div>
</div>
</div>
</div>
</template>
<script>
// 预览效果见付费文章
import { getToken } from '@/api/qiniu'
export default {
name: 'SingleImageUpload',
props: {
value: {
type: String,
default: ''
}
},
data() {
return {
tempUrl: '',
dataObj: { token: '', key: '' }
}
},
computed: {
imageUrl() {
return this.value
}
},
methods: {
rmImage() {
this.emitInput('')
},
emitInput(val) {
this.$emit('input', val)
},
handleImageSuccess() {
this.emitInput(this.tempUrl)
},
beforeUpload() {
const _self = this
return new Promise((resolve, reject) => {
getToken().then(response => {
const key = response.data.qiniu_key
const token = response.data.qiniu_token
_self._data.dataObj.token = token
_self._data.dataObj.key = key
this.tempUrl = response.data.qiniu_url
resolve(true)
}).catch(err => {
console.log(err)
reject(false)
})
})
}
}
}
</script>
<style rel="stylesheet/scss" lang="scss" scoped>
@import "src/styles/mixin.scss";
.upload-container {
width: 100%;
position: relative;
@include clearfix;
.image-uploader {
width: 60%;
float: left;
}
.image-preview {
width: 200px;
height: 200px;
position: relative;
border: 1px dashed #d9d9d9;
float: left;
margin-left: 50px;
.image-preview-wrapper {
position: relative;
width: 100%;
height: 100%;
img {
width: 100%;
height: 100%;
}
}
.image-preview-action {
position: absolute;
width: 100%;
height: 100%;
left: 0;
top: 0;
cursor: default;
text-align: center;
color: #fff;
opacity: 0;
font-size: 20px;
background-color: rgba(0, 0, 0, .5);
transition: opacity .3s;
cursor: pointer;
text-align: center;
line-height: 200px;
.el-icon-delete {
font-size: 36px;
}
}
&:hover {
.image-preview-action {
opacity: 1;
}
}
}
}
</style>

2
frontend/src/i18n/index.js
View File

@@ -6,7 +6,7 @@ import zh from './zh'
Vue.use(VueI18n)
const i18n = new VueI18n({
locale: localStorage.getItem('lang') || 'en',
locale: localStorage.getItem('lang') || 'zh',
messages: {
en,
zh

15
frontend/src/i18n/zh.js
View File

@@ -30,13 +30,13 @@ export default {
'Latest Tasks': '最近任务',
'Latest Deploys': '最近部署',
// 状态
PENDING: '待定',
STARTED: '已开始',
SUCCESS: '成功',
FAILURE: '错误',
UNAVAILABLE: '未知',
REVOKED: '已取消',
// 任务状态
Pending: '待定',
Running: '进行中',
Finished: '已完成',
Error: '错误',
NA: '未知',
Cancelled: '已取消',
// 操作
Add: '添加',
@@ -72,6 +72,7 @@ export default {
'Node Info': '节点信息',
'Node Name': '节点名称',
'Node IP': '节点IP',
'Node MAC': '节点MAC',
'Node Port': '节点端口',
'Description': '描述',

35
frontend/src/store/modules/node.js
View File

@@ -27,27 +27,11 @@ const actions = {
getNodeList ({ state, commit }) {
request.get('/nodes', {})
.then(response => {
commit('SET_NODE_LIST', response.data.items)
})
},
addNode ({ state, dispatch }) {
request.put('/nodes', {
name: state.nodeForm.name,
ip: state.nodeForm.ip,
port: state.nodeForm.port,
description: state.nodeForm.description
})
.then(() => {
dispatch('getNodeList')
commit('SET_NODE_LIST', response.data.data)
})
},
editNode ({ state, dispatch }) {
request.post(`/nodes/${state.nodeForm._id}`, {
name: state.nodeForm.name,
ip: state.nodeForm.ip,
port: state.nodeForm.port,
description: state.nodeForm.description
})
request.post(`/nodes/${state.nodeForm._id}`, state.nodeForm)
.then(() => {
dispatch('getNodeList')
})
@@ -61,23 +45,14 @@ const actions = {
getNodeData ({ state, commit }, id) {
request.get(`/nodes/${id}`)
.then(response => {
commit('SET_NODE_FORM', response.data)
})
},
getDeployList ({ state, commit }, id) {
return request.get(`/nodes/${id}/get_deploys`)
.then(response => {
commit('deploy/SET_DEPLOY_LIST',
response.data.items.map(d => d)
.sort((a, b) => a.finish_ts < b.finish_ts ? 1 : -1),
{ root: true })
commit('SET_NODE_FORM', response.data.data)
})
},
getTaskList ({ state, commit }, id) {
return request.get(`/nodes/${id}/get_tasks`)
return request.get(`/nodes/${id}/tasks`)
.then(response => {
commit('task/SET_TASK_LIST',
response.data.items.map(d => d)
response.data.data.map(d => d)
.sort((a, b) => a.create_ts < b.create_ts ? 1 : -1),
{ root: true })
})

2
frontend/src/store/modules/schedule.js
View File

@@ -20,7 +20,7 @@ const actions = {
getScheduleList ({ state, commit }) {
request.get('/schedules')
.then(response => {
commit('SET_SCHEDULE_LIST', response.data.items)
commit('SET_SCHEDULE_LIST', response.data.data)
})
},
addSchedule ({ state }) {

93
frontend/src/store/modules/spider.js
View File

@@ -76,43 +76,13 @@ const actions = {
if (state.filterSite) {
params.site = state.filterSite
}
console.log(params)
return request.get('/spiders', params)
.then(response => {
commit('SET_SPIDER_LIST', response.data.items)
})
},
addSpider ({ state, dispatch }) {
return request.put('/spiders', {
name: state.spiderForm.name,
col: state.spiderForm.col,
type: 'configurable',
site: state.spiderForm.site
})
.then(() => {
dispatch('getSpiderList')
commit('SET_SPIDER_LIST', response.data.data)
})
},
editSpider ({ state, dispatch }) {
return request.post(`/spiders/${state.spiderForm._id}`, {
name: state.spiderForm.name,
src: state.spiderForm.src,
cmd: state.spiderForm.cmd,
type: state.spiderForm.type,
lang: state.spiderForm.lang,
col: state.spiderForm.col,
site: state.spiderForm.site,
// configurable spider
crawl_type: state.spiderForm.crawl_type,
start_url: state.spiderForm.start_url,
url_pattern: state.spiderForm.url_pattern,
item_selector: state.spiderForm.item_selector,
item_selector_type: state.spiderForm.item_selector_type,
pagination_selector: state.spiderForm.pagination_selector,
pagination_selector_type: state.spiderForm.pagination_selector_type,
obey_robots_txt: state.spiderForm.obey_robots_txt,
item_threshold: state.spiderForm.item_threshold
})
return request.post(`/spiders/${state.spiderForm._id}`, state.spiderForm)
.then(() => {
dispatch('getSpiderList')
})
@@ -123,60 +93,24 @@ const actions = {
dispatch('getSpiderList')
})
},
updateSpiderEnvs ({ state }) {
return request.post(`/spiders/${state.spiderForm._id}/update_envs`, {
envs: JSON.stringify(state.spiderForm.envs)
})
},
updateSpiderFields ({ state }) {
return request.post(`/spiders/${state.spiderForm._id}/update_fields`, {
fields: JSON.stringify(state.spiderForm.fields)
})
},
updateSpiderDetailFields ({ state }) {
return request.post(`/spiders/${state.spiderForm._id}/update_detail_fields`, {
detail_fields: JSON.stringify(state.spiderForm.detail_fields)
})
},
getSpiderData ({ state, commit }, id) {
return request.get(`/spiders/${id}`)
.then(response => {
let data = response.data
data.cron_enabled = !!data.cron_enabled
let data = response.data.data
commit('SET_SPIDER_FORM', data)
})
},
deploySpider ({ state, dispatch }, id) {
return request.post(`/spiders/${id}/deploy`)
.then(response => {
console.log(response.data)
})
.then(response => {
dispatch('getSpiderData', id)
dispatch('getSpiderList')
})
},
crawlSpider ({ state, dispatch }, id) {
return request.post(`/spiders/${id}/on_crawl`)
.then(response => {
console.log(response.data)
})
},
getDeployList ({ state, commit }, id) {
return request.get(`/spiders/${id}/get_deploys`)
.then(response => {
commit('deploy/SET_DEPLOY_LIST',
response.data.items.map(d => {
return d
}).sort((a, b) => a.finish_ts < b.finish_ts ? 1 : -1),
{ root: true })
})
return request.put(`/tasks`, {
spider_id: id
// TODO: node_id
})
},
getTaskList ({ state, commit }, id) {
return request.get(`/spiders/${id}/get_tasks`)
return request.get(`/spiders/${id}/tasks`)
.then(response => {
commit('task/SET_TASK_LIST',
response.data.items.map(d => {
response.data.data.map(d => {
return d
}).sort((a, b) => a.create_ts < b.create_ts ? 1 : -1),
{ root: true })
@@ -185,15 +119,6 @@ const actions = {
importGithub ({ state }) {
const url = state.importForm.url
return request.post('/spiders/import/github', { url })
.then(response => {
console.log(response)
})
},
deployAll () {
return request.post('/spiders/manage/deploy_all')
.then(response => {
console.log(response)
})
},
getSpiderStats ({ state, commit }) {
return request.get('/stats/get_spider_stats?spider_id=' + state.spiderForm._id)

44
frontend/src/store/modules/task.js
View File

@@ -15,14 +15,28 @@ const state = {
spider_id: ''
},
// pagination
pageNum: 0,
pageNum: 1,
pageSize: 10,
// results
resultsPageNum: 1,
resultsPageSize: 10
}
const getters = {}
const getters = {
taskResultsColumns (state) {
if (!state.taskResultsData.length) {
return []
}
const keys = []
const item = state.taskResultsData[0]
for (const key in item) {
if (item.hasOwnProperty(key)) {
keys.push(key)
}
}
return keys
}
}
const mutations = {
SET_TASK_FORM (state, value) {
@@ -64,8 +78,9 @@ const actions = {
getTaskData ({ state, dispatch, commit }, id) {
return request.get(`/tasks/${id}`)
.then(response => {
let data = response.data
let data = response.data.data
commit('SET_TASK_FORM', data)
console.log(data)
dispatch('spider/getSpiderData', data.spider_id, { root: true })
dispatch('node/getNodeData', data.node_id, { root: true })
})
@@ -74,14 +89,12 @@ const actions = {
return request.get('/tasks', {
page_num: state.pageNum,
page_size: state.pageSize,
filter: {
node_id: state.filter.node_id || undefined,
spider_id: state.filter.spider_id || undefined
}
node_id: state.filter.node_id || undefined,
spider_id: state.filter.spider_id || undefined
})
.then(response => {
commit('SET_TASK_LIST', response.data.items)
commit('SET_TASK_LIST_TOTAL_COUNT', response.data.total_count)
commit('SET_TASK_LIST', response.data.data || [])
commit('SET_TASK_LIST_TOTAL_COUNT', response.data.total)
})
},
deleteTask ({ state, dispatch }, id) {
@@ -97,21 +110,20 @@ const actions = {
})
},
getTaskLog ({ state, commit }, id) {
return request.get(`/tasks/${id}/get_log`)
return request.get(`/tasks/${id}/log`)
.then(response => {
commit('SET_TASK_LOG', response.data.log)
commit('SET_TASK_LOG', response.data.data)
})
},
getTaskResults ({ state, commit }, id) {
return request.get(`/tasks/${id}/get_results`, {
return request.get(`/tasks/${id}/results`, {
page_num: state.resultsPageNum,
page_size: state.resultsPageSize
})
.then(response => {
commit('SET_TASK_RESULTS_DATA', response.data.items)
commit('SET_TASK_RESULTS_COLUMNS', response.data.fields)
commit('SET_TASK_RESULTS_COLUMNS', response.data.fields)
commit('SET_TASK_RESULTS_TOTAL_COUNT', response.data.total_count)
commit('SET_TASK_RESULTS_DATA', response.data.data)
// commit('SET_TASK_RESULTS_COLUMNS', response.data.fields)
commit('SET_TASK_RESULTS_TOTAL_COUNT', response.data.total)
})
}
}

3
frontend/src/views/node/NodeDetail.vue
View File

@@ -57,9 +57,6 @@ export default {
// get node basic info
this.$store.dispatch('node/getNodeData', this.$route.params.id)
// get node deploy list
this.$store.dispatch('node/getDeployList', this.$route.params.id)
// get node task list
this.$store.dispatch('node/getTaskList', this.$route.params.id)
}

7
frontend/src/views/node/NodeList.vue
View File

@@ -41,7 +41,7 @@
:property="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
align="center"
:align="col.align || 'left'"
:width="col.width">
</el-table-column>
</template>
@@ -50,9 +50,6 @@
<el-tooltip :content="$t('View')" placement="top">
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
</el-tooltip>
<!--<el-tooltip :content="$t('Edit')" placement="top">-->
<!--<el-button type="warning" icon="el-icon-edit" size="mini" @click="onView(scope.row)"></el-button>-->
<!--</el-tooltip>-->
<el-tooltip :content="$t('Remove')" placement="top">
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
</el-tooltip>
@@ -95,7 +92,7 @@ export default {
columns: [
{ name: 'name', label: 'Name', width: '220' },
{ name: 'ip', label: 'IP', width: '160' },
{ name: 'port', label: 'Port', width: '80' },
// { name: 'port', label: 'Port', width: '80' },
{ name: 'status', label: 'Status', width: '120', sortable: true },
{ name: 'description', label: 'Description', width: 'auto' }
],

2
frontend/src/views/result/ResultDetail.vue
View File

@@ -28,7 +28,7 @@ import FileList from '../../components/FileList/FileList'
import SpiderOverview from '../../components/Overview/SpiderOverview'
export default {
name: 'NodeDetail',
name: 'ResultDetail',
components: {
FileList,
SpiderOverview

28
frontend/src/views/schedule/ScheduleList.vue
View File

@@ -14,6 +14,18 @@
<el-form-item :label="$t('Schedule Name')" prop="name" required>
<el-input v-model="scheduleForm.name" :placeholder="$t('Schedule Name')"></el-input>
</el-form-item>
<el-form-item :label="$t('Node')" prop="node_id">
<el-select v-model="scheduleForm.node_id">
<el-option :label="$t('All Nodes')" value="000000000000000000000000"></el-option>
<el-option
v-for="op in nodeList"
:key="op._id"
:value="op._id"
:label="op.name"
:disabled="op.status === 'offline'"
/>
</el-select>
</el-form-item>
<el-form-item :label="$t('Spider')" prop="spider_id" required>
<el-select v-model="scheduleForm.spider_id" filterable>
<el-option
@@ -21,7 +33,7 @@
:key="op._id"
:value="op._id"
:label="op.name"
:disabled="!op.cmd || !op.deploy_ts"
:disabled="!op.cmd"
>
</el-option>
</el-select>
@@ -90,11 +102,11 @@
:property="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
align="center"
:align="col.align"
:width="col.width">
</el-table-column>
</template>
<el-table-column :label="$t('Action')" align="left" width="250">
<el-table-column :label="$t('Action')" align="left" width="250" fixed="right">
<template slot-scope="scope">
<el-tooltip :content="$t('Edit')" placement="top">
<el-button type="warning" icon="el-icon-edit" size="mini" @click="onEdit(scope.row)"></el-button>
@@ -136,8 +148,10 @@ export default {
}
return {
columns: [
{ name: 'name', label: 'Name', width: '220' },
{ name: 'cron', label: 'Cron', width: '220' },
{ name: 'name', label: 'Name', width: '180' },
{ name: 'cron', label: 'Cron', width: '120' },
{ name: 'node_name', label: 'Node', width: '150' },
{ name: 'spider_name', label: 'Spider', width: '150' },
{ name: 'description', label: 'Description', width: 'auto' }
],
isEdit: false,
@@ -158,6 +172,9 @@ export default {
...mapState('spider', [
'spiderList'
]),
...mapState('node', [
'nodeList'
]),
filteredTableData () {
return this.scheduleList
},
@@ -245,6 +262,7 @@ export default {
created () {
this.$store.dispatch('schedule/getScheduleList')
this.$store.dispatch('spider/getSpiderList')
this.$store.dispatch('node/getNodeList')
}
}
</script>

5
frontend/src/views/spider/SpiderDetail.vue
View File

@@ -40,7 +40,7 @@ import SpiderStats from '../../components/Stats/SpiderStats'
import ConfigList from '../../components/Config/ConfigList'
export default {
name: 'NodeDetail',
name: 'SpiderDetail',
components: {
ConfigList,
SpiderStats,
@@ -96,9 +96,6 @@ export default {
this.$store.dispatch('file/getFileList', this.spiderForm.src)
})
// get spider deploys
this.$store.dispatch('spider/getDeployList', this.$route.params.id)
// get spider tasks
this.$store.dispatch('spider/getTaskList', this.$route.params.id)
}

59
frontend/src/views/spider/SpiderList.vue
View File

@@ -111,9 +111,6 @@
</el-autocomplete>
</div>
<div class="right">
<el-button type="primary" icon="fa fa-cloud" @click="onDeployAll">
{{$t('Deploy All')}}
</el-button>
<el-button type="primary" icon="fa fa-download" @click="openImportDialog">
{{$t('Import Spiders')}}
</el-button>
@@ -173,6 +170,15 @@
</div>
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'cmd'"
:key="col.name"
:label="$t(col.label)"
:width="col.width"
align="left">
<template slot-scope="scope">
<el-input v-model="scope.row[col.name]"></el-input>
</template>
</el-table-column>
<el-table-column v-else
:key="col.name"
:property="col.name"
@@ -182,21 +188,18 @@
:width="col.width">
</el-table-column>
</template>
<el-table-column :label="$t('Action')" align="left" width="auto" fixed="right">
<el-table-column :label="$t('Action')" align="left" width="150" fixed="right">
<template slot-scope="scope">
<el-tooltip :content="$t('View')" placement="top">
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
</el-tooltip>
<!--<el-tooltip :content="$t('Edit')" placement="top">-->
<!--<el-button type="warning" icon="el-icon-edit" size="mini" @click="onView(scope.row)"></el-button>-->
<!--</el-tooltip>-->
<el-tooltip :content="$t('Remove')" placement="top">
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
</el-tooltip>
<el-tooltip v-if="scope.row.type === 'customized'" :content="$t('Deploy')" placement="top">
<el-button type="primary" icon="fa fa-cloud" size="mini" @click="onDeploy(scope.row)"></el-button>
<el-tooltip v-if="!isShowRun(scope.row)" :content="$t('No command line')" placement="top">
<el-button disabled type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
</el-tooltip>
<el-tooltip v-if="isShowRun(scope.row)" :content="$t('Run')" placement="top">
<el-tooltip v-else :content="$t('Run')" placement="top">
<el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
</el-tooltip>
</template>
@@ -244,7 +247,8 @@ export default {
{ name: 'name', label: 'Name', width: '180', align: 'left' },
{ name: 'site_name', label: 'Site', width: '140', align: 'left' },
{ name: 'type', label: 'Spider Type', width: '120' },
{ name: 'lang', label: 'Language', width: '120', sortable: true },
// { name: 'cmd', label: 'Command Line', width: '200' },
// { name: 'lang', label: 'Language', width: '120', sortable: true },
{ name: 'task_ts', label: 'Last Run', width: '160' },
{ name: 'last_7d_tasks', label: 'Last 7-Day Tasks', width: '80' },
{ name: 'last_5_errors', label: 'Last 5-Run Errors', width: '80' }
@@ -371,22 +375,6 @@ export default {
this.$st.sendEv('爬虫', '删除')
})
},
onDeploy (row) {
this.$confirm(this.$t('Are you sure to deploy this spider?'), this.$t('Notification'), {
confirmButtonText: this.$t('Confirm'),
cancelButtonText: this.$t('Cancel'),
type: 'warning'
}).then(() => {
this.$store.dispatch('spider/deploySpider', row._id)
.then(() => {
this.$message({
type: 'success',
message: 'Deployed successfully'
})
})
this.$st.sendEv('爬虫', '部署')
})
},
onCrawl (row) {
this.$confirm(this.$t('Are you sure to run this spider?'), this.$t('Notification'), {
confirmButtonText: this.$t('Confirm'),
@@ -431,26 +419,9 @@ export default {
openImportDialog () {
this.dialogVisible = true
},
onDeployAll () {
this.$confirm(this.$t('Are you sure to deploy all spiders to active nodes?'), this.$t('Notification'), {
confirmButtonText: this.$t('Confirm'),
cancelButtonText: this.$t('Cancel'),
type: 'warning'
})
.then(() => {
this.$store.dispatch('spider/deployAll')
.then(() => {
this.$message.success(this.$t('Deployed all spiders successfully'))
})
this.$st.sendEv('爬虫', '部署所有爬虫')
})
},
isShowRun (row) {
if (this.isCustomized(row)) {
// customized spider
if (!row.deploy_ts) {
return false
}
return !!row.cmd
} else {
// configurable spider

17
frontend/src/views/task/TaskDetail.vue
View File

@@ -29,7 +29,8 @@
<script>
import {
mapState
mapState,
mapGetters
} from 'vuex'
import TaskOverview from '../../components/Overview/TaskOverview'
import GeneralTableView from '../../components/TableView/GeneralTableView'
@@ -44,16 +45,19 @@ export default {
},
data () {
return {
activeTabName: 'overview'
activeTabName: 'overview',
handle: undefined
}
},
computed: {
...mapState('task', [
'taskLog',
'taskResultsData',
'taskResultsColumns',
'taskResultsTotalCount'
]),
...mapGetters('task', [
'taskResultsColumns'
]),
...mapState('file', [
'currentPath'
]),
@@ -99,6 +103,13 @@ export default {
this.$store.dispatch('task/getTaskData', this.$route.params.id)
this.$store.dispatch('task/getTaskLog', this.$route.params.id)
this.$store.dispatch('task/getTaskResults', this.$route.params.id)
this.handle = setInterval(() => {
this.$store.dispatch('task/getTaskLog', this.$route.params.id)
}, 5000)
},
destroyed () {
clearInterval(this.handle)
}
}
</script>

90
frontend/src/views/task/TaskList.vue
View File

@@ -40,17 +40,57 @@
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
align="center"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
<a href="javascript:" class="a-tag" @click="onClickSpider(scope.row)">{{scope.row[col.name]}}</a>
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'node_id'"
<el-table-column v-else-if="col.name.match(/_ts$/)"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
align="center"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{getTime(scope.row[col.name])}}
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'wait_duration'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{getWaitDuration(scope.row)}}
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'runtime_duration'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{getRuntimeDuration(scope.row)}}
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'total_duration'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{getTotalDuration(scope.row)}}
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'node_name'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
<a href="javascript:" class="a-tag" @click="onClickNode(scope.row)">{{scope.row[col.name]}}</a>
@@ -60,13 +100,10 @@
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
align="center"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
<el-tag type="success" v-if="scope.row.status === 'SUCCESS'">{{$t('SUCCESS')}}</el-tag>
<el-tag type="warning" v-else-if="scope.row.status === 'STARTED'">{{$t('STARTED')}}</el-tag>
<el-tag type="danger" v-else-if="scope.row.status === 'FAILURE'">{{$t('FAILURE')}}</el-tag>
<el-tag type="info" v-else>{{$t(scope.row[col.name])}}</el-tag>
<status-tag :status="scope.row[col.name]"/>
</template>
</el-table-column>
<el-table-column v-else
@@ -74,11 +111,11 @@
:property="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
align="center"
:align="col.align"
:width="col.width">
</el-table-column>
</template>
<el-table-column :label="$t('Action')" align="left" width="auto" fixed="right">
<el-table-column :label="$t('Action')" align="left" width="150" fixed="right">
<template slot-scope="scope">
<el-tooltip :content="$t('View')" placement="top">
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
@@ -107,9 +144,12 @@
import {
mapState
} from 'vuex'
import dayjs from 'dayjs'
import StatusTag from '../../components/Status/StatusTag'
export default {
name: 'TaskList',
components: { StatusTag },
data () {
return {
// setInterval handle
@@ -123,15 +163,17 @@ export default {
// table columns
columns: [
{ name: 'node_name', label: 'Node', width: '120' },
{ name: 'spider_name', label: 'Spider', width: '120' },
{ name: 'status', label: 'Status', width: '120' },
{ name: 'create_ts', label: 'Create Time', width: '100' },
{ name: 'start_ts', label: 'Start Time', width: '100' },
{ name: 'finish_ts', label: 'Finish Time', width: '100' },
{ name: 'duration', label: 'Duration (sec)', width: '80' },
{ name: 'spider_name', label: 'Spider', width: '120' },
{ name: 'node_id', label: 'Node', width: '160' },
{ name: 'num_results', label: 'Results Count', width: '80' },
{ name: 'avg_num_results', label: 'Average Results Count per Second', width: '80' },
{ name: 'status', label: 'Status', width: '80' }
{ name: 'wait_duration', label: 'Wait Duration (sec)', width: '80', align: 'right' },
{ name: 'runtime_duration', label: 'Runtime Duration (sec)', width: '80', align: 'right' },
{ name: 'total_duration', label: 'Total Duration (sec)', width: '80', align: 'right' },
{ name: 'num_results', label: 'Results Count', width: '80' }
// { name: 'avg_num_results', label: 'Average Results Count per Second', width: '80' }
]
}
},
@@ -233,6 +275,22 @@ export default {
setTimeout(() => {
this.$store.dispatch('task/getTaskList')
}, 0)
},
getTime (str) {
if (str.match('^0001')) return 'NA'
return dayjs(str).format('YYYY-MM-DD HH:mm:ss')
},
getWaitDuration (row) {
if (row.start_ts.match('^0001')) return 'NA'
return dayjs(row.start_ts).diff(row.create_ts, 'second')
},
getRuntimeDuration (row) {
if (row.finish_ts.match('^0001')) return 'NA'
return dayjs(row.finish_ts).diff(row.start_ts, 'second')
},
getTotalDuration (row) {
if (row.finish_ts.match('^0001')) return 'NA'
return dayjs(row.finish_ts).diff(row.create_ts, 'second')
}
},
created () {

61
spiders/article/article_spider.js
View File

@@ -1,61 +0,0 @@
const puppeteer = require('puppeteer');
const MongoClient = require('mongodb').MongoClient;
(async () => {
// browser
const browser = await (puppeteer.launch({
headless: true
}));
// page
const page = await browser.newPage();
// open database connection
const client = await MongoClient.connect('mongodb://127.0.0.1:27017');
let db = await client.db('crawlab_test');
const colName = process.env.CRAWLAB_COLLECTION || 'results';
const col = db.collection(colName);
const col_src = db.collection('results');
const results = await col_src.find({content: {$exists: false}}).toArray();
for (let i = 0; i < results.length; i++) {
let item = results[i];
// define article anchor
let anchor;
if (item.source === 'juejin') {
anchor = '.article-content';
} else if (item.source === 'segmentfault') {
anchor = '.article';
} else if (item.source === 'csdn') {
anchor = '#content_views';
} else {
continue;
}
console.log(`anchor: ${anchor}`);
// navigate to the article
try {
await page.goto(item.url, {waitUntil: 'domcontentloaded'});
await page.waitFor(2000);
} catch (e) {
console.error(e);
continue;
}
// scrape article content
item.content = await page.$eval(anchor, el => el.innerHTML);
// save to database
await col.save(item);
console.log(`saved item: ${JSON.stringify(item)}`)
}
// close mongodb
client.close();
// close browser
browser.close();
})();

11
spiders/sinastock/sinastock/spiders/sinastock_spider.py
View File

@@ -5,13 +5,9 @@ from datetime import datetime
import scrapy
from pymongo import MongoClient
import pytz
from sinastock.items import NewsItem
# 时区
tz = pytz.timezone('Asia/Shanghai')
class SinastockSpiderSpider(scrapy.Spider):
name = 'sinastock_spider'
@@ -22,13 +18,12 @@ class SinastockSpiderSpider(scrapy.Spider):
)
db = mongo[os.environ.get('MONGO_DB') or 'crawlab_test']
col = db.get_collection(os.environ.get('CRAWLAB_COLLECTION') or 'stock_news')
page_num = int(os.environ.get('PAGE_NUM')) or 3
def start_requests(self):
col = self.db['stocks']
for s in col.find({}):
code, ex = s['ts_code'].split('.')
for i in range(self.page_num):
for i in range(10):
url = f'http://vip.stock.finance.sina.com.cn/corp/view/vCB_AllNewsStock.php?symbol={ex.lower()}{code}&Page={i + 1}'
yield scrapy.Request(
url=url,
@@ -61,7 +56,5 @@ class SinastockSpiderSpider(scrapy.Spider):
if item['text'] is None or item['ts_str'] is None:
pass
else:
ts = datetime.strptime(item['ts_str'], '%Y年%m月%d日 %H:%M')
ts = tz.localize(ts)
item['ts'] = ts
item['ts'] = datetime.strptime(item['ts_str'], '%Y年%m月%d日 %H:%M')
yield item