diff --git a/.gitignore b/.gitignore
index 622a9c22..0afb0ac2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,8 +16,6 @@ dist/
downloads/
eggs/
.eggs/
-lib/
-lib64/
parts/
sdist/
var/
diff --git a/frontend/src/components/Environment/EnvironmentList.vue b/frontend/src/components/Environment/EnvironmentList.vue
index a46432c0..6858aba4 100644
--- a/frontend/src/components/Environment/EnvironmentList.vue
+++ b/frontend/src/components/Environment/EnvironmentList.vue
@@ -56,7 +56,7 @@ export default {
this.$st.sendEv('爬虫详情-环境', '删除')
},
save () {
- this.$store.dispatch('spider/updateSpiderEnvs')
+ this.$store.dispatch('spider/editSpider')
.then(() => {
this.$message.success(this.$t('Spider info has been saved successfully'))
})
diff --git a/frontend/src/components/InfoView/TaskInfoView.vue b/frontend/src/components/InfoView/TaskInfoView.vue
index a9c5d534..5eaefb3b 100644
--- a/frontend/src/components/InfoView/TaskInfoView.vue
+++ b/frontend/src/components/InfoView/TaskInfoView.vue
@@ -66,7 +66,7 @@ export default {
onRestart () {
},
onStop () {
- this.$store.dispatch('task/stopTask', this.$route.params.id)
+ this.$store.dispatch('task/cancelTask', this.$route.params.id)
.then(() => {
this.$message.success(`Task "${this.$route.params.id}" has been sent signal to stop`)
})
diff --git a/frontend/src/components/TableView/GeneralTableView.vue b/frontend/src/components/TableView/GeneralTableView.vue
index 25cd1923..819a0572 100644
--- a/frontend/src/components/TableView/GeneralTableView.vue
+++ b/frontend/src/components/TableView/GeneralTableView.vue
@@ -5,7 +5,7 @@
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
border>
-
+
diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index fc6d545c..978f2aea 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -75,6 +75,7 @@ export default {
'Node MAC': '节点MAC',
'Node Port': '节点端口',
'Description': '描述',
+ 'All Nodes': '所有节点',
// 节点列表
'IP': 'IP地址',
diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js
index 1d603e87..5bb52e77 100644
--- a/frontend/src/store/modules/spider.js
+++ b/frontend/src/store/modules/spider.js
@@ -110,9 +110,9 @@ const actions = {
return request.get(`/spiders/${id}/tasks`)
.then(response => {
commit('task/SET_TASK_LIST',
- response.data.data.map(d => {
+ response.data.data ? response.data.data.map(d => {
return d
- }).sort((a, b) => a.create_ts < b.create_ts ? 1 : -1),
+ }).sort((a, b) => a.create_ts < b.create_ts ? 1 : -1) : [],
{ root: true })
})
},
diff --git a/frontend/src/store/modules/task.js b/frontend/src/store/modules/task.js
index d0520087..545a169b 100644
--- a/frontend/src/store/modules/task.js
+++ b/frontend/src/store/modules/task.js
@@ -24,7 +24,7 @@ const state = {
const getters = {
taskResultsColumns (state) {
- if (!state.taskResultsData.length) {
+ if (!state.taskResultsData || !state.taskResultsData.length) {
return []
}
const keys = []
@@ -80,7 +80,6 @@ const actions = {
.then(response => {
let data = response.data.data
commit('SET_TASK_FORM', data)
- console.log(data)
dispatch('spider/getSpiderData', data.spider_id, { root: true })
dispatch('node/getNodeData', data.node_id, { root: true })
})
@@ -103,13 +102,8 @@ const actions = {
dispatch('getTaskList')
})
},
- stopTask ({ state, dispatch }, id) {
- return request.post(`/tasks/${id}/stop`)
- .then(() => {
- dispatch('getTaskList')
- })
- },
getTaskLog ({ state, commit }, id) {
+ commit('SET_TASK_LOG', '')
return request.get(`/tasks/${id}/log`)
.then(response => {
commit('SET_TASK_LOG', response.data.data)
@@ -125,6 +119,12 @@ const actions = {
// commit('SET_TASK_RESULTS_COLUMNS', response.data.fields)
commit('SET_TASK_RESULTS_TOTAL_COUNT', response.data.total)
})
+ },
+ cancelTask ({ state, dispatch }, id) {
+ return request.post(`/tasks/${id}/cancel`)
+ .then(() => {
+ dispatch('getTaskData')
+ })
}
}
diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue
index e097ff3f..bb65ddac 100644
--- a/frontend/src/views/schedule/ScheduleList.vue
+++ b/frontend/src/views/schedule/ScheduleList.vue
@@ -104,6 +104,9 @@
:sortable="col.sortable"
:align="col.align"
:width="col.width">
+
+ {{$t(scope.row[col.name])}}
+
diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue
index d5eff01d..fcddbc12 100644
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -179,6 +179,16 @@
+
+
+ {{getTime(scope.row[col.name])}}
+
+
-
+
@@ -223,6 +233,7 @@
import {
mapState
} from 'vuex'
+import dayjs from 'dayjs'
export default {
name: 'SpiderList',
@@ -249,9 +260,9 @@ export default {
{ name: 'type', label: 'Spider Type', width: '120' },
// { name: 'cmd', label: 'Command Line', width: '200' },
// { name: 'lang', label: 'Language', width: '120', sortable: true },
- { name: 'task_ts', label: 'Last Run', width: '160' },
- { name: 'last_7d_tasks', label: 'Last 7-Day Tasks', width: '80' },
- { name: 'last_5_errors', label: 'Last 5-Run Errors', width: '80' }
+ { name: 'last_run_ts', label: 'Last Run', width: '160' }
+ // { name: 'last_7d_tasks', label: 'Last 7-Day Tasks', width: '80' },
+ // { name: 'last_5_errors', label: 'Last 5-Run Errors', width: '80' }
],
spiderFormRules: {
name: [{ required: true, message: 'Required Field', trigger: 'change' }]
@@ -474,6 +485,10 @@ export default {
// close popup
this.addCustomizedDialogVisible = false
+ },
+ getTime (str) {
+ if (!str || str.match('^0001')) return 'NA'
+ return dayjs(str).format('YYYY-MM-DD HH:mm:ss')
}
},
created () {
diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue
index 86f1a33a..b9bb3ba6 100644
--- a/frontend/src/views/task/TaskDetail.vue
+++ b/frontend/src/views/task/TaskDetail.vue
@@ -104,9 +104,11 @@ export default {
this.$store.dispatch('task/getTaskLog', this.$route.params.id)
this.$store.dispatch('task/getTaskResults', this.$route.params.id)
- this.handle = setInterval(() => {
- this.$store.dispatch('task/getTaskLog', this.$route.params.id)
- }, 5000)
+ if (['running'].includes(this.taskForm.status)) {
+ this.handle = setInterval(() => {
+ this.$store.dispatch('task/getTaskLog', this.$route.params.id)
+ }, 5000)
+ }
},
destroyed () {
clearInterval(this.handle)
diff --git a/spiders/realestate/realestate/pipelines.py b/spiders/realestate/realestate/pipelines.py
index a73934b3..7b9eb9f2 100644
--- a/spiders/realestate/realestate/pipelines.py
+++ b/spiders/realestate/realestate/pipelines.py
@@ -8,9 +8,9 @@ import os
from pymongo import MongoClient
-MONGO_HOST = os.environ['MONGO_HOST']
-MONGO_PORT = int(os.environ['MONGO_PORT'])
-MONGO_DB = os.environ['MONGO_DB']
+MONGO_HOST = os.environ.get('MONGO_HOST') or 'localhost'
+MONGO_PORT = int(os.environ.get('MONGO_PORT') or '27017')
+MONGO_DB = os.environ.get('MONGO_DB') or 'crawlab_test'
class MongoPipeline(object):