git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

fixed inconsistency of timestamp issue

Browse Source
This commit is contained in:
Marvin Zhang
2019-03-12 20:28:50 +08:00
parent 9180e00d94
commit 508712ba07
19 changed files with 73 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
crawlab/routes/deploys.py
View File

@@ -35,7 +35,7 @@ class DeployApi(BaseApi):
spider = db_manager.get('spiders', id=str(spider_id))
item['spider_name'] = spider['name']
deploys.append(item)
return jsonify({
return {
'status': 'ok',
'items': deploys
})
'items': jsonify(deploys)
}

18
crawlab/routes/nodes.py
View File

@@ -38,10 +38,10 @@ class NodeApi(BaseApi):
# iterate db nodes to update status
nodes = db_manager.list('nodes', {})
return jsonify({
return {
'status': 'ok',
'items': nodes
})
'items': jsonify(nodes)
}
def get_spiders(self, id=None):
items = db_manager.list('spiders')
@@ -54,10 +54,10 @@ class NodeApi(BaseApi):
spider = db_manager.get('spiders', id=str(spider_id))
item['spider_name'] = spider['name']
deploys.append(item)
return jsonify({
return {
'status': 'ok',
'items': deploys
})
'items': jsonify(deploys)
}
def get_tasks(self, id):
items = db_manager.list('tasks', {'node_id': id}, limit=10, sort_key='create_ts')
@@ -70,7 +70,7 @@ class NodeApi(BaseApi):
item['status'] = _task['status']
else:
item['status'] = TaskStatus.UNAVAILABLE
return jsonify({
return {
'status': 'ok',
'items': items
})
'items': jsonify(items)
}

20
crawlab/routes/spiders.py
View File

@@ -110,10 +110,10 @@ class SpiderApi(BaseApi):
# append spider
items.append(spider)
return jsonify({
return {
'status': 'ok',
'items': items
})
'items': jsonify(items)
}
def crawl(self, id):
args = self.parser.parse_args()
@@ -166,7 +166,7 @@ class SpiderApi(BaseApi):
def deploy(self, id):
spider = db_manager.get('spiders', id=id)
nodes = db_manager.list('nodes', {})
nodes = db_manager.list('nodes', {'status': NodeStatus.ONLINE})
for node in nodes:
node_id = node['_id']
@@ -261,10 +261,10 @@ class SpiderApi(BaseApi):
spider = db_manager.get('spiders', id=str(spider_id))
item['spider_name'] = spider['name']
deploys.append(item)
return jsonify({
return {
'status': 'ok',
'items': deploys
})
'items': jsonify(deploys)
}
def get_tasks(self, id):
items = db_manager.list('tasks', cond={'spider_id': ObjectId(id)}, limit=10, sort_key='finish_ts')
@@ -277,10 +277,10 @@ class SpiderApi(BaseApi):
item['status'] = task['status']
else:
item['status'] = TaskStatus.UNAVAILABLE
return jsonify({
return {
'status': 'ok',
'items': items
})
'items': jsonify(items)
}
def after_update(self, id=None):
scheduler.update()

14
crawlab/routes/tasks.py
View File

@@ -58,10 +58,10 @@ class TaskApi(BaseApi):
task['status'] = TaskStatus.UNAVAILABLE
task['spider_name'] = _spider['name']
items.append(task)
return jsonify({
return {
'status': 'ok',
'items': items
})
'items': jsonify(items)
}
def on_get_log(self, id):
try:
@@ -109,11 +109,11 @@ class TaskApi(BaseApi):
return []
fields = get_spider_col_fields(col_name)
items = db_manager.list(col_name, {'task_id': id})
return jsonify({
return {
'status': 'ok',
'fields': fields,
'items': items
})
'fields': jsonify(fields),
'items': jsonify(items)
}
def stop(self, id):
revoke(id, terminate=True)

13
crawlab/utils/__init__.py
View File

@@ -1,5 +1,6 @@
import json
import re
from datetime import datetime
from bson import json_util
@@ -8,7 +9,17 @@ def is_object_id(id):
return re.search('^[a-zA-Z0-9]{24}$', id) is not None
def jsonify(obj: dict):
def jsonify(obj):
dump_str = json_util.dumps(obj)
converted_obj = json.loads(dump_str)
if type(converted_obj) == dict:
for k, v in converted_obj.items():
if type(v) == dict:
if v.get('$oid') is not None:
converted_obj[k] = v['$oid']
elif v.get('$date') is not None:
converted_obj[k] = datetime.fromtimestamp(v['$date'] / 1000).strftime('%Y-%m-%d %H:%M:%S')
elif type(converted_obj) == list:
for i, v in enumerate(converted_obj):
converted_obj[i] = jsonify(v)
return converted_obj

8
frontend/src/components/Common/DialogView.vue
View File

@@ -9,8 +9,8 @@
<label>{{message}}</label>
<!--selection for node-->
<el-select v-if="type === 'node'" v-model="activeSpider._id.$oid">
<el-option v-for="op in spiderList" :key="op._id.$oid" :value="op._id.$oid" :label="op.name"></el-option>
<el-select v-if="type === 'node'" v-model="activeSpider._id">
<el-option v-for="op in spiderList" :key="op._id" :value="op._id" :label="op.name"></el-option>
</el-select>
<!--selection for spider-->
@@ -119,7 +119,7 @@ export default {
} else if (this.dialogType === 'nodeRun') {
} else if (this.dialogType === 'spiderDeploy') {
this.$store.dispatch('spider/deploySpider', {
id: this.spiderForm._id.$oid,
id: this.spiderForm._id,
nodeId: this.activeNode._id
})
.then(() => {
@@ -133,7 +133,7 @@ export default {
this.$store.commit('dialogView/SET_DIALOG_VISIBLE', false)
})
} else if (this.dialogType === 'spiderRun') {
this.$store.dispatch('spider/crawlSpider', this.spiderForm._id.$oid)
this.$store.dispatch('spider/crawlSpider', this.spiderForm._id)
.then(() => {
this.$message.success(`Spider "${this.spiderForm.name}" started to run on node "${this.activeNode._id}"`)
})

10
frontend/src/components/InfoView/SpiderInfoView.vue
View File

@@ -7,7 +7,7 @@
class="spider-form"
label-position="right">
<el-form-item label="Spider ID">
<el-input v-model="spiderForm._id.$oid" placeholder="Spider ID" disabled></el-input>
<el-input v-model="spiderForm._id" placeholder="Spider ID" disabled></el-input>
</el-form-item>
<el-form-item label="Spider Name">
<el-input v-model="spiderForm.name" placeholder="Spider Name" :disabled="isView"></el-input>
@@ -121,9 +121,9 @@ export default {
cancelButtonText: 'Cancel'
})
.then(() => {
this.$store.dispatch('spider/crawlSpider', row._id.$oid)
this.$store.dispatch('spider/crawlSpider', row._id)
.then(() => {
this.$message.success(`Running spider "${row._id.$oid}" has been scheduled`)
this.$message.success(`Running spider "${row._id}" has been scheduled`)
})
})
}
@@ -138,9 +138,9 @@ export default {
cancelButtonText: 'Cancel'
})
.then(() => {
this.$store.dispatch('spider/deploySpider', row._id.$oid)
this.$store.dispatch('spider/deploySpider', row._id)
.then(() => {
this.$message.success(`Spider "${row._id.$oid}" has been deployed`)
this.$message.success(`Spider "${row._id}" has been deployed`)
})
})
}

2
frontend/src/components/Overview/TaskOverview.vue
View File

@@ -54,7 +54,7 @@ export default {
this.$router.push(`/nodes/${this.nodeForm._id}`)
},
onClickSpiderTitle () {
this.$router.push(`/spiders/${this.spiderForm._id.$oid}`)
this.$router.push(`/spiders/${this.spiderForm._id}`)
}
},
created () {

4
frontend/src/components/TableView/DeployTableView.vue
View File

@@ -42,13 +42,13 @@ export default {
},
methods: {
onClickSpider (row) {
this.$router.push(`/spiders/${row.spider_id.$oid}`)
this.$router.push(`/spiders/${row.spider_id}`)
},
onClickNode (row) {
this.$router.push(`/nodes/${row.node_id}`)
},
onRefresh () {
this.$store.dispatch('deploy/getDeployList', this.spiderForm._id.$oid)
this.$store.dispatch('deploy/getDeployList', this.spiderForm._id)
}
}
}

4
frontend/src/components/TableView/GeneralTableView.vue
View File

@@ -54,8 +54,8 @@ export default {
if (d.hasOwnProperty(k)) {
if (d[k] === undefined || d[k] === null) continue
if (typeof d[k] === 'object') {
if (d[k].$oid) {
d[k] = d[k].$oid
if (d[k]) {
d[k] = d[k]
}
}
}

2
frontend/src/components/TableView/TaskTableView.vue
View File

@@ -56,7 +56,7 @@ export default {
},
methods: {
onClickSpider (row) {
this.$router.push(`/spiders/${row.spider_id.$oid}`)
this.$router.push(`/spiders/${row.spider_id}`)
},
onClickNode (row) {
this.$router.push(`/nodes/${row.node_id}`)

2
frontend/src/store/modules/spider.js
View File

@@ -58,7 +58,7 @@ const actions = {
})
},
editSpider ({ state, dispatch }) {
return request.post(`/spiders/${state.spiderForm._id.$oid}`, {
return request.post(`/spiders/${state.spiderForm._id}`, {
name: state.spiderForm.name,
src: state.spiderForm.src,
cmd: state.spiderForm.cmd,

2
frontend/src/store/modules/task.js
View File

@@ -41,7 +41,7 @@ const actions = {
if (data.create_ts) data.create_ts = dayjs(data.create_ts.$date).format('YYYY-MM-DD HH:mm:ss')
if (data.finish_ts) data.finish_ts = dayjs(data.finish_ts.$date).format('YYYY-MM-DD HH:mm:ss')
commit('SET_TASK_FORM', data)
dispatch('spider/getSpiderData', data.spider_id.$oid, { root: true })
dispatch('spider/getSpiderData', data.spider_id, { root: true })
dispatch('node/getNodeData', data.node_id, { root: true })
})
},

2
frontend/src/views/deploy/DeployList.vue
View File

@@ -139,7 +139,7 @@ export default {
this.$router.push(`/deploys/${row._id}`)
},
onClickSpider (row) {
this.$router.push(`/spiders/${row.spider_id.$oid}`)
this.$router.push(`/spiders/${row.spider_id}`)
},
onClickNode (row) {
this.$router.push(`/nodes/${row.node_id}`)

4
frontend/src/views/result/ResultDetail.vue
View File

@@ -3,8 +3,8 @@
<!--selector-->
<div class="selector">
<label class="label">Spider: </label>
<el-select v-model="spiderForm._id.$oid" @change="onSpiderChange">
<el-option v-for="op in spiderList" :key="op._id.$oid" :value="op._id.$oid" :label="op.name"></el-option>
<el-select v-model="spiderForm._id" @change="onSpiderChange">
<el-option v-for="op in spiderList" :key="op._id" :value="op._id" :label="op.name"></el-option>
</el-select>
</div>

8
frontend/src/views/result/ResultList.vue
View File

@@ -233,7 +233,7 @@ export default {
cancelButtonText: 'Cancel',
type: 'warning'
}).then(() => {
this.$store.dispatch('spider/deleteSpider', row._id.$oid)
this.$store.dispatch('spider/deleteSpider', row._id)
.then(() => {
this.$message({
type: 'success',
@@ -243,17 +243,17 @@ export default {
})
},
onDeploy (row) {
this.$store.dispatch('spider/getSpiderData', row._id.$oid)
this.$store.dispatch('spider/getSpiderData', row._id)
this.$store.commit('dialogView/SET_DIALOG_VISIBLE', true)
this.$store.commit('dialogView/SET_DIALOG_TYPE', 'spiderDeploy')
},
onCrawl (row) {
this.$store.dispatch('spider/getSpiderData', row._id.$oid)
this.$store.dispatch('spider/getSpiderData', row._id)
this.$store.commit('dialogView/SET_DIALOG_VISIBLE', true)
this.$store.commit('dialogView/SET_DIALOG_TYPE', 'spiderRun')
},
onView (row) {
this.$router.push(`/spiders/${row._id.$oid}`)
this.$router.push(`/spiders/${row._id}`)
}
},
created () {

4
frontend/src/views/spider/SpiderDetail.vue
View File

@@ -3,8 +3,8 @@
<!--selector-->
<div class="selector">
<label class="label">Spider: </label>
<el-select v-model="spiderForm._id.$oid" @change="onSpiderChange">
<el-option v-for="op in spiderList" :key="op._id.$oid" :value="op._id.$oid" :label="op.name"></el-option>
<el-select v-model="spiderForm._id" @change="onSpiderChange">
<el-option v-for="op in spiderList" :key="op._id" :value="op._id" :label="op.name"></el-option>
</el-select>
</div>

10
frontend/src/views/spider/SpiderList.vue
View File

@@ -231,7 +231,7 @@ export default {
cancelButtonText: 'Cancel',
type: 'warning'
}).then(() => {
this.$store.dispatch('spider/deleteSpider', row._id.$oid)
this.$store.dispatch('spider/deleteSpider', row._id)
.then(() => {
this.$message({
type: 'success',
@@ -246,7 +246,7 @@ export default {
cancelButtonText: 'Cancel',
type: 'warning'
}).then(() => {
this.$store.dispatch('spider/deploySpider', row._id.$oid)
this.$store.dispatch('spider/deploySpider', row._id)
.then(() => {
this.$message({
type: 'success',
@@ -261,14 +261,14 @@ export default {
cancelButtonText: 'Cancel'
})
.then(() => {
this.$store.dispatch('spider/crawlSpider', row._id.$oid)
this.$store.dispatch('spider/crawlSpider', row._id)
.then(() => {
this.$message.success(`Running spider "${row._id.$oid}" has been scheduled`)
this.$message.success(`Running spider "${row._id}" has been scheduled`)
})
})
},
onView (row) {
this.$router.push(`/spiders/${row._id.$oid}`)
this.$router.push(`/spiders/${row._id}`)
},
onPageChange () {
this.$store.dispatch('spider/getSpiderList')

2
frontend/src/views/task/TaskList.vue
View File

@@ -129,7 +129,7 @@ export default {
if (d.finish_ts) d.finish_ts = dayjs(d.finish_ts.$date).format('YYYY-MM-DD HH:mm:ss')
try {
d.spider_id = d.spider_id.$oid
d.spider_id = d.spider_id
} catch (e) {
if (d.spider_id) d.spider_id = d.spider_id.toString()
}