diff --git a/crawlab/routes/deploys.py b/crawlab/routes/deploys.py
index 2fd032ef..9cf4bf7c 100644
--- a/crawlab/routes/deploys.py
+++ b/crawlab/routes/deploys.py
@@ -35,7 +35,7 @@ class DeployApi(BaseApi):
spider = db_manager.get('spiders', id=str(spider_id))
item['spider_name'] = spider['name']
deploys.append(item)
- return jsonify({
+ return {
'status': 'ok',
- 'items': deploys
- })
+ 'items': jsonify(deploys)
+ }
diff --git a/crawlab/routes/nodes.py b/crawlab/routes/nodes.py
index 91bb9686..d0daa347 100644
--- a/crawlab/routes/nodes.py
+++ b/crawlab/routes/nodes.py
@@ -38,10 +38,10 @@ class NodeApi(BaseApi):
# iterate db nodes to update status
nodes = db_manager.list('nodes', {})
- return jsonify({
+ return {
'status': 'ok',
- 'items': nodes
- })
+ 'items': jsonify(nodes)
+ }
def get_spiders(self, id=None):
items = db_manager.list('spiders')
@@ -54,10 +54,10 @@ class NodeApi(BaseApi):
spider = db_manager.get('spiders', id=str(spider_id))
item['spider_name'] = spider['name']
deploys.append(item)
- return jsonify({
+ return {
'status': 'ok',
- 'items': deploys
- })
+ 'items': jsonify(deploys)
+ }
def get_tasks(self, id):
items = db_manager.list('tasks', {'node_id': id}, limit=10, sort_key='create_ts')
@@ -70,7 +70,7 @@ class NodeApi(BaseApi):
item['status'] = _task['status']
else:
item['status'] = TaskStatus.UNAVAILABLE
- return jsonify({
+ return {
'status': 'ok',
- 'items': items
- })
+ 'items': jsonify(items)
+ }
diff --git a/crawlab/routes/spiders.py b/crawlab/routes/spiders.py
index af6fdbea..f5fee0ab 100644
--- a/crawlab/routes/spiders.py
+++ b/crawlab/routes/spiders.py
@@ -110,10 +110,10 @@ class SpiderApi(BaseApi):
# append spider
items.append(spider)
- return jsonify({
+ return {
'status': 'ok',
- 'items': items
- })
+ 'items': jsonify(items)
+ }
def crawl(self, id):
args = self.parser.parse_args()
@@ -166,7 +166,7 @@ class SpiderApi(BaseApi):
def deploy(self, id):
spider = db_manager.get('spiders', id=id)
- nodes = db_manager.list('nodes', {})
+ nodes = db_manager.list('nodes', {'status': NodeStatus.ONLINE})
for node in nodes:
node_id = node['_id']
@@ -261,10 +261,10 @@ class SpiderApi(BaseApi):
spider = db_manager.get('spiders', id=str(spider_id))
item['spider_name'] = spider['name']
deploys.append(item)
- return jsonify({
+ return {
'status': 'ok',
- 'items': deploys
- })
+ 'items': jsonify(deploys)
+ }
def get_tasks(self, id):
items = db_manager.list('tasks', cond={'spider_id': ObjectId(id)}, limit=10, sort_key='finish_ts')
@@ -277,10 +277,10 @@ class SpiderApi(BaseApi):
item['status'] = task['status']
else:
item['status'] = TaskStatus.UNAVAILABLE
- return jsonify({
+ return {
'status': 'ok',
- 'items': items
- })
+ 'items': jsonify(items)
+ }
def after_update(self, id=None):
scheduler.update()
diff --git a/crawlab/routes/tasks.py b/crawlab/routes/tasks.py
index 175ba857..916fc4cc 100644
--- a/crawlab/routes/tasks.py
+++ b/crawlab/routes/tasks.py
@@ -58,10 +58,10 @@ class TaskApi(BaseApi):
task['status'] = TaskStatus.UNAVAILABLE
task['spider_name'] = _spider['name']
items.append(task)
- return jsonify({
+ return {
'status': 'ok',
- 'items': items
- })
+ 'items': jsonify(items)
+ }
def on_get_log(self, id):
try:
@@ -109,11 +109,11 @@ class TaskApi(BaseApi):
return []
fields = get_spider_col_fields(col_name)
items = db_manager.list(col_name, {'task_id': id})
- return jsonify({
+ return {
'status': 'ok',
- 'fields': fields,
- 'items': items
- })
+ 'fields': jsonify(fields),
+ 'items': jsonify(items)
+ }
def stop(self, id):
revoke(id, terminate=True)
diff --git a/crawlab/utils/__init__.py b/crawlab/utils/__init__.py
index feab944e..c1ce608b 100644
--- a/crawlab/utils/__init__.py
+++ b/crawlab/utils/__init__.py
@@ -1,5 +1,6 @@
import json
import re
+from datetime import datetime
from bson import json_util
@@ -8,7 +9,17 @@ def is_object_id(id):
return re.search('^[a-zA-Z0-9]{24}$', id) is not None
-def jsonify(obj: dict):
+def jsonify(obj):
dump_str = json_util.dumps(obj)
converted_obj = json.loads(dump_str)
+ if type(converted_obj) == dict:
+ for k, v in converted_obj.items():
+ if type(v) == dict:
+ if v.get('$oid') is not None:
+ converted_obj[k] = v['$oid']
+ elif v.get('$date') is not None:
+ converted_obj[k] = datetime.fromtimestamp(v['$date'] / 1000).strftime('%Y-%m-%d %H:%M:%S')
+ elif type(converted_obj) == list:
+ for i, v in enumerate(converted_obj):
+ converted_obj[i] = jsonify(v)
return converted_obj
diff --git a/frontend/src/components/Common/DialogView.vue b/frontend/src/components/Common/DialogView.vue
index 0f1f5932..7976171e 100644
--- a/frontend/src/components/Common/DialogView.vue
+++ b/frontend/src/components/Common/DialogView.vue
@@ -9,8 +9,8 @@
-
-
+
+
@@ -119,7 +119,7 @@ export default {
} else if (this.dialogType === 'nodeRun') {
} else if (this.dialogType === 'spiderDeploy') {
this.$store.dispatch('spider/deploySpider', {
- id: this.spiderForm._id.$oid,
+ id: this.spiderForm._id,
nodeId: this.activeNode._id
})
.then(() => {
@@ -133,7 +133,7 @@ export default {
this.$store.commit('dialogView/SET_DIALOG_VISIBLE', false)
})
} else if (this.dialogType === 'spiderRun') {
- this.$store.dispatch('spider/crawlSpider', this.spiderForm._id.$oid)
+ this.$store.dispatch('spider/crawlSpider', this.spiderForm._id)
.then(() => {
this.$message.success(`Spider "${this.spiderForm.name}" started to run on node "${this.activeNode._id}"`)
})
diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue
index d5d1fc71..7c82a6d0 100644
--- a/frontend/src/components/InfoView/SpiderInfoView.vue
+++ b/frontend/src/components/InfoView/SpiderInfoView.vue
@@ -7,7 +7,7 @@
class="spider-form"
label-position="right">
-
+
@@ -121,9 +121,9 @@ export default {
cancelButtonText: 'Cancel'
})
.then(() => {
- this.$store.dispatch('spider/crawlSpider', row._id.$oid)
+ this.$store.dispatch('spider/crawlSpider', row._id)
.then(() => {
- this.$message.success(`Running spider "${row._id.$oid}" has been scheduled`)
+ this.$message.success(`Running spider "${row._id}" has been scheduled`)
})
})
}
@@ -138,9 +138,9 @@ export default {
cancelButtonText: 'Cancel'
})
.then(() => {
- this.$store.dispatch('spider/deploySpider', row._id.$oid)
+ this.$store.dispatch('spider/deploySpider', row._id)
.then(() => {
- this.$message.success(`Spider "${row._id.$oid}" has been deployed`)
+ this.$message.success(`Spider "${row._id}" has been deployed`)
})
})
}
diff --git a/frontend/src/components/Overview/TaskOverview.vue b/frontend/src/components/Overview/TaskOverview.vue
index 37033f23..32a5fa00 100644
--- a/frontend/src/components/Overview/TaskOverview.vue
+++ b/frontend/src/components/Overview/TaskOverview.vue
@@ -54,7 +54,7 @@ export default {
this.$router.push(`/nodes/${this.nodeForm._id}`)
},
onClickSpiderTitle () {
- this.$router.push(`/spiders/${this.spiderForm._id.$oid}`)
+ this.$router.push(`/spiders/${this.spiderForm._id}`)
}
},
created () {
diff --git a/frontend/src/components/TableView/DeployTableView.vue b/frontend/src/components/TableView/DeployTableView.vue
index 0ebbef7a..3b6666bf 100644
--- a/frontend/src/components/TableView/DeployTableView.vue
+++ b/frontend/src/components/TableView/DeployTableView.vue
@@ -42,13 +42,13 @@ export default {
},
methods: {
onClickSpider (row) {
- this.$router.push(`/spiders/${row.spider_id.$oid}`)
+ this.$router.push(`/spiders/${row.spider_id}`)
},
onClickNode (row) {
this.$router.push(`/nodes/${row.node_id}`)
},
onRefresh () {
- this.$store.dispatch('deploy/getDeployList', this.spiderForm._id.$oid)
+ this.$store.dispatch('deploy/getDeployList', this.spiderForm._id)
}
}
}
diff --git a/frontend/src/components/TableView/GeneralTableView.vue b/frontend/src/components/TableView/GeneralTableView.vue
index 5a91b104..94d7f758 100644
--- a/frontend/src/components/TableView/GeneralTableView.vue
+++ b/frontend/src/components/TableView/GeneralTableView.vue
@@ -54,8 +54,8 @@ export default {
if (d.hasOwnProperty(k)) {
if (d[k] === undefined || d[k] === null) continue
if (typeof d[k] === 'object') {
- if (d[k].$oid) {
- d[k] = d[k].$oid
+ if (d[k]) {
+ d[k] = d[k]
}
}
}
diff --git a/frontend/src/components/TableView/TaskTableView.vue b/frontend/src/components/TableView/TaskTableView.vue
index afa2c053..49d08fce 100644
--- a/frontend/src/components/TableView/TaskTableView.vue
+++ b/frontend/src/components/TableView/TaskTableView.vue
@@ -56,7 +56,7 @@ export default {
},
methods: {
onClickSpider (row) {
- this.$router.push(`/spiders/${row.spider_id.$oid}`)
+ this.$router.push(`/spiders/${row.spider_id}`)
},
onClickNode (row) {
this.$router.push(`/nodes/${row.node_id}`)
diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js
index 30fb7a69..87060362 100644
--- a/frontend/src/store/modules/spider.js
+++ b/frontend/src/store/modules/spider.js
@@ -58,7 +58,7 @@ const actions = {
})
},
editSpider ({ state, dispatch }) {
- return request.post(`/spiders/${state.spiderForm._id.$oid}`, {
+ return request.post(`/spiders/${state.spiderForm._id}`, {
name: state.spiderForm.name,
src: state.spiderForm.src,
cmd: state.spiderForm.cmd,
diff --git a/frontend/src/store/modules/task.js b/frontend/src/store/modules/task.js
index 3e1e5b9c..992012e4 100644
--- a/frontend/src/store/modules/task.js
+++ b/frontend/src/store/modules/task.js
@@ -41,7 +41,7 @@ const actions = {
if (data.create_ts) data.create_ts = dayjs(data.create_ts.$date).format('YYYY-MM-DD HH:mm:ss')
if (data.finish_ts) data.finish_ts = dayjs(data.finish_ts.$date).format('YYYY-MM-DD HH:mm:ss')
commit('SET_TASK_FORM', data)
- dispatch('spider/getSpiderData', data.spider_id.$oid, { root: true })
+ dispatch('spider/getSpiderData', data.spider_id, { root: true })
dispatch('node/getNodeData', data.node_id, { root: true })
})
},
diff --git a/frontend/src/views/deploy/DeployList.vue b/frontend/src/views/deploy/DeployList.vue
index 0e936318..1b562fc9 100644
--- a/frontend/src/views/deploy/DeployList.vue
+++ b/frontend/src/views/deploy/DeployList.vue
@@ -139,7 +139,7 @@ export default {
this.$router.push(`/deploys/${row._id}`)
},
onClickSpider (row) {
- this.$router.push(`/spiders/${row.spider_id.$oid}`)
+ this.$router.push(`/spiders/${row.spider_id}`)
},
onClickNode (row) {
this.$router.push(`/nodes/${row.node_id}`)
diff --git a/frontend/src/views/result/ResultDetail.vue b/frontend/src/views/result/ResultDetail.vue
index 1cce669d..339d56b0 100644
--- a/frontend/src/views/result/ResultDetail.vue
+++ b/frontend/src/views/result/ResultDetail.vue
@@ -3,8 +3,8 @@
-
-
+
+
diff --git a/frontend/src/views/result/ResultList.vue b/frontend/src/views/result/ResultList.vue
index 5339a4a5..e724b9c7 100644
--- a/frontend/src/views/result/ResultList.vue
+++ b/frontend/src/views/result/ResultList.vue
@@ -233,7 +233,7 @@ export default {
cancelButtonText: 'Cancel',
type: 'warning'
}).then(() => {
- this.$store.dispatch('spider/deleteSpider', row._id.$oid)
+ this.$store.dispatch('spider/deleteSpider', row._id)
.then(() => {
this.$message({
type: 'success',
@@ -243,17 +243,17 @@ export default {
})
},
onDeploy (row) {
- this.$store.dispatch('spider/getSpiderData', row._id.$oid)
+ this.$store.dispatch('spider/getSpiderData', row._id)
this.$store.commit('dialogView/SET_DIALOG_VISIBLE', true)
this.$store.commit('dialogView/SET_DIALOG_TYPE', 'spiderDeploy')
},
onCrawl (row) {
- this.$store.dispatch('spider/getSpiderData', row._id.$oid)
+ this.$store.dispatch('spider/getSpiderData', row._id)
this.$store.commit('dialogView/SET_DIALOG_VISIBLE', true)
this.$store.commit('dialogView/SET_DIALOG_TYPE', 'spiderRun')
},
onView (row) {
- this.$router.push(`/spiders/${row._id.$oid}`)
+ this.$router.push(`/spiders/${row._id}`)
}
},
created () {
diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue
index 75f3c0c3..dfd4f917 100644
--- a/frontend/src/views/spider/SpiderDetail.vue
+++ b/frontend/src/views/spider/SpiderDetail.vue
@@ -3,8 +3,8 @@
-
-
+
+
diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue
index cdcfc497..543298bd 100644
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -231,7 +231,7 @@ export default {
cancelButtonText: 'Cancel',
type: 'warning'
}).then(() => {
- this.$store.dispatch('spider/deleteSpider', row._id.$oid)
+ this.$store.dispatch('spider/deleteSpider', row._id)
.then(() => {
this.$message({
type: 'success',
@@ -246,7 +246,7 @@ export default {
cancelButtonText: 'Cancel',
type: 'warning'
}).then(() => {
- this.$store.dispatch('spider/deploySpider', row._id.$oid)
+ this.$store.dispatch('spider/deploySpider', row._id)
.then(() => {
this.$message({
type: 'success',
@@ -261,14 +261,14 @@ export default {
cancelButtonText: 'Cancel'
})
.then(() => {
- this.$store.dispatch('spider/crawlSpider', row._id.$oid)
+ this.$store.dispatch('spider/crawlSpider', row._id)
.then(() => {
- this.$message.success(`Running spider "${row._id.$oid}" has been scheduled`)
+ this.$message.success(`Running spider "${row._id}" has been scheduled`)
})
})
},
onView (row) {
- this.$router.push(`/spiders/${row._id.$oid}`)
+ this.$router.push(`/spiders/${row._id}`)
},
onPageChange () {
this.$store.dispatch('spider/getSpiderList')
diff --git a/frontend/src/views/task/TaskList.vue b/frontend/src/views/task/TaskList.vue
index 60498977..2cfdd344 100644
--- a/frontend/src/views/task/TaskList.vue
+++ b/frontend/src/views/task/TaskList.vue
@@ -129,7 +129,7 @@ export default {
if (d.finish_ts) d.finish_ts = dayjs(d.finish_ts.$date).format('YYYY-MM-DD HH:mm:ss')
try {
- d.spider_id = d.spider_id.$oid
+ d.spider_id = d.spider_id
} catch (e) {
if (d.spider_id) d.spider_id = d.spider_id.toString()
}