diff --git a/crawlab/routes/deploys.py b/crawlab/routes/deploys.py index 2fd032ef..9cf4bf7c 100644 --- a/crawlab/routes/deploys.py +++ b/crawlab/routes/deploys.py @@ -35,7 +35,7 @@ class DeployApi(BaseApi): spider = db_manager.get('spiders', id=str(spider_id)) item['spider_name'] = spider['name'] deploys.append(item) - return jsonify({ + return { 'status': 'ok', - 'items': deploys - }) + 'items': jsonify(deploys) + } diff --git a/crawlab/routes/nodes.py b/crawlab/routes/nodes.py index 91bb9686..d0daa347 100644 --- a/crawlab/routes/nodes.py +++ b/crawlab/routes/nodes.py @@ -38,10 +38,10 @@ class NodeApi(BaseApi): # iterate db nodes to update status nodes = db_manager.list('nodes', {}) - return jsonify({ + return { 'status': 'ok', - 'items': nodes - }) + 'items': jsonify(nodes) + } def get_spiders(self, id=None): items = db_manager.list('spiders') @@ -54,10 +54,10 @@ class NodeApi(BaseApi): spider = db_manager.get('spiders', id=str(spider_id)) item['spider_name'] = spider['name'] deploys.append(item) - return jsonify({ + return { 'status': 'ok', - 'items': deploys - }) + 'items': jsonify(deploys) + } def get_tasks(self, id): items = db_manager.list('tasks', {'node_id': id}, limit=10, sort_key='create_ts') @@ -70,7 +70,7 @@ class NodeApi(BaseApi): item['status'] = _task['status'] else: item['status'] = TaskStatus.UNAVAILABLE - return jsonify({ + return { 'status': 'ok', - 'items': items - }) + 'items': jsonify(items) + } diff --git a/crawlab/routes/spiders.py b/crawlab/routes/spiders.py index af6fdbea..f5fee0ab 100644 --- a/crawlab/routes/spiders.py +++ b/crawlab/routes/spiders.py @@ -110,10 +110,10 @@ class SpiderApi(BaseApi): # append spider items.append(spider) - return jsonify({ + return { 'status': 'ok', - 'items': items - }) + 'items': jsonify(items) + } def crawl(self, id): args = self.parser.parse_args() @@ -166,7 +166,7 @@ class SpiderApi(BaseApi): def deploy(self, id): spider = db_manager.get('spiders', id=id) - nodes = db_manager.list('nodes', {}) + nodes = db_manager.list('nodes', {'status': NodeStatus.ONLINE}) for node in nodes: node_id = node['_id'] @@ -261,10 +261,10 @@ class SpiderApi(BaseApi): spider = db_manager.get('spiders', id=str(spider_id)) item['spider_name'] = spider['name'] deploys.append(item) - return jsonify({ + return { 'status': 'ok', - 'items': deploys - }) + 'items': jsonify(deploys) + } def get_tasks(self, id): items = db_manager.list('tasks', cond={'spider_id': ObjectId(id)}, limit=10, sort_key='finish_ts') @@ -277,10 +277,10 @@ class SpiderApi(BaseApi): item['status'] = task['status'] else: item['status'] = TaskStatus.UNAVAILABLE - return jsonify({ + return { 'status': 'ok', - 'items': items - }) + 'items': jsonify(items) + } def after_update(self, id=None): scheduler.update() diff --git a/crawlab/routes/tasks.py b/crawlab/routes/tasks.py index 175ba857..916fc4cc 100644 --- a/crawlab/routes/tasks.py +++ b/crawlab/routes/tasks.py @@ -58,10 +58,10 @@ class TaskApi(BaseApi): task['status'] = TaskStatus.UNAVAILABLE task['spider_name'] = _spider['name'] items.append(task) - return jsonify({ + return { 'status': 'ok', - 'items': items - }) + 'items': jsonify(items) + } def on_get_log(self, id): try: @@ -109,11 +109,11 @@ class TaskApi(BaseApi): return [] fields = get_spider_col_fields(col_name) items = db_manager.list(col_name, {'task_id': id}) - return jsonify({ + return { 'status': 'ok', - 'fields': fields, - 'items': items - }) + 'fields': jsonify(fields), + 'items': jsonify(items) + } def stop(self, id): revoke(id, terminate=True) diff --git a/crawlab/utils/__init__.py b/crawlab/utils/__init__.py index feab944e..c1ce608b 100644 --- a/crawlab/utils/__init__.py +++ b/crawlab/utils/__init__.py @@ -1,5 +1,6 @@ import json import re +from datetime import datetime from bson import json_util @@ -8,7 +9,17 @@ def is_object_id(id): return re.search('^[a-zA-Z0-9]{24}$', id) is not None -def jsonify(obj: dict): +def jsonify(obj): dump_str = json_util.dumps(obj) converted_obj = json.loads(dump_str) + if type(converted_obj) == dict: + for k, v in converted_obj.items(): + if type(v) == dict: + if v.get('$oid') is not None: + converted_obj[k] = v['$oid'] + elif v.get('$date') is not None: + converted_obj[k] = datetime.fromtimestamp(v['$date'] / 1000).strftime('%Y-%m-%d %H:%M:%S') + elif type(converted_obj) == list: + for i, v in enumerate(converted_obj): + converted_obj[i] = jsonify(v) return converted_obj diff --git a/frontend/src/components/Common/DialogView.vue b/frontend/src/components/Common/DialogView.vue index 0f1f5932..7976171e 100644 --- a/frontend/src/components/Common/DialogView.vue +++ b/frontend/src/components/Common/DialogView.vue @@ -9,8 +9,8 @@ - - + + @@ -119,7 +119,7 @@ export default { } else if (this.dialogType === 'nodeRun') { } else if (this.dialogType === 'spiderDeploy') { this.$store.dispatch('spider/deploySpider', { - id: this.spiderForm._id.$oid, + id: this.spiderForm._id, nodeId: this.activeNode._id }) .then(() => { @@ -133,7 +133,7 @@ export default { this.$store.commit('dialogView/SET_DIALOG_VISIBLE', false) }) } else if (this.dialogType === 'spiderRun') { - this.$store.dispatch('spider/crawlSpider', this.spiderForm._id.$oid) + this.$store.dispatch('spider/crawlSpider', this.spiderForm._id) .then(() => { this.$message.success(`Spider "${this.spiderForm.name}" started to run on node "${this.activeNode._id}"`) }) diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index d5d1fc71..7c82a6d0 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -7,7 +7,7 @@ class="spider-form" label-position="right"> - + @@ -121,9 +121,9 @@ export default { cancelButtonText: 'Cancel' }) .then(() => { - this.$store.dispatch('spider/crawlSpider', row._id.$oid) + this.$store.dispatch('spider/crawlSpider', row._id) .then(() => { - this.$message.success(`Running spider "${row._id.$oid}" has been scheduled`) + this.$message.success(`Running spider "${row._id}" has been scheduled`) }) }) } @@ -138,9 +138,9 @@ export default { cancelButtonText: 'Cancel' }) .then(() => { - this.$store.dispatch('spider/deploySpider', row._id.$oid) + this.$store.dispatch('spider/deploySpider', row._id) .then(() => { - this.$message.success(`Spider "${row._id.$oid}" has been deployed`) + this.$message.success(`Spider "${row._id}" has been deployed`) }) }) } diff --git a/frontend/src/components/Overview/TaskOverview.vue b/frontend/src/components/Overview/TaskOverview.vue index 37033f23..32a5fa00 100644 --- a/frontend/src/components/Overview/TaskOverview.vue +++ b/frontend/src/components/Overview/TaskOverview.vue @@ -54,7 +54,7 @@ export default { this.$router.push(`/nodes/${this.nodeForm._id}`) }, onClickSpiderTitle () { - this.$router.push(`/spiders/${this.spiderForm._id.$oid}`) + this.$router.push(`/spiders/${this.spiderForm._id}`) } }, created () { diff --git a/frontend/src/components/TableView/DeployTableView.vue b/frontend/src/components/TableView/DeployTableView.vue index 0ebbef7a..3b6666bf 100644 --- a/frontend/src/components/TableView/DeployTableView.vue +++ b/frontend/src/components/TableView/DeployTableView.vue @@ -42,13 +42,13 @@ export default { }, methods: { onClickSpider (row) { - this.$router.push(`/spiders/${row.spider_id.$oid}`) + this.$router.push(`/spiders/${row.spider_id}`) }, onClickNode (row) { this.$router.push(`/nodes/${row.node_id}`) }, onRefresh () { - this.$store.dispatch('deploy/getDeployList', this.spiderForm._id.$oid) + this.$store.dispatch('deploy/getDeployList', this.spiderForm._id) } } } diff --git a/frontend/src/components/TableView/GeneralTableView.vue b/frontend/src/components/TableView/GeneralTableView.vue index 5a91b104..94d7f758 100644 --- a/frontend/src/components/TableView/GeneralTableView.vue +++ b/frontend/src/components/TableView/GeneralTableView.vue @@ -54,8 +54,8 @@ export default { if (d.hasOwnProperty(k)) { if (d[k] === undefined || d[k] === null) continue if (typeof d[k] === 'object') { - if (d[k].$oid) { - d[k] = d[k].$oid + if (d[k]) { + d[k] = d[k] } } } diff --git a/frontend/src/components/TableView/TaskTableView.vue b/frontend/src/components/TableView/TaskTableView.vue index afa2c053..49d08fce 100644 --- a/frontend/src/components/TableView/TaskTableView.vue +++ b/frontend/src/components/TableView/TaskTableView.vue @@ -56,7 +56,7 @@ export default { }, methods: { onClickSpider (row) { - this.$router.push(`/spiders/${row.spider_id.$oid}`) + this.$router.push(`/spiders/${row.spider_id}`) }, onClickNode (row) { this.$router.push(`/nodes/${row.node_id}`) diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js index 30fb7a69..87060362 100644 --- a/frontend/src/store/modules/spider.js +++ b/frontend/src/store/modules/spider.js @@ -58,7 +58,7 @@ const actions = { }) }, editSpider ({ state, dispatch }) { - return request.post(`/spiders/${state.spiderForm._id.$oid}`, { + return request.post(`/spiders/${state.spiderForm._id}`, { name: state.spiderForm.name, src: state.spiderForm.src, cmd: state.spiderForm.cmd, diff --git a/frontend/src/store/modules/task.js b/frontend/src/store/modules/task.js index 3e1e5b9c..992012e4 100644 --- a/frontend/src/store/modules/task.js +++ b/frontend/src/store/modules/task.js @@ -41,7 +41,7 @@ const actions = { if (data.create_ts) data.create_ts = dayjs(data.create_ts.$date).format('YYYY-MM-DD HH:mm:ss') if (data.finish_ts) data.finish_ts = dayjs(data.finish_ts.$date).format('YYYY-MM-DD HH:mm:ss') commit('SET_TASK_FORM', data) - dispatch('spider/getSpiderData', data.spider_id.$oid, { root: true }) + dispatch('spider/getSpiderData', data.spider_id, { root: true }) dispatch('node/getNodeData', data.node_id, { root: true }) }) }, diff --git a/frontend/src/views/deploy/DeployList.vue b/frontend/src/views/deploy/DeployList.vue index 0e936318..1b562fc9 100644 --- a/frontend/src/views/deploy/DeployList.vue +++ b/frontend/src/views/deploy/DeployList.vue @@ -139,7 +139,7 @@ export default { this.$router.push(`/deploys/${row._id}`) }, onClickSpider (row) { - this.$router.push(`/spiders/${row.spider_id.$oid}`) + this.$router.push(`/spiders/${row.spider_id}`) }, onClickNode (row) { this.$router.push(`/nodes/${row.node_id}`) diff --git a/frontend/src/views/result/ResultDetail.vue b/frontend/src/views/result/ResultDetail.vue index 1cce669d..339d56b0 100644 --- a/frontend/src/views/result/ResultDetail.vue +++ b/frontend/src/views/result/ResultDetail.vue @@ -3,8 +3,8 @@
- - + +
diff --git a/frontend/src/views/result/ResultList.vue b/frontend/src/views/result/ResultList.vue index 5339a4a5..e724b9c7 100644 --- a/frontend/src/views/result/ResultList.vue +++ b/frontend/src/views/result/ResultList.vue @@ -233,7 +233,7 @@ export default { cancelButtonText: 'Cancel', type: 'warning' }).then(() => { - this.$store.dispatch('spider/deleteSpider', row._id.$oid) + this.$store.dispatch('spider/deleteSpider', row._id) .then(() => { this.$message({ type: 'success', @@ -243,17 +243,17 @@ export default { }) }, onDeploy (row) { - this.$store.dispatch('spider/getSpiderData', row._id.$oid) + this.$store.dispatch('spider/getSpiderData', row._id) this.$store.commit('dialogView/SET_DIALOG_VISIBLE', true) this.$store.commit('dialogView/SET_DIALOG_TYPE', 'spiderDeploy') }, onCrawl (row) { - this.$store.dispatch('spider/getSpiderData', row._id.$oid) + this.$store.dispatch('spider/getSpiderData', row._id) this.$store.commit('dialogView/SET_DIALOG_VISIBLE', true) this.$store.commit('dialogView/SET_DIALOG_TYPE', 'spiderRun') }, onView (row) { - this.$router.push(`/spiders/${row._id.$oid}`) + this.$router.push(`/spiders/${row._id}`) } }, created () { diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue index 75f3c0c3..dfd4f917 100644 --- a/frontend/src/views/spider/SpiderDetail.vue +++ b/frontend/src/views/spider/SpiderDetail.vue @@ -3,8 +3,8 @@
- - + +
diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index cdcfc497..543298bd 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -231,7 +231,7 @@ export default { cancelButtonText: 'Cancel', type: 'warning' }).then(() => { - this.$store.dispatch('spider/deleteSpider', row._id.$oid) + this.$store.dispatch('spider/deleteSpider', row._id) .then(() => { this.$message({ type: 'success', @@ -246,7 +246,7 @@ export default { cancelButtonText: 'Cancel', type: 'warning' }).then(() => { - this.$store.dispatch('spider/deploySpider', row._id.$oid) + this.$store.dispatch('spider/deploySpider', row._id) .then(() => { this.$message({ type: 'success', @@ -261,14 +261,14 @@ export default { cancelButtonText: 'Cancel' }) .then(() => { - this.$store.dispatch('spider/crawlSpider', row._id.$oid) + this.$store.dispatch('spider/crawlSpider', row._id) .then(() => { - this.$message.success(`Running spider "${row._id.$oid}" has been scheduled`) + this.$message.success(`Running spider "${row._id}" has been scheduled`) }) }) }, onView (row) { - this.$router.push(`/spiders/${row._id.$oid}`) + this.$router.push(`/spiders/${row._id}`) }, onPageChange () { this.$store.dispatch('spider/getSpiderList') diff --git a/frontend/src/views/task/TaskList.vue b/frontend/src/views/task/TaskList.vue index 60498977..2cfdd344 100644 --- a/frontend/src/views/task/TaskList.vue +++ b/frontend/src/views/task/TaskList.vue @@ -129,7 +129,7 @@ export default { if (d.finish_ts) d.finish_ts = dayjs(d.finish_ts.$date).format('YYYY-MM-DD HH:mm:ss') try { - d.spider_id = d.spider_id.$oid + d.spider_id = d.spider_id } catch (e) { if (d.spider_id) d.spider_id = d.spider_id.toString() }