From d48fee0d032a62139e6acf6a8064e18561e1c5ec Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 15 Apr 2019 19:50:25 +0800 Subject: [PATCH] enforce deploy before running a spider --- crawlab/db/manager.py | 12 +++++++++- crawlab/routes/spiders.py | 24 ++++++++++++++++++- crawlab/routes/tasks.py | 14 +++++++---- .../components/InfoView/SpiderInfoView.vue | 18 ++++++++++++-- .../src/components/InfoView/TaskInfoView.vue | 4 ++-- frontend/src/i18n/zh.js | 2 ++ frontend/src/store/modules/spider.js | 4 ++++ frontend/src/views/node/NodeList.vue | 6 ++--- frontend/src/views/spider/SpiderList.vue | 21 +++++++++++----- frontend/src/views/task/TaskList.vue | 3 +++ 10 files changed, 89 insertions(+), 19 deletions(-) diff --git a/crawlab/db/manager.py b/crawlab/db/manager.py index 902992cf..d210b81c 100644 --- a/crawlab/db/manager.py +++ b/crawlab/db/manager.py @@ -147,7 +147,7 @@ class DbManager(object): def get_last_deploy(self, spider_id): """ - @deprecated + Get latest deploy for a given spider_id """ col = self.db['deploys'] for item in col.find({'spider_id': ObjectId(spider_id)}) \ @@ -155,6 +155,16 @@ class DbManager(object): return item return None + def get_last_task(self, spider_id): + """ + Get latest deploy for a given spider_id + """ + col = self.db['tasks'] + for item in col.find({'spider_id': ObjectId(spider_id)}) \ + .sort('create_ts', DESCENDING): + return item + return None + def aggregate(self, col_name: str, pipelines, **kwargs): """ Perform MongoDB col.aggregate action to aggregate stats given collection name and pipelines. diff --git a/crawlab/routes/spiders.py b/crawlab/routes/spiders.py index b4b3aab1..f36903e3 100644 --- a/crawlab/routes/spiders.py +++ b/crawlab/routes/spiders.py @@ -81,7 +81,14 @@ class SpiderApi(BaseApi): # get one node elif id is not None: - return jsonify(db_manager.get('spiders', id=id)) + spider = db_manager.get('spiders', id=id) + + # get deploy + last_deploy = db_manager.get_last_deploy(spider_id=spider['_id']) + if last_deploy is not None: + spider['deploy_ts'] = last_deploy['finish_ts'] + + return jsonify(spider) # get a list of items else: @@ -108,8 +115,23 @@ class SpiderApi(BaseApi): # existing spider else: + # get last deploy + last_deploy = db_manager.get_last_deploy(spider_id=spider['_id']) + if last_deploy is not None: + spider['deploy_ts'] = last_deploy['finish_ts'] + + # get last task + last_task = db_manager.get_last_task(spider_id=spider['_id']) + if last_task is not None: + spider['task_ts'] = last_task['create_ts'] + + # file stats stats = get_file_suffix_stats(dir_path) + + # language lang = get_lang_by_stats(stats) + + # update spider data db_manager.update_one('spiders', id=str(spider['_id']), values={ 'lang': lang, 'suffix_stats': stats, diff --git a/crawlab/routes/tasks.py b/crawlab/routes/tasks.py index d864f859..59e8469b 100644 --- a/crawlab/routes/tasks.py +++ b/crawlab/routes/tasks.py @@ -1,8 +1,9 @@ import json +from datetime import datetime import requests from bson import ObjectId -from celery.worker.control import revoke +from tasks.celery import celery_app from constants.task import TaskStatus from db.manager import db_manager @@ -42,6 +43,8 @@ class TaskApi(BaseApi): task = db_manager.get(col_name=self.col_name, id=id) spider = db_manager.get(col_name='spiders', id=str(task['spider_id'])) task['spider_name'] = spider['name'] + if task.get('finish_ts') is not None: + task['duration'] = (task['finish_ts'] - task['create_ts']).total_seconds() try: with open(task['log_file_path']) as f: task['log'] = f.read() @@ -61,7 +64,8 @@ class TaskApi(BaseApi): _spider = db_manager.get(col_name='spiders', id=str(task['spider_id'])) if task.get('status') is None: task['status'] = TaskStatus.UNAVAILABLE - task['spider_name'] = _spider['name'] + if _spider: + task['spider_name'] = _spider['name'] items.append(task) return { 'status': 'ok', @@ -146,11 +150,13 @@ class TaskApi(BaseApi): def stop(self, id): """ Stop the task in progress. - TODO: work in progress :param id: :return: """ - revoke(id, terminate=True) + celery_app.control.revoke(id, terminate=True) + db_manager.update_one('tasks', id=id, values={ + 'status': TaskStatus.REVOKED + }) return { 'id': id, 'status': 'ok', diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index 852b9c1f..a02088ad 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -61,7 +61,7 @@ - {{$t('Run')}} + {{$t('Run')}} {{$t('Deploy')}} {{$t('Save')}} @@ -109,7 +109,16 @@ export default { computed: { ...mapState('spider', [ 'spiderForm' - ]) + ]), + isShowRun () { + if (!this.spiderForm.deploy_ts) { + return false + } + if (!this.spiderForm.cmd) { + return false + } + return true + } }, methods: { onRun () { @@ -131,6 +140,11 @@ export default { }, onDeploy () { const row = this.spiderForm + + // save spider + this.$store.dispatch('spider/editSpider', row._id) + + // validate fields this.$refs['spiderForm'].validate(res => { if (res) { this.$confirm(this.$t('Are you sure to deploy this spider?'), this.$t('Notification'), { diff --git a/frontend/src/components/InfoView/TaskInfoView.vue b/frontend/src/components/InfoView/TaskInfoView.vue index 98ba7c8f..8b6fdd16 100644 --- a/frontend/src/components/InfoView/TaskInfoView.vue +++ b/frontend/src/components/InfoView/TaskInfoView.vue @@ -32,13 +32,13 @@
- {{taskForm.result}} + {{taskForm.log}}
- Stop + {{$t('Stop')}} diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index f1f0cf6f..ff19069b 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -31,6 +31,7 @@ export default { SUCCESS: '成功', FAILURE: '错误', UNAVAILABLE: '未知', + REVOKED: '已取消', // 操作 Run: '运行', @@ -46,6 +47,7 @@ export default { Edit: '编辑', Remove: '删除', Confirm: '确认', + Stop: '停止', // 主页 'Total Tasks': '总任务数', diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js index dba4876e..1e18ecde 100644 --- a/frontend/src/store/modules/spider.js +++ b/frontend/src/store/modules/spider.js @@ -95,6 +95,10 @@ const actions = { .then(response => { console.log(response.data) }) + .then(response => { + dispatch('getSpiderData', id) + dispatch('getSpiderList') + }) }, crawlSpider ({ state, dispatch }, id) { return request.post(`/spiders/${id}/on_crawl`) diff --git a/frontend/src/views/node/NodeList.vue b/frontend/src/views/node/NodeList.vue index d0cb11d4..35dc7d18 100644 --- a/frontend/src/views/node/NodeList.vue +++ b/frontend/src/views/node/NodeList.vue @@ -50,9 +50,9 @@ - - - + + + diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 4ae640ea..60785efe 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -93,21 +93,21 @@ :width="col.width"> - + @@ -151,7 +151,7 @@ export default { { name: 'name', label: 'Name', width: 'auto' }, { name: 'type', label: 'Spider Type', width: '160', sortable: true }, { name: 'lang', label: 'Language', width: '160', sortable: true }, - { name: 'last_run_ts', label: 'Last Run', width: '120' } + { name: 'task_ts', label: 'Last Run', width: '160' } ], spiderFormRules: { name: [{ required: true, message: 'Required Field', trigger: 'change' }] @@ -301,6 +301,15 @@ export default { this.$message.success(this.$t('Deployed all spiders successfully')) }) }) + }, + isShowRun (row) { + if (!row.deploy_ts) { + return false + } + if (!row.cmd) { + return false + } + return true } }, created () { diff --git a/frontend/src/views/task/TaskList.vue b/frontend/src/views/task/TaskList.vue index bf42dfce..87fa6804 100644 --- a/frontend/src/views/task/TaskList.vue +++ b/frontend/src/views/task/TaskList.vue @@ -71,6 +71,9 @@ + + +