From 6ed8a449ce7d7f832a4438fcfc1532ead093c31a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 13 Mar 2019 21:33:47 +0800 Subject: [PATCH] changed the way how task is saved to database --- crawlab/routes/spiders.py | 10 +++- crawlab/routes/tasks.py | 8 ++- crawlab/tasks/spider.py | 10 ++-- .../components/TableView/TaskTableView.vue | 2 +- frontend/src/store/modules/task.js | 1 - frontend/src/views/task/TaskList.vue | 11 ++-- spiders/segmentfault/package.json | 3 +- spiders/segmentfault/screenshot.png | Bin 147399 -> 0 bytes spiders/segmentfault/segmentfault_spider.js | 48 +++++++++++++----- 9 files changed, 59 insertions(+), 34 deletions(-) delete mode 100644 spiders/segmentfault/screenshot.png diff --git a/crawlab/routes/spiders.py b/crawlab/routes/spiders.py index 4cfc9457..89881223 100644 --- a/crawlab/routes/spiders.py +++ b/crawlab/routes/spiders.py @@ -155,6 +155,14 @@ class SpiderApi(BaseApi): def on_crawl(self, id): job = execute_spider.delay(id) + # create a new task + db_manager.save('tasks', { + '_id': job.id, + 'spider_id': ObjectId(id), + 'create_ts': datetime.now(), + 'status': TaskStatus.PENDING + }) + return { 'code': 200, 'status': 'ok', @@ -267,7 +275,7 @@ class SpiderApi(BaseApi): } def get_tasks(self, id): - items = db_manager.list('tasks', cond={'spider_id': ObjectId(id)}, limit=10, sort_key='finish_ts') + items = db_manager.list('tasks', cond={'spider_id': ObjectId(id)}, limit=10, sort_key='create_ts') for item in items: spider_id = item['spider_id'] spider = db_manager.get('spiders', id=str(spider_id)) diff --git a/crawlab/routes/tasks.py b/crawlab/routes/tasks.py index 24a2f7ca..1b13f87e 100644 --- a/crawlab/routes/tasks.py +++ b/crawlab/routes/tasks.py @@ -49,14 +49,12 @@ class TaskApi(BaseApi): args = self.parser.parse_args() page_size = args.get('page_size') or 10 page_num = args.get('page_num') or 1 - tasks = db_manager.list('tasks', {}, limit=page_size, skip=page_size * (page_num - 1), sort_key='finish_ts') + tasks = db_manager.list('tasks', {}, limit=page_size, skip=page_size * (page_num - 1), sort_key='create_ts') items = [] for task in tasks: - _task = db_manager.get('tasks_celery', id=task['_id']) + # _task = db_manager.get('tasks_celery', id=task['_id']) _spider = db_manager.get('spiders', id=str(task['spider_id'])) - if _task: - task['status'] = _task['status'] - else: + if task.get('status') is None: task['status'] = TaskStatus.UNAVAILABLE task['spider_name'] = _spider['name'] items.append(task) diff --git a/crawlab/tasks/spider.py b/crawlab/tasks/spider.py index feef3003..93f27a67 100644 --- a/crawlab/tasks/spider.py +++ b/crawlab/tasks/spider.py @@ -38,17 +38,15 @@ def execute_spider(self, id: str): stderr = open(log_file_path, 'a') # create a new task - db_manager.save('tasks', { - '_id': task_id, - 'spider_id': ObjectId(id), - 'create_ts': datetime.now(), + db_manager.update_one('tasks', id=task_id, values={ + 'start_ts': datetime.now(), 'node_id': 'celery@%s' % hostname, 'hostname': hostname, 'log_file_path': log_file_path, - 'status': TaskStatus.PENDING + 'status': TaskStatus.STARTED }) - # execute the command + # start the process and pass params as env variables env = os.environ.copy() env['CRAWLAB_TASK_ID'] = task_id if spider.get('col'): diff --git a/frontend/src/components/TableView/TaskTableView.vue b/frontend/src/components/TableView/TaskTableView.vue index 49d08fce..83ffc2a7 100644 --- a/frontend/src/components/TableView/TaskTableView.vue +++ b/frontend/src/components/TableView/TaskTableView.vue @@ -20,7 +20,7 @@ width="100"> diff --git a/frontend/src/store/modules/task.js b/frontend/src/store/modules/task.js index 7b2321c7..c421a79b 100644 --- a/frontend/src/store/modules/task.js +++ b/frontend/src/store/modules/task.js @@ -1,4 +1,3 @@ -import dayjs from 'dayjs' import request from '../../api/request' const state = { diff --git a/frontend/src/views/task/TaskList.vue b/frontend/src/views/task/TaskList.vue index bee8e0c5..94899d6c 100644 --- a/frontend/src/views/task/TaskList.vue +++ b/frontend/src/views/task/TaskList.vue @@ -52,7 +52,7 @@ :width="col.width"> @@ -66,7 +66,7 @@ :width="col.width"> - +