From 0ad030328543c5fa4cb8c2e7a89a0239d60186d2 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 9 May 2019 22:14:37 +0800 Subject: [PATCH] added site list --- crawlab/app.py | 4 + crawlab/routes/sites.py | 64 +++++++ crawlab/routes/tasks.py | 4 - .../components/InfoView/SpiderInfoView.vue | 20 -- frontend/src/store/modules/site.js | 59 ++++++ frontend/src/views/site/SiteList.vue | 172 ++++++++++++++++++ 6 files changed, 299 insertions(+), 24 deletions(-) create mode 100644 crawlab/routes/sites.py create mode 100644 frontend/src/store/modules/site.js create mode 100644 frontend/src/views/site/SiteList.vue diff --git a/crawlab/app.py b/crawlab/app.py index 4eebc804..c984f140 100644 --- a/crawlab/app.py +++ b/crawlab/app.py @@ -9,6 +9,7 @@ from flask import Flask from flask_cors import CORS from flask_restful import Api # from flask_restplus import Api +from routes.sites import SiteApi from utils.log import other from constants.node import NodeStatus from db.manager import db_manager @@ -68,6 +69,9 @@ api.add_resource(StatsApi, api.add_resource(ScheduleApi, '/api/schedules', '/api/schedules/') +api.add_resource(SiteApi, + '/api/sites', + '/api/sites/') def monitor_nodes_status(celery_app): diff --git a/crawlab/routes/sites.py b/crawlab/routes/sites.py new file mode 100644 index 00000000..443cecd5 --- /dev/null +++ b/crawlab/routes/sites.py @@ -0,0 +1,64 @@ +import json + +from bson import ObjectId +from pymongo import ASCENDING + +from db.manager import db_manager +from routes.base import BaseApi +from utils import jsonify + + +class SiteApi(BaseApi): + col_name = 'sites' + + arguments = ( + ('keyword', str), + ('category', str), + ) + + def get(self, id: str = None, action: str = None): + # action by id + if action is not None: + if not hasattr(self, action): + return { + 'status': 'ok', + 'code': 400, + 'error': 'action "%s" invalid' % action + }, 400 + return getattr(self, action)(id) + + elif id is not None: + site = db_manager.get(col_name=self.col_name, id=id) + return jsonify(site) + + # list tasks + args = self.parser.parse_args() + page_size = args.get('page_size') or 10 + page_num = args.get('page_num') or 1 + filter_str = args.get('filter') + keyword = args.get('keyword') + filter_ = {} + if filter_str is not None: + filter_ = json.loads(filter_str) + if keyword is not None: + filter_['$or'] = [ + {'description': {'$regex': keyword}}, + {'name': {'$regex': keyword}} + ] + + items = db_manager.list( + col_name=self.col_name, + cond=filter_, + limit=page_size, + skip=page_size * (page_num - 1), + sort_key='rank', + sort_direction=ASCENDING + ) + + return { + 'status': 'ok', + 'total_count': db_manager.count(self.col_name, filter_), + 'page_num': page_num, + 'page_size': page_size, + 'items': jsonify(items) + } diff --git a/crawlab/routes/tasks.py b/crawlab/routes/tasks.py index 2afb0cf9..e0cdd0e7 100644 --- a/crawlab/routes/tasks.py +++ b/crawlab/routes/tasks.py @@ -36,7 +36,6 @@ class TaskApi(BaseApi): 'code': 400, 'error': 'action "%s" invalid' % action }, 400 - # other.info(f"到这了{action},{id}") return getattr(self, action)(id) elif id is not None: @@ -78,9 +77,6 @@ class TaskApi(BaseApi): sort_key='create_ts') items = [] for task in tasks: - # celery tasks - # _task = db_manager.get('tasks_celery', id=task['_id']) - # get spider _spider = db_manager.get(col_name='spiders', id=str(task['spider_id'])) diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index 6bc1a157..e72673b2 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -38,26 +38,6 @@ - - - - - - - - - - - - - - - - - - - - diff --git a/frontend/src/store/modules/site.js b/frontend/src/store/modules/site.js new file mode 100644 index 00000000..496fedb5 --- /dev/null +++ b/frontend/src/store/modules/site.js @@ -0,0 +1,59 @@ +import request from '../../api/request' + +const state = { + tableData: [], + + // filter + filter: {}, + keyword: '', + + // pagination + pageNum: 1, + pageSize: 10, + totalCount: 0 +} + +const getters = {} + +const mutations = { + SET_KEYWORD (state, value) { + state.keyword = value + }, + SET_TABLE_DATA (state, value) { + state.tableData = value + }, + SET_PAGE_NUM (state, value) { + state.pageNum = value + }, + SET_PAGE_SIZE (state, value) { + state.pageSize = value + }, + SET_TOTAL_COUNT (state, value) { + state.totalCount = value + } +} + +const actions = { + getSiteList ({ state, commit }) { + return request.get('/sites', { + page_num: state.pageNum, + page_size: state.pageSize, + keyword: state.keyword || undefined, + filter: { + category: state.filter.category || undefined + } + }) + .then(response => { + commit('SET_TABLE_DATA', response.data.items) + commit('SET_TOTAL_COUNT', response.data.total_count) + }) + } +} + +export default { + namespaced: true, + state, + getters, + mutations, + actions +} diff --git a/frontend/src/views/site/SiteList.vue b/frontend/src/views/site/SiteList.vue new file mode 100644 index 00000000..4f15c36c --- /dev/null +++ b/frontend/src/views/site/SiteList.vue @@ -0,0 +1,172 @@ + + + + +