diff --git a/crawlab/routes/base.py b/crawlab/routes/base.py index 0df9d9b5..da2a25df 100644 --- a/crawlab/routes/base.py +++ b/crawlab/routes/base.py @@ -4,7 +4,7 @@ from db.manager import db_manager from utils import jsonify DEFAULT_ARGS = [ - 'page', + 'page_num', 'page_size', 'filter' ] @@ -17,7 +17,7 @@ class BaseApi(Resource): def __init__(self): super(BaseApi).__init__() - self.parser.add_argument('page', type=int) + self.parser.add_argument('page_num', type=int) self.parser.add_argument('page_size', type=int) self.parser.add_argument('filter', type=dict) @@ -47,7 +47,7 @@ class BaseApi(Resource): # page number page = 1 - if args.get('page') is not None: + if args.get('page_num') is not None: page = args.page # page = int(args.page) @@ -73,7 +73,7 @@ class BaseApi(Resource): return jsonify({ 'status': 'ok', 'total_count': total_count, - 'page': page, + 'page_num': page, 'page_size': page_size, 'items': items }) diff --git a/crawlab/routes/tasks.py b/crawlab/routes/tasks.py index 916fc4cc..7d997187 100644 --- a/crawlab/routes/tasks.py +++ b/crawlab/routes/tasks.py @@ -36,8 +36,6 @@ class TaskApi(BaseApi): if _task: if not task.get('status'): task['status'] = _task['status'] - # else: - # task['status'] = TaskStatus.UNAVAILABLE task['result'] = _task['result'] task['spider_name'] = _spider['name'] try: @@ -47,7 +45,11 @@ class TaskApi(BaseApi): task['log'] = '' return jsonify(task) - tasks = db_manager.list('tasks', {}, limit=1000, sort_key='finish_ts') + # list tasks + args = self.parser.parse_args() + page_size = args.get('page_size') or 10 + page_num = args.get('page_num') or 1 + tasks = db_manager.list('tasks', {}, limit=page_size, skip=page_size * (page_num - 1), sort_key='finish_ts') items = [] for task in tasks: _task = db_manager.get('tasks_celery', id=task['_id']) @@ -60,6 +62,9 @@ class TaskApi(BaseApi): items.append(task) return { 'status': 'ok', + 'total_count': db_manager.count('tasks', {}), + 'page_num': page_num, + 'page_size': page_size, 'items': jsonify(items) } diff --git a/frontend/src/components/TableView/GeneralTableView.vue b/frontend/src/components/TableView/GeneralTableView.vue index 94d7f758..42f13a57 100644 --- a/frontend/src/components/TableView/GeneralTableView.vue +++ b/frontend/src/components/TableView/GeneralTableView.vue @@ -49,19 +49,7 @@ export default { computed: { filteredData () { return this.data - .map(d => { - for (let k in d) { - if (d.hasOwnProperty(k)) { - if (d[k] === undefined || d[k] === null) continue - if (typeof d[k] === 'object') { - if (d[k]) { - d[k] = d[k] - } - } - } - } - return d - }) + .map(d => d) .filter((d, index) => { // pagination const { pageNum, pageSize } = this.pagination diff --git a/frontend/src/store/modules/task.js b/frontend/src/store/modules/task.js index 992012e4..742f1c66 100644 --- a/frontend/src/store/modules/task.js +++ b/frontend/src/store/modules/task.js @@ -4,10 +4,14 @@ import request from '../../api/request' const state = { // TaskList taskList: [], + taskListTotalCount: 0, taskForm: {}, taskLog: '', taskResultsData: [], - taskResultsColumns: [] + taskResultsColumns: [], + // pagination + pageNum: 0, + pageSize: 10 } const getters = {} @@ -27,6 +31,15 @@ const mutations = { }, SET_TASK_RESULTS_COLUMNS (state, value) { state.taskResultsColumns = value + }, + SET_PAGE_NUM (state, value) { + state.pageNum = value + }, + SET_PAGE_SIZE (state, value) { + state.pageSize = value + }, + SET_TASK_LIST_TOTAL_COUNT (state, value) { + state.taskListTotalCount = value } } @@ -46,9 +59,13 @@ const actions = { }) }, getTaskList ({ state, commit }) { - return request.get('/tasks', {}) + return request.get('/tasks', { + page_num: state.pageNum, + page_size: state.pageSize + }) .then(response => { commit('SET_TASK_LIST', response.data.items) + commit('SET_TASK_LIST_TOTAL_COUNT', response.data.total_count) }) }, deleteTask ({ state, dispatch }, id) { diff --git a/frontend/src/views/task/TaskList.vue b/frontend/src/views/task/TaskList.vue index 2cfdd344..dbaf7300 100644 --- a/frontend/src/views/task/TaskList.vue +++ b/frontend/src/views/task/TaskList.vue @@ -75,14 +75,16 @@ @@ -92,16 +94,11 @@ import { mapState } from 'vuex' -import dayjs from 'dayjs' export default { name: 'TaskList', data () { return { - pagination: { - pageNum: 0, - pageSize: 10 - }, isEditMode: false, dialogVisible: false, filter: { @@ -120,21 +117,28 @@ export default { computed: { ...mapState('task', [ 'taskList', + 'taskListTotalCount', 'taskForm' ]), + pageNum: { + get () { + return this.$store.state.task.pageNum + }, + set (value) { + this.$store.commit('task/SET_PAGE_NUM', value) + } + }, + pageSize: { + get () { + return this.$store.state.task.pageSize + }, + set (value) { + this.$store.commit('task/SET_PAGE_SIZE', value) + } + }, filteredTableData () { return this.taskList - .map(d => { - if (d.create_ts) d.create_ts = dayjs(d.create_ts.$date).format('YYYY-MM-DD HH:mm:ss') - if (d.finish_ts) d.finish_ts = dayjs(d.finish_ts.$date).format('YYYY-MM-DD HH:mm:ss') - - try { - d.spider_id = d.spider_id - } catch (e) { - if (d.spider_id) d.spider_id = d.spider_id.toString() - } - return d - }) + .map(d => d) .sort((a, b) => a.create_ts < b.create_ts ? 1 : -1) .filter(d => { // keyword @@ -147,11 +151,12 @@ export default { } return false }) - .filter((d, index) => { - // pagination - const { pageNum, pageSize } = this.pagination - return (pageSize * (pageNum - 1) <= index) && (index < pageSize * pageNum) - }) + // .filter((d, index) => { + // // pagination + // const pageNum = this.pageNum + // const pageSize = this.pageSize + // return (pageSize * (pageNum - 1) <= index) && (index < pageSize * pageNum) + // }) } }, methods: { @@ -186,7 +191,9 @@ export default { this.$router.push(`/nodes/${row.node_id}`) }, onPageChange () { - this.$store.dispatch('task/getTaskList') + setTimeout(() => { + this.$store.dispatch('task/getTaskList') + }, 0) } }, created () {