added pagination for tasks

crawlab/routes/base.py

@@ -4,7 +4,7 @@ from db.manager import db_manager
 from utils import jsonify
 
 DEFAULT_ARGS = [
-    'page',
+    'page_num',
     'page_size',
     'filter'
 ]
@@ -17,7 +17,7 @@ class BaseApi(Resource):
 
     def __init__(self):
         super(BaseApi).__init__()
-        self.parser.add_argument('page', type=int)
+        self.parser.add_argument('page_num', type=int)
         self.parser.add_argument('page_size', type=int)
         self.parser.add_argument('filter', type=dict)
 
@@ -47,7 +47,7 @@ class BaseApi(Resource):
 
             # page number
             page = 1
-            if args.get('page') is not None:
+            if args.get('page_num') is not None:
                 page = args.page
                 # page = int(args.page)
 
@@ -73,7 +73,7 @@ class BaseApi(Resource):
             return jsonify({
                 'status': 'ok',
                 'total_count': total_count,
-                'page': page,
+                'page_num': page,
                 'page_size': page_size,
                 'items': items
             })

crawlab/routes/tasks.py

@@ -36,8 +36,6 @@ class TaskApi(BaseApi):
             if _task:
                 if not task.get('status'):
                     task['status'] = _task['status']
-            # else:
-            #     task['status'] = TaskStatus.UNAVAILABLE
             task['result'] = _task['result']
             task['spider_name'] = _spider['name']
             try:
@@ -47,7 +45,11 @@ class TaskApi(BaseApi):
                 task['log'] = ''
             return jsonify(task)
 
-        tasks = db_manager.list('tasks', {}, limit=1000, sort_key='finish_ts')
+        # list tasks
+        args = self.parser.parse_args()
+        page_size = args.get('page_size') or 10
+        page_num = args.get('page_num') or 1
+        tasks = db_manager.list('tasks', {}, limit=page_size, skip=page_size * (page_num - 1), sort_key='finish_ts')
         items = []
         for task in tasks:
             _task = db_manager.get('tasks_celery', id=task['_id'])
@@ -60,6 +62,9 @@ class TaskApi(BaseApi):
             items.append(task)
         return {
             'status': 'ok',
+            'total_count': db_manager.count('tasks', {}),
+            'page_num': page_num,
+            'page_size': page_size,
             'items': jsonify(items)
         }
 

frontend/src/components/TableView/GeneralTableView.vue

@@ -49,19 +49,7 @@ export default {
   computed: {
     filteredData () {
       return this.data
-        .map(d => {
-          for (let k in d) {
-            if (d.hasOwnProperty(k)) {
-              if (d[k] === undefined || d[k] === null) continue
-              if (typeof d[k] === 'object') {
-                if (d[k]) {
-                  d[k] = d[k]
-                }
-              }
-            }
-          }
-          return d
-        })
+        .map(d => d)
         .filter((d, index) => {
           // pagination
           const { pageNum, pageSize } = this.pagination

frontend/src/store/modules/task.js

@@ -4,10 +4,14 @@ import request from '../../api/request'
 const state = {
   // TaskList
   taskList: [],
+  taskListTotalCount: 0,
   taskForm: {},
   taskLog: '',
   taskResultsData: [],
-  taskResultsColumns: []
+  taskResultsColumns: [],
+  // pagination
+  pageNum: 0,
+  pageSize: 10
 }
 
 const getters = {}
@@ -27,6 +31,15 @@ const mutations = {
   },
   SET_TASK_RESULTS_COLUMNS (state, value) {
     state.taskResultsColumns = value
+  },
+  SET_PAGE_NUM (state, value) {
+    state.pageNum = value
+  },
+  SET_PAGE_SIZE (state, value) {
+    state.pageSize = value
+  },
+  SET_TASK_LIST_TOTAL_COUNT (state, value) {
+    state.taskListTotalCount = value
   }
 }
 
@@ -46,9 +59,13 @@ const actions = {
       })
   },
   getTaskList ({ state, commit }) {
-    return request.get('/tasks', {})
+    return request.get('/tasks', {
+      page_num: state.pageNum,
+      page_size: state.pageSize
+    })
       .then(response => {
         commit('SET_TASK_LIST', response.data.items)
+        commit('SET_TASK_LIST_TOTAL_COUNT', response.data.total_count)
       })
   },
   deleteTask ({ state, dispatch }, id) {

frontend/src/views/task/TaskList.vue

@@ -75,14 +75,16 @@
       </el-table-column>
     </el-table>
     <div class="pagination">
+      pageNum: {{pageNum}}
+      pageSize: {{pageSize}}
       <el-pagination
         @current-change="onPageChange"
         @size-change="onPageChange"
-        :current-page.sync="pagination.pageNum"
+        :current-page.sync="pageNum"
         :page-sizes="[10, 20, 50, 100]"
-        :page-size.sync="pagination.pageSize"
+        :page-size.sync="pageSize"
         layout="sizes, prev, pager, next"
-        :total="taskList.length">
+        :total="taskListTotalCount">
       </el-pagination>
     </div>
   </div>
@@ -92,16 +94,11 @@
 import {
   mapState
 } from 'vuex'
-import dayjs from 'dayjs'
 
 export default {
   name: 'TaskList',
   data () {
     return {
-      pagination: {
-        pageNum: 0,
-        pageSize: 10
-      },
       isEditMode: false,
       dialogVisible: false,
       filter: {
@@ -120,21 +117,28 @@ export default {
   computed: {
     ...mapState('task', [
       'taskList',
+      'taskListTotalCount',
       'taskForm'
     ]),
+    pageNum: {
+      get () {
+        return this.$store.state.task.pageNum
+      },
+      set (value) {
+        this.$store.commit('task/SET_PAGE_NUM', value)
+      }
+    },
+    pageSize: {
+      get () {
+        return this.$store.state.task.pageSize
+      },
+      set (value) {
+        this.$store.commit('task/SET_PAGE_SIZE', value)
+      }
+    },
     filteredTableData () {
       return this.taskList
-        .map(d => {
-          if (d.create_ts) d.create_ts = dayjs(d.create_ts.$date).format('YYYY-MM-DD HH:mm:ss')
-          if (d.finish_ts) d.finish_ts = dayjs(d.finish_ts.$date).format('YYYY-MM-DD HH:mm:ss')
-
-          try {
-            d.spider_id = d.spider_id
-          } catch (e) {
-            if (d.spider_id) d.spider_id = d.spider_id.toString()
-          }
-          return d
-        })
+        .map(d => d)
         .sort((a, b) => a.create_ts < b.create_ts ? 1 : -1)
         .filter(d => {
           // keyword
@@ -147,11 +151,12 @@ export default {
           }
           return false
         })
-        .filter((d, index) => {
-          // pagination
-          const { pageNum, pageSize } = this.pagination
-          return (pageSize * (pageNum - 1) <= index) && (index < pageSize * pageNum)
-        })
+        // .filter((d, index) => {
+        //   // pagination
+        //   const pageNum = this.pageNum
+        //   const pageSize = this.pageSize
+        //   return (pageSize * (pageNum - 1) <= index) && (index < pageSize * pageNum)
+        // })
     }
   },
   methods: {
@@ -186,7 +191,9 @@ export default {
       this.$router.push(`/nodes/${row.node_id}`)
     },
     onPageChange () {
-      this.$store.dispatch('task/getTaskList')
+      setTimeout(() => {
+        this.$store.dispatch('task/getTaskList')
+      }, 0)
     }
   },
   created () {