added pagination

app.py

@@ -34,7 +34,8 @@ api.add_resource(DeployApi,
                  '/api/deploys/<string:id>/<string:action>')
 api.add_resource(TaskApi,
                  '/api/tasks',
-                 '/api/tasks/<string:id>'
+                 '/api/tasks/<string:id>',
+                 '/api/tasks/<string:id>/<string:action>'
                  )
 api.add_resource(FileApi,
                  '/api/files',

routes/nodes.py

@@ -87,7 +87,7 @@ class NodeApi(BaseApi):
         items = db_manager.list('spiders')
 
     def get_deploys(self, id):
-        items = db_manager.list('deploys', {'node_id': id})
+        items = db_manager.list('deploys', {'node_id': id}, limit=10)
         deploys = []
         for item in items:
             spider_id = item['spider_id']
@@ -100,7 +100,7 @@ class NodeApi(BaseApi):
         })
 
     def get_tasks(self, id):
-        items = db_manager.list('tasks', {'node_id': id})
+        items = db_manager.list('tasks', {'node_id': id}, limit=10)
         for item in items:
             spider_id = item['spider_id']
             spider = db_manager.get('spiders', id=str(spider_id))

routes/spiders.py

@@ -142,7 +142,7 @@ class SpiderApi(BaseApi):
             })
 
     def get_deploys(self, id):
-        items = db_manager.list('deploys', {'spider_id': ObjectId(id)})
+        items = db_manager.list('deploys', {'spider_id': ObjectId(id)}, limit=10)
         deploys = []
         for item in items:
             spider_id = item['spider_id']
@@ -155,7 +155,7 @@ class SpiderApi(BaseApi):
         })
 
     def get_tasks(self, id):
-        items = db_manager.list('tasks', {'spider_id': ObjectId(id)})
+        items = db_manager.list('tasks', {'spider_id': ObjectId(id)}, limit=10)
         for item in items:
             spider_id = item['spider_id']
             spider = db_manager.get('spiders', id=str(spider_id))

routes/tasks.py

@@ -11,8 +11,18 @@ class TaskApi(BaseApi):
         ('file_path', str)
     )
 
-    def get(self, id=None):
-        if id is not None:
+    def get(self, id=None, action=None):
+        # action by id
+        if action is not None:
+            if not hasattr(self, action):
+                return {
+                           'status': 'ok',
+                           'code': 400,
+                           'error': 'action "%s" invalid' % action
+                       }, 400
+            return getattr(self, action)(id)
+
+        elif id is not None:
             task = db_manager.get('tasks', id=id)
             _task = db_manager.get('tasks_celery', id=task['_id'])
             _spider = db_manager.get('spiders', id=str(task['spider_id']))
@@ -35,3 +45,12 @@ class TaskApi(BaseApi):
             'status': 'ok',
             'items': items
         })
+
+    def get_log(self, id):
+        task = db_manager.get('tasks', id=id)
+        with open(task['log_file_path']) as f:
+            log = f.read()
+            return {
+                'status': 'ok',
+                'log': log
+            }

tasks/spider.py

@@ -49,8 +49,7 @@ def execute_spider(self, id: str, node_id: str):
     })
 
     # execute the command
-    p = subprocess.Popen(command,
-                         shell=True,
+    p = subprocess.Popen(command.split(' '),
                          stdout=stdout.fileno(),
                          stderr=stderr.fileno(),
                          cwd=current_working_directory,