From 55733a7b8cfdc56b8e3a8b4a82fbe2773b633b45 Mon Sep 17 00:00:00 2001
From: Marvin Zhang <tikazyq@gmail.com>
Date: Sat, 23 Feb 2019 21:40:26 +0800
Subject: [PATCH] added pagination

---
 app.py            |  3 ++-
 routes/nodes.py   |  4 ++--
 routes/spiders.py |  4 ++--
 routes/tasks.py   | 23 +++++++++++++++++++++--
 tasks/spider.py   |  3 +--
 5 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/app.py b/app.py
index 573ea416..bfda920c 100644
--- a/app.py
+++ b/app.py
@@ -34,7 +34,8 @@ api.add_resource(DeployApi,
                  '/api/deploys/<string:id>/<string:action>')
 api.add_resource(TaskApi,
                  '/api/tasks',
-                 '/api/tasks/<string:id>'
+                 '/api/tasks/<string:id>',
+                 '/api/tasks/<string:id>/<string:action>'
                  )
 api.add_resource(FileApi,
                  '/api/files',
diff --git a/routes/nodes.py b/routes/nodes.py
index 26efa6c9..3eb84a26 100644
--- a/routes/nodes.py
+++ b/routes/nodes.py
@@ -87,7 +87,7 @@ class NodeApi(BaseApi):
         items = db_manager.list('spiders')
 
     def get_deploys(self, id):
-        items = db_manager.list('deploys', {'node_id': id})
+        items = db_manager.list('deploys', {'node_id': id}, limit=10)
         deploys = []
         for item in items:
             spider_id = item['spider_id']
@@ -100,7 +100,7 @@ class NodeApi(BaseApi):
         })
 
     def get_tasks(self, id):
-        items = db_manager.list('tasks', {'node_id': id})
+        items = db_manager.list('tasks', {'node_id': id}, limit=10)
         for item in items:
             spider_id = item['spider_id']
             spider = db_manager.get('spiders', id=str(spider_id))
diff --git a/routes/spiders.py b/routes/spiders.py
index 173edea8..e3e62ba1 100644
--- a/routes/spiders.py
+++ b/routes/spiders.py
@@ -142,7 +142,7 @@ class SpiderApi(BaseApi):
             })
 
     def get_deploys(self, id):
-        items = db_manager.list('deploys', {'spider_id': ObjectId(id)})
+        items = db_manager.list('deploys', {'spider_id': ObjectId(id)}, limit=10)
         deploys = []
         for item in items:
             spider_id = item['spider_id']
@@ -155,7 +155,7 @@ class SpiderApi(BaseApi):
         })
 
     def get_tasks(self, id):
-        items = db_manager.list('tasks', {'spider_id': ObjectId(id)})
+        items = db_manager.list('tasks', {'spider_id': ObjectId(id)}, limit=10)
         for item in items:
             spider_id = item['spider_id']
             spider = db_manager.get('spiders', id=str(spider_id))
diff --git a/routes/tasks.py b/routes/tasks.py
index 9b5af97d..8a0d686b 100644
--- a/routes/tasks.py
+++ b/routes/tasks.py
@@ -11,8 +11,18 @@ class TaskApi(BaseApi):
         ('file_path', str)
     )
 
-    def get(self, id=None):
-        if id is not None:
+    def get(self, id=None, action=None):
+        # action by id
+        if action is not None:
+            if not hasattr(self, action):
+                return {
+                           'status': 'ok',
+                           'code': 400,
+                           'error': 'action "%s" invalid' % action
+                       }, 400
+            return getattr(self, action)(id)
+
+        elif id is not None:
             task = db_manager.get('tasks', id=id)
             _task = db_manager.get('tasks_celery', id=task['_id'])
             _spider = db_manager.get('spiders', id=str(task['spider_id']))
@@ -35,3 +45,12 @@ class TaskApi(BaseApi):
             'status': 'ok',
             'items': items
         })
+
+    def get_log(self, id):
+        task = db_manager.get('tasks', id=id)
+        with open(task['log_file_path']) as f:
+            log = f.read()
+            return {
+                'status': 'ok',
+                'log': log
+            }
diff --git a/tasks/spider.py b/tasks/spider.py
index a253bdb5..edcf7ab0 100644
--- a/tasks/spider.py
+++ b/tasks/spider.py
@@ -49,8 +49,7 @@ def execute_spider(self, id: str, node_id: str):
     })
 
     # execute the command
-    p = subprocess.Popen(command,
-                         shell=True,
+    p = subprocess.Popen(command.split(' '),
                          stdout=stdout.fileno(),
                          stderr=stderr.fileno(),
                          cwd=current_working_directory,