updated spider/node overviews

db/manager.py

@@ -40,7 +40,7 @@ class DbManager(object):
         col = self.db[col_name]
         col.remove({'_id': ObjectId(id)})
 
-    def list(self, col_name: str, cond: dict, skip: int = 0, limit: int = 10, **kwargs):
+    def list(self, col_name: str, cond: dict, skip: int = 0, limit: int = 100, **kwargs):
         col = self.db[col_name]
         data = []
         for item in col.find(cond).skip(skip).limit(limit):

routes/deploys.py

@@ -1,4 +1,6 @@
+from db.manager import db_manager
 from routes.base import BaseApi
+from utils import jsonify
 
 
 class DeployApi(BaseApi):
@@ -9,3 +11,31 @@ class DeployApi(BaseApi):
         ('node_id', str),
     )
 
+    def get(self, id=None, action=None):
+        # action by id
+        if action is not None:
+            if not hasattr(self, action):
+                return {
+                           'status': 'ok',
+                           'code': 400,
+                           'error': 'action "%s" invalid' % action
+                       }, 400
+            return getattr(self, action)(id)
+
+        # get one node
+        elif id is not None:
+            return jsonify(db_manager.get('deploys', id=id))
+
+        # get a list of items
+        else:
+            items = db_manager.list('deploys', {})
+            deploys = []
+            for item in items:
+                spider_id = item['spider_id']
+                spider = db_manager.get('spiders', id=str(spider_id))
+                item['spider_name'] = spider['name']
+                deploys.append(item)
+            return jsonify({
+                'status': 'ok',
+                'items': deploys
+            })

routes/nodes.py

@@ -1,6 +1,7 @@
 import json
 
 import requests
+from bson import ObjectId
 
 from config.celery import FLOWER_API_ENDPOINT
 from constants.node import NodeType
@@ -80,5 +81,18 @@ class NodeApi(BaseApi):
             'items': nodes
         })
 
-    def spider(self, id=None):
+    def get_spiders(self, id=None):
         items = db_manager.list('spiders')
+
+    def get_deploys(self, id):
+        items = db_manager.list('deploys', {'node_id': id})
+        deploys = []
+        for item in items:
+            spider_id = item['spider_id']
+            spider = db_manager.get('spiders', id=str(spider_id))
+            item['spider_name'] = spider['name']
+            deploys.append(item)
+        return jsonify({
+            'status': 'ok',
+            'items': deploys
+        })

routes/spiders.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+from datetime import datetime
 
 from bson import ObjectId
 
@@ -21,6 +22,9 @@ class SpiderApi(BaseApi):
         ('src', str),
         ('type', str),
         ('lang', str),
+
+        # for deploy only
+        ('node_id', str),
     )
 
     def get(self, id=None, action=None):
@@ -86,11 +90,16 @@ class SpiderApi(BaseApi):
         }
 
     def deploy(self, id):
+        args = self.parser.parse_args()
+        node_id = args.get('node_id')
+
         # get spider given the id
         spider = db_manager.get(col_name=self.col_name, id=id)
         if spider is None:
             return
 
+        # TODO: deploy spiders to other node rather than in local machine
+
         # get latest version
         latest_version = db_manager.get_latest_version(spider_id=id)
 
@@ -122,5 +131,19 @@ class SpiderApi(BaseApi):
             db_manager.save('deploys', {
                 'spider_id': ObjectId(id),
                 'version': version,
-                'node_id': None  # TODO: deploy to corresponding node
+                'node_id': node_id,
+                'finish_ts': datetime.now()
             })
+
+    def get_deploys(self, id):
+        items = db_manager.list('deploys', {'spider_id': ObjectId(id)})
+        deploys = []
+        for item in items:
+            spider_id = item['spider_id']
+            spider = db_manager.get('spiders', id=str(spider_id))
+            item['spider_name'] = spider['name']
+            deploys.append(item)
+        return jsonify({
+            'status': 'ok',
+            'items': deploys
+        })