diff --git a/db/manager.py b/db/manager.py index 115fbdfa..4d15b6e0 100644 --- a/db/manager.py +++ b/db/manager.py @@ -40,7 +40,7 @@ class DbManager(object): col = self.db[col_name] col.remove({'_id': ObjectId(id)}) - def list(self, col_name: str, cond: dict, skip: int = 0, limit: int = 10, **kwargs): + def list(self, col_name: str, cond: dict, skip: int = 0, limit: int = 100, **kwargs): col = self.db[col_name] data = [] for item in col.find(cond).skip(skip).limit(limit): diff --git a/routes/deploys.py b/routes/deploys.py index 25a5d4d6..2fd032ef 100644 --- a/routes/deploys.py +++ b/routes/deploys.py @@ -1,4 +1,6 @@ +from db.manager import db_manager from routes.base import BaseApi +from utils import jsonify class DeployApi(BaseApi): @@ -9,3 +11,31 @@ class DeployApi(BaseApi): ('node_id', str), ) + def get(self, id=None, action=None): + # action by id + if action is not None: + if not hasattr(self, action): + return { + 'status': 'ok', + 'code': 400, + 'error': 'action "%s" invalid' % action + }, 400 + return getattr(self, action)(id) + + # get one node + elif id is not None: + return jsonify(db_manager.get('deploys', id=id)) + + # get a list of items + else: + items = db_manager.list('deploys', {}) + deploys = [] + for item in items: + spider_id = item['spider_id'] + spider = db_manager.get('spiders', id=str(spider_id)) + item['spider_name'] = spider['name'] + deploys.append(item) + return jsonify({ + 'status': 'ok', + 'items': deploys + }) diff --git a/routes/nodes.py b/routes/nodes.py index b49cd856..c07532ae 100644 --- a/routes/nodes.py +++ b/routes/nodes.py @@ -1,6 +1,7 @@ import json import requests +from bson import ObjectId from config.celery import FLOWER_API_ENDPOINT from constants.node import NodeType @@ -80,5 +81,18 @@ class NodeApi(BaseApi): 'items': nodes }) - def spider(self, id=None): + def get_spiders(self, id=None): items = db_manager.list('spiders') + + def get_deploys(self, id): + items = db_manager.list('deploys', {'node_id': id}) + deploys = [] + for item in items: + spider_id = item['spider_id'] + spider = db_manager.get('spiders', id=str(spider_id)) + item['spider_name'] = spider['name'] + deploys.append(item) + return jsonify({ + 'status': 'ok', + 'items': deploys + }) diff --git a/routes/spiders.py b/routes/spiders.py index 4eeca7fc..cbe90bc0 100644 --- a/routes/spiders.py +++ b/routes/spiders.py @@ -1,5 +1,6 @@ import os import shutil +from datetime import datetime from bson import ObjectId @@ -21,6 +22,9 @@ class SpiderApi(BaseApi): ('src', str), ('type', str), ('lang', str), + + # for deploy only + ('node_id', str), ) def get(self, id=None, action=None): @@ -86,11 +90,16 @@ class SpiderApi(BaseApi): } def deploy(self, id): + args = self.parser.parse_args() + node_id = args.get('node_id') + # get spider given the id spider = db_manager.get(col_name=self.col_name, id=id) if spider is None: return + # TODO: deploy spiders to other node rather than in local machine + # get latest version latest_version = db_manager.get_latest_version(spider_id=id) @@ -122,5 +131,19 @@ class SpiderApi(BaseApi): db_manager.save('deploys', { 'spider_id': ObjectId(id), 'version': version, - 'node_id': None # TODO: deploy to corresponding node + 'node_id': node_id, + 'finish_ts': datetime.now() }) + + def get_deploys(self, id): + items = db_manager.list('deploys', {'spider_id': ObjectId(id)}) + deploys = [] + for item in items: + spider_id = item['spider_id'] + spider = db_manager.get('spiders', id=str(spider_id)) + item['spider_name'] = spider['name'] + deploys.append(item) + return jsonify({ + 'status': 'ok', + 'items': deploys + })