mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-21 17:21:09 +01:00
117 lines
3.5 KiB
Python
117 lines
3.5 KiB
Python
import json
|
|
|
|
import requests
|
|
from bson import ObjectId
|
|
|
|
from config import FLOWER_API_ENDPOINT
|
|
from constants.node import NodeType
|
|
from db.manager import db_manager
|
|
from routes.base import BaseApi
|
|
from utils import jsonify
|
|
from utils.node import check_nodes_status
|
|
|
|
|
|
class NodeApi(BaseApi):
|
|
col_name = 'nodes'
|
|
|
|
arguments = (
|
|
('name', str),
|
|
('description', str),
|
|
('ip', str),
|
|
('port', str),
|
|
)
|
|
|
|
def get(self, id=None, action=None):
|
|
# action by id
|
|
if action is not None:
|
|
if not hasattr(self, action):
|
|
return {
|
|
'status': 'ok',
|
|
'code': 400,
|
|
'error': 'action "%s" invalid' % action
|
|
}, 400
|
|
return getattr(self, action)(id)
|
|
|
|
# get one node
|
|
elif id is not None:
|
|
return db_manager.get('nodes', id=id)
|
|
|
|
# TODO: use query "?status=1" to get status of nodes
|
|
|
|
# get status for each node
|
|
status_data = check_nodes_status()
|
|
|
|
# get a list of items
|
|
res = requests.get('%s/workers' % FLOWER_API_ENDPOINT)
|
|
online_node_ids = []
|
|
for k, v in json.loads(res.content.decode('utf-8')).items():
|
|
node_name = k
|
|
node_celery = v
|
|
node = db_manager.get('nodes', id=node_name)
|
|
|
|
# new node
|
|
if node is None:
|
|
node = {}
|
|
for _k, _v in node_celery.items():
|
|
node[_k] = _v
|
|
node['_id'] = node_name
|
|
node['name'] = node_name
|
|
db_manager.save('nodes', node)
|
|
|
|
# existing node
|
|
else:
|
|
for _k, _v in v.items():
|
|
node[_k] = _v
|
|
node['name'] = node_name
|
|
db_manager.save('nodes', node)
|
|
|
|
online_node_ids.append(node_name)
|
|
|
|
# iterate db nodes to update status
|
|
nodes = []
|
|
items = db_manager.list('nodes', {})
|
|
for item in items:
|
|
node_status = status_data.get(item['name'])
|
|
if item['_id'] in online_node_ids:
|
|
item['status'] = NodeType.ONLINE if node_status else NodeType.OFFLINE
|
|
else:
|
|
item['status'] = NodeType.OFFLINE
|
|
db_manager.update_one('nodes', item['_id'], {
|
|
'status': item['status']
|
|
})
|
|
nodes.append(item)
|
|
|
|
return jsonify({
|
|
'status': 'ok',
|
|
'items': nodes
|
|
})
|
|
|
|
def get_spiders(self, id=None):
|
|
items = db_manager.list('spiders')
|
|
|
|
def get_deploys(self, id):
|
|
items = db_manager.list('deploys', {'node_id': id}, limit=10, sort_key='finish_ts')
|
|
deploys = []
|
|
for item in items:
|
|
spider_id = item['spider_id']
|
|
spider = db_manager.get('spiders', id=str(spider_id))
|
|
item['spider_name'] = spider['name']
|
|
deploys.append(item)
|
|
return jsonify({
|
|
'status': 'ok',
|
|
'items': deploys
|
|
})
|
|
|
|
def get_tasks(self, id):
|
|
items = db_manager.list('tasks', {'node_id': id}, limit=10, sort_key='create_ts')
|
|
for item in items:
|
|
spider_id = item['spider_id']
|
|
spider = db_manager.get('spiders', id=str(spider_id))
|
|
item['spider_name'] = spider['name']
|
|
task = db_manager.get('tasks_celery', id=item['_id'])
|
|
item['status'] = task['status']
|
|
return jsonify({
|
|
'status': 'ok',
|
|
'items': items
|
|
})
|