modify config setting

crawlab/config/__init__.py

@@ -0,0 +1,10 @@
+# encoding: utf-8
+
+import os
+
+run_env = os.environ.get("RUNENV", "local")
+
+if run_env == "local":  # 加载本地配置
+    from config.config_local import *
+else:
+    from config.config import *

crawlab/config/config_local.py

@@ -0,0 +1,38 @@
+# encoding: utf-8
+
+import os
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+PROJECT_SOURCE_FILE_FOLDER = os.path.join(BASE_DIR, "spiders")
+
+# 配置python虚拟环境的路径
+PYTHON_ENV_PATH = '/Users/chennan/Desktop/2019/env/bin/python'
+
+# 爬虫部署路径
+PROJECT_DEPLOY_FILE_FOLDER = os.path.join(BASE_DIR, 'deployfile')
+
+PROJECT_LOGS_FOLDER = os.path.join(BASE_DIR, 'deployfile/logs')
+PROJECT_TMP_FOLDER = '/tmp'
+
+# celery variables
+BROKER_URL = 'redis://127.0.0.1:56379/0'
+CELERY_RESULT_BACKEND = 'mongodb://127.0.0.1:57017/'
+CELERY_MONGODB_BACKEND_SETTINGS = {
+    'database': 'crawlab_test',
+    'taskmeta_collection': 'tasks_celery',
+}
+CELERY_TIMEZONE = 'Asia/Shanghai'
+CELERY_ENABLE_UTC = True
+
+# flower variables
+FLOWER_API_ENDPOINT = 'http://localhost:5555/api'
+
+# database variables
+MONGO_HOST = '127.0.0.1'
+MONGO_PORT = 57017
+MONGO_DB = 'crawlab_test'
+
+# flask variables
+DEBUG = True
+FLASK_HOST = '127.0.0.1'
+FLASK_PORT = 8000

crawlab/utils/node.py

@@ -8,16 +8,27 @@ from db.manager import db_manager
 
 
 def check_nodes_status():
+    """
+    Update node status from Flower.
+    """
     res = requests.get('%s/workers?status=1' % FLOWER_API_ENDPOINT)
     return json.loads(res.content.decode('utf-8'))
 
 
 def update_nodes_status(refresh=False):
+    """
+    Update all nodes status
+    :param refresh:
+    """
     online_node_ids = []
     url = '%s/workers?status=1' % FLOWER_API_ENDPOINT
     if refresh:
         url += '&refresh=1'
+
     res = requests.get(url)
+    if res.status_code != 200:
+        return online_node_ids
+
     for k, v in json.loads(res.content.decode('utf-8')).items():
         node_name = k
         node_status = NodeStatus.ONLINE if v else NodeStatus.OFFLINE
@@ -26,9 +37,10 @@ def update_nodes_status(refresh=False):
 
         # new node
         if node is None:
-            node = {'_id': node_name, 'name': node_name, 'status': node_status}
+            node = {'_id': node_name, 'name': node_name, 'status': node_status, 'ip': 'localhost', 'port': '8000'}
             db_manager.save('nodes', node)
 
+        # existing node
         else:
             node['status'] = node_status
             db_manager.save('nodes', node)