Merge pull request #62 from tikazyq/develop

Develop

Dockerfile

@@ -30,13 +30,13 @@ ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules
 ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
 
 # install frontend
-RUN npm install -g yarn --registry=https://registry.npm.taobao.org \
+RUN npm install -g yarn \
 	&& cd /opt/crawlab/frontend \
-	&& yarn install --registry=https://registry.npm.taobao.org
+	&& yarn install
 
 # install backend
-RUN pip install -U setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple \
-	&& pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
+RUN pip install -U setuptools \
+	&& pip install -r /opt/crawlab/crawlab/requirements.txt
 
 # start backend
 EXPOSE 8080

crawlab/config/config.py

@@ -14,11 +14,21 @@ PROJECT_LOGS_FOLDER = '/var/log/crawlab'
 # 打包临时文件夹
 PROJECT_TMP_FOLDER = '/tmp'
 
+# MongoDB 变量
+MONGO_HOST = '127.0.0.1'
+MONGO_PORT = 27017
+MONGO_USERNAME = None
+MONGO_PASSWORD = None
+MONGO_DB = 'crawlab_test'
+
 # Celery中间者URL
 BROKER_URL = 'redis://127.0.0.1:6379/0'
 
 # Celery后台URL
-CELERY_RESULT_BACKEND = 'mongodb://127.0.0.1:27017/'
+if MONGO_USERNAME is not None:
+    CELERY_RESULT_BACKEND = f'mongodb://{MONGO_USERNAME}:{MONGO_PASSWORD}@{MONGO_HOST}:{MONGO_PORT}/'
+else:
+    CELERY_RESULT_BACKEND = f'mongodb://{MONGO_HOST}:{MONGO_PORT}/'
 
 # Celery MongoDB设置
 CELERY_MONGODB_BACKEND_SETTINGS = {
@@ -35,11 +45,6 @@ CELERY_ENABLE_UTC = True
 # flower variables
 FLOWER_API_ENDPOINT = 'http://localhost:5555/api'
 
-# MongoDB 变量
-MONGO_HOST = '127.0.0.1'
-MONGO_PORT = 27017
-MONGO_DB = 'crawlab_test'
-
 # Flask 变量
 DEBUG = False
 FLASK_HOST = '0.0.0.0'

crawlab/db/manager.py

@@ -1,7 +1,7 @@
 from bson import ObjectId
 from mongoengine import connect
 from pymongo import MongoClient, DESCENDING
-from config import MONGO_HOST, MONGO_PORT, MONGO_DB
+from config import MONGO_HOST, MONGO_PORT, MONGO_DB, MONGO_USERNAME, MONGO_PASSWORD
 from utils import is_object_id
 
 connect(db=MONGO_DB, host=MONGO_HOST, port=MONGO_PORT)
@@ -13,7 +13,11 @@ class DbManager(object):
     """
 
     def __init__(self):
-        self.mongo = MongoClient(host=MONGO_HOST, port=MONGO_PORT, connect=False)
+        self.mongo = MongoClient(host=MONGO_HOST,
+                                 port=MONGO_PORT,
+                                 username=MONGO_USERNAME,
+                                 password=MONGO_PASSWORD,
+                                 connect=False)
         self.db = self.mongo[MONGO_DB]
 
     def save(self, col_name: str, item: dict, **kwargs) -> None:

crawlab/spiders/spiders/db.py

@@ -2,11 +2,15 @@ import os
 
 from pymongo import MongoClient
 
-MONGO_HOST = os.environ.get('MONGO_HOST')
-MONGO_PORT = int(os.environ.get('MONGO_PORT'))
-MONGO_DB = os.environ.get('MONGO_DB')
+MONGO_HOST = os.environ.get('MONGO_HOST') or 'localhost'
+MONGO_PORT = int(os.environ.get('MONGO_PORT')) or 27017
+MONGO_USERNAME = os.environ.get('MONGO_USERNAME')
+MONGO_PASSWORD = os.environ.get('MONGO_PASSWORD')
+MONGO_DB = os.environ.get('MONGO_DB') or 'crawlab_test'
 mongo = MongoClient(host=MONGO_HOST,
-                    port=MONGO_PORT)
+                    port=MONGO_PORT,
+                    username=MONGO_USERNAME,
+                    password=MONGO_PASSWORD)
 db = mongo[MONGO_DB]
 task_id = os.environ.get('CRAWLAB_TASK_ID')
 col_name = os.environ.get('CRAWLAB_COLLECTION')

crawlab/tasks/spider.py

@@ -6,7 +6,8 @@ from time import sleep
 from bson import ObjectId
 from pymongo import ASCENDING, DESCENDING
 
-from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_LOGS_FOLDER, MONGO_HOST, MONGO_PORT, MONGO_DB
+from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_LOGS_FOLDER, MONGO_HOST, MONGO_PORT, MONGO_DB, MONGO_USERNAME, \
+    MONGO_PASSWORD
 from constants.task import TaskStatus
 from db.manager import db_manager
 from .celery import celery_app
@@ -212,6 +213,8 @@ def execute_config_spider(self, id: str, params: str = None):
     env['MONGO_HOST'] = MONGO_HOST
     env['MONGO_PORT'] = str(MONGO_PORT)
     env['MONGO_DB'] = MONGO_DB
+    env['MONGO_USERNAME'] = MONGO_USERNAME
+    env['MONGO_PASSWORD'] = MONGO_PASSWORD
 
     cmd_arr = [
         sys.executable,

docker-compose.yml

@@ -1,18 +1,24 @@
-version: '3.3' # 表示该 Docker-Compose 文件使用的是 Version 2 file
+version: '3.3'
 services:
-  app:  # 指定服务名称
-    build: .  # 指定 Dockerfile 所在路径
-    ports:    # 指定端口映射
-      - "5001:5000"
-  task:
+  master: 
     image: crawlab:latest
-  db:
-    image: mongo
+    restart: always
+    volumns:
+      - /home/yeqing/config.py:/opt/crawlab/crawlab/config/config.py # 后端配置文件
+      - /home/yeqing/.env.production:/opt/crawlab/frontend/.env.production # 前端配置文件
+    ports:    
+      - "8080:8080" # nginx
+      - "8000:8000" # app
+    depends_on:
+      - mongo
+      - redis
+  mongo:
+    image: mongo:latest
     restart: always
     ports:
       - "27017:27017"
   redis:
-    image: redis
+    image: redis:latest
     restart: always
     ports:
       - "6379:6379"