diff --git a/Dockerfile b/Dockerfile index 6fdb5142..5622fb0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,13 +30,13 @@ ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH # install frontend -RUN npm install -g yarn --registry=https://registry.npm.taobao.org \ +RUN npm install -g yarn \ && cd /opt/crawlab/frontend \ - && yarn install --registry=https://registry.npm.taobao.org + && yarn install # install backend -RUN pip install -U setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple \ - && pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple +RUN pip install -U setuptools \ + && pip install -r /opt/crawlab/crawlab/requirements.txt # start backend EXPOSE 8080 diff --git a/crawlab/config/config.py b/crawlab/config/config.py index da185c57..14380b73 100644 --- a/crawlab/config/config.py +++ b/crawlab/config/config.py @@ -14,11 +14,21 @@ PROJECT_LOGS_FOLDER = '/var/log/crawlab' # 打包临时文件夹 PROJECT_TMP_FOLDER = '/tmp' +# MongoDB 变量 +MONGO_HOST = '127.0.0.1' +MONGO_PORT = 27017 +MONGO_USERNAME = None +MONGO_PASSWORD = None +MONGO_DB = 'crawlab_test' + # Celery中间者URL BROKER_URL = 'redis://127.0.0.1:6379/0' # Celery后台URL -CELERY_RESULT_BACKEND = 'mongodb://127.0.0.1:27017/' +if MONGO_USERNAME is not None: + CELERY_RESULT_BACKEND = f'mongodb://{MONGO_USERNAME}:{MONGO_PASSWORD}@{MONGO_HOST}:{MONGO_PORT}/' +else: + CELERY_RESULT_BACKEND = f'mongodb://{MONGO_HOST}:{MONGO_PORT}/' # Celery MongoDB设置 CELERY_MONGODB_BACKEND_SETTINGS = { @@ -35,11 +45,6 @@ CELERY_ENABLE_UTC = True # flower variables FLOWER_API_ENDPOINT = 'http://localhost:5555/api' -# MongoDB 变量 -MONGO_HOST = '127.0.0.1' -MONGO_PORT = 27017 -MONGO_DB = 'crawlab_test' - # Flask 变量 DEBUG = False FLASK_HOST = '0.0.0.0' diff --git a/crawlab/db/manager.py b/crawlab/db/manager.py index e80aada6..17d6b1ae 100644 --- a/crawlab/db/manager.py +++ b/crawlab/db/manager.py @@ -1,7 +1,7 @@ from bson import ObjectId from mongoengine import connect from pymongo import MongoClient, DESCENDING -from config import MONGO_HOST, MONGO_PORT, MONGO_DB +from config import MONGO_HOST, MONGO_PORT, MONGO_DB, MONGO_USERNAME, MONGO_PASSWORD from utils import is_object_id connect(db=MONGO_DB, host=MONGO_HOST, port=MONGO_PORT) @@ -13,7 +13,11 @@ class DbManager(object): """ def __init__(self): - self.mongo = MongoClient(host=MONGO_HOST, port=MONGO_PORT, connect=False) + self.mongo = MongoClient(host=MONGO_HOST, + port=MONGO_PORT, + username=MONGO_USERNAME, + password=MONGO_PASSWORD, + connect=False) self.db = self.mongo[MONGO_DB] def save(self, col_name: str, item: dict, **kwargs) -> None: diff --git a/crawlab/spiders/spiders/db.py b/crawlab/spiders/spiders/db.py index 18925f8d..0521d784 100644 --- a/crawlab/spiders/spiders/db.py +++ b/crawlab/spiders/spiders/db.py @@ -2,11 +2,15 @@ import os from pymongo import MongoClient -MONGO_HOST = os.environ.get('MONGO_HOST') -MONGO_PORT = int(os.environ.get('MONGO_PORT')) -MONGO_DB = os.environ.get('MONGO_DB') +MONGO_HOST = os.environ.get('MONGO_HOST') or 'localhost' +MONGO_PORT = int(os.environ.get('MONGO_PORT')) or 27017 +MONGO_USERNAME = os.environ.get('MONGO_USERNAME') +MONGO_PASSWORD = os.environ.get('MONGO_PASSWORD') +MONGO_DB = os.environ.get('MONGO_DB') or 'crawlab_test' mongo = MongoClient(host=MONGO_HOST, - port=MONGO_PORT) + port=MONGO_PORT, + username=MONGO_USERNAME, + password=MONGO_PASSWORD) db = mongo[MONGO_DB] task_id = os.environ.get('CRAWLAB_TASK_ID') col_name = os.environ.get('CRAWLAB_COLLECTION') diff --git a/crawlab/tasks/spider.py b/crawlab/tasks/spider.py index a568dc0f..e03bab66 100644 --- a/crawlab/tasks/spider.py +++ b/crawlab/tasks/spider.py @@ -6,7 +6,8 @@ from time import sleep from bson import ObjectId from pymongo import ASCENDING, DESCENDING -from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_LOGS_FOLDER, MONGO_HOST, MONGO_PORT, MONGO_DB +from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_LOGS_FOLDER, MONGO_HOST, MONGO_PORT, MONGO_DB, MONGO_USERNAME, \ + MONGO_PASSWORD from constants.task import TaskStatus from db.manager import db_manager from .celery import celery_app @@ -212,6 +213,8 @@ def execute_config_spider(self, id: str, params: str = None): env['MONGO_HOST'] = MONGO_HOST env['MONGO_PORT'] = str(MONGO_PORT) env['MONGO_DB'] = MONGO_DB + env['MONGO_USERNAME'] = MONGO_USERNAME + env['MONGO_PASSWORD'] = MONGO_PASSWORD cmd_arr = [ sys.executable, diff --git a/docker-compose.yml b/docker-compose.yml index 088fdbb8..dcf85bdb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,18 +1,24 @@ -version: '3.3' # 表示该 Docker-Compose 文件使用的是 Version 2 file +version: '3.3' services: - app: # 指定服务名称 - build: . # 指定 Dockerfile 所在路径 - ports: # 指定端口映射 - - "5001:5000" - task: + master: image: crawlab:latest - db: - image: mongo + restart: always + volumns: + - /home/yeqing/config.py:/opt/crawlab/crawlab/config/config.py # 后端配置文件 + - /home/yeqing/.env.production:/opt/crawlab/frontend/.env.production # 前端配置文件 + ports: + - "8080:8080" # nginx + - "8000:8000" # app + depends_on: + - mongo + - redis + mongo: + image: mongo:latest restart: always ports: - "27017:27017" redis: - image: redis + image: redis:latest restart: always ports: - "6379:6379"