diff --git a/Dockerfile b/Dockerfile index 61beffb7..92afee28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ && ln -s /usr/bin/python3 /usr/local/bin/python # install backend -RUN pip install scrapy pymongo bs4 requests +RUN pip install scrapy pymongo bs4 requests crawlab-sdk # add files ADD . /app diff --git a/Dockerfile.local b/Dockerfile.local index f095920d..91e24a63 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -33,7 +33,7 @@ RUN chmod 777 /tmp \ && ln -s /usr/bin/python3 /usr/local/bin/python # install backend -RUN pip install scrapy pymongo bs4 requests -i https://pypi.tuna.tsinghua.edu.cn/simple +RUN pip install scrapy pymongo bs4 requests crawlab-sdk -i https://pypi.tuna.tsinghua.edu.cn/simple # add files ADD . /app diff --git a/README-zh.md b/README-zh.md index cc3b3494..b6319a07 100644 --- a/README-zh.md +++ b/README-zh.md @@ -41,8 +41,9 @@ ### 要求(Docker) - Docker 18.03+ -- Redis +- Redis 5.x+ - MongoDB 3.6+ +- Docker Compose 1.24+ (可选,但推荐) ### 要求(直接部署) - Go 1.12+ @@ -52,12 +53,16 @@ ## 快速开始 +请打开命令行并执行下列命令。请保证您已经提前安装了 `docker-compose`。 + ```bash git clone https://github.com/crawlab-team/crawlab cd crawlab docker-compose up -d ``` +接下来,您可以看看 `docker-compose.yml` (包含详细配置参数),以及参考 [文档](http://docs.crawlab.cn) 来查看更多信息。 + ## 运行 ### Docker @@ -71,13 +76,11 @@ services: image: tikazyq/crawlab:latest container_name: master environment: - CRAWLAB_API_ADDRESS: "http://localhost:8000" CRAWLAB_SERVER_MASTER: "Y" CRAWLAB_MONGO_HOST: "mongo" CRAWLAB_REDIS_ADDRESS: "redis" ports: - - "8080:8080" # frontend - - "8000:8000" # backend + - "8080:8080" depends_on: - mongo - redis diff --git a/README.md b/README.md index 02b2c5f0..20bc9e4b 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,9 @@ Two methods: ### Pre-requisite (Docker) - Docker 18.03+ -- Redis +- Redis 5.x+ - MongoDB 3.6+ +- Docker Compose 1.24+ (optional but recommended) ### Pre-requisite (Direct Deploy) - Go 1.12+ @@ -52,12 +53,16 @@ Two methods: ## Quick Start +Please open the command line prompt and execute the command beloe. Make sure you have installed `docker-compose` in advance. + ```bash git clone https://github.com/crawlab-team/crawlab cd crawlab docker-compose up -d ``` +Next, you can look into the `docker-compose.yml` (with detailed config params) and the [Documentation (Chinese)](http://docs.crawlab.cn) for further information. + ## Run ### Docker @@ -76,8 +81,7 @@ services: CRAWLAB_MONGO_HOST: "mongo" CRAWLAB_REDIS_ADDRESS: "redis" ports: - - "8080:8080" # frontend - - "8000:8000" # backend + - "8080:8080" depends_on: - mongo - redis diff --git a/backend/services/task.go b/backend/services/task.go index 7da6b022..7a2eed2a 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -119,6 +119,20 @@ func SetEnv(cmd *exec.Cmd, envs []model.Env, taskId string, dataCol string) *exe // 默认环境变量 cmd.Env = append(os.Environ(), "CRAWLAB_TASK_ID="+taskId) cmd.Env = append(cmd.Env, "CRAWLAB_COLLECTION="+dataCol) + cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_HOST="+viper.GetString("mongo.host")) + cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_PORT="+viper.GetString("mongo.port")) + if viper.GetString("mongo.db") != "" { + cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_DB="+viper.GetString("mongo.db")) + } + if viper.GetString("mongo.username") != "" { + cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_USERNAME="+viper.GetString("mongo.username")) + } + if viper.GetString("mongo.password") != "" { + cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_PASSWORD="+viper.GetString("mongo.password")) + } + if viper.GetString("mongo.authSource") != "" { + cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_AUTHSOURCE="+viper.GetString("mongo.authSource")) + } cmd.Env = append(cmd.Env, "PYTHONUNBUFFERED=0") cmd.Env = append(cmd.Env, "PYTHONIOENCODING=utf-8") cmd.Env = append(cmd.Env, "TZ=Asia/Shanghai")