diff --git a/Dockerfile b/Dockerfile index 61beffb7..92afee28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ && ln -s /usr/bin/python3 /usr/local/bin/python # install backend -RUN pip install scrapy pymongo bs4 requests +RUN pip install scrapy pymongo bs4 requests crawlab-sdk # add files ADD . /app diff --git a/Dockerfile.local b/Dockerfile.local index f095920d..91e24a63 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -33,7 +33,7 @@ RUN chmod 777 /tmp \ && ln -s /usr/bin/python3 /usr/local/bin/python # install backend -RUN pip install scrapy pymongo bs4 requests -i https://pypi.tuna.tsinghua.edu.cn/simple +RUN pip install scrapy pymongo bs4 requests crawlab-sdk -i https://pypi.tuna.tsinghua.edu.cn/simple # add files ADD . /app diff --git a/backend/services/task.go b/backend/services/task.go index 7da6b022..7a2eed2a 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -119,6 +119,20 @@ func SetEnv(cmd *exec.Cmd, envs []model.Env, taskId string, dataCol string) *exe // 默认环境变量 cmd.Env = append(os.Environ(), "CRAWLAB_TASK_ID="+taskId) cmd.Env = append(cmd.Env, "CRAWLAB_COLLECTION="+dataCol) + cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_HOST="+viper.GetString("mongo.host")) + cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_PORT="+viper.GetString("mongo.port")) + if viper.GetString("mongo.db") != "" { + cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_DB="+viper.GetString("mongo.db")) + } + if viper.GetString("mongo.username") != "" { + cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_USERNAME="+viper.GetString("mongo.username")) + } + if viper.GetString("mongo.password") != "" { + cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_PASSWORD="+viper.GetString("mongo.password")) + } + if viper.GetString("mongo.authSource") != "" { + cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_AUTHSOURCE="+viper.GetString("mongo.authSource")) + } cmd.Env = append(cmd.Env, "PYTHONUNBUFFERED=0") cmd.Env = append(cmd.Env, "PYTHONIOENCODING=utf-8") cmd.Env = append(cmd.Env, "TZ=Asia/Shanghai")