git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

improved compatibility with crawlab-sdk

Browse Source
This commit is contained in:
marvzhang
2020-01-13 12:32:06 +08:00
parent e137bf4773
commit 746abbdc04
3 changed files with 16 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@@ -35,7 +35,7 @@ RUN apt-get update \
&& ln -s /usr/bin/python3 /usr/local/bin/python
# install backend
RUN pip install scrapy pymongo bs4 requests
RUN pip install scrapy pymongo bs4 requests crawlab-sdk
# add files
ADD . /app

2
Dockerfile.local
View File

@@ -33,7 +33,7 @@ RUN chmod 777 /tmp \
&& ln -s /usr/bin/python3 /usr/local/bin/python
# install backend
RUN pip install scrapy pymongo bs4 requests -i https://pypi.tuna.tsinghua.edu.cn/simple
RUN pip install scrapy pymongo bs4 requests crawlab-sdk -i https://pypi.tuna.tsinghua.edu.cn/simple
# add files
ADD . /app

14
backend/services/task.go
View File

@@ -119,6 +119,20 @@ func SetEnv(cmd *exec.Cmd, envs []model.Env, taskId string, dataCol string) *exe
// 默认环境变量
cmd.Env = append(os.Environ(), "CRAWLAB_TASK_ID="+taskId)
cmd.Env = append(cmd.Env, "CRAWLAB_COLLECTION="+dataCol)
cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_HOST="+viper.GetString("mongo.host"))
cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_PORT="+viper.GetString("mongo.port"))
if viper.GetString("mongo.db") != "" {
cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_DB="+viper.GetString("mongo.db"))
}
if viper.GetString("mongo.username") != "" {
cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_USERNAME="+viper.GetString("mongo.username"))
}
if viper.GetString("mongo.password") != "" {
cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_PASSWORD="+viper.GetString("mongo.password"))
}
if viper.GetString("mongo.authSource") != "" {
cmd.Env = append(cmd.Env, "CRAWLAB_MONGO_AUTHSOURCE="+viper.GetString("mongo.authSource"))
}
cmd.Env = append(cmd.Env, "PYTHONUNBUFFERED=0")
cmd.Env = append(cmd.Env, "PYTHONIOENCODING=utf-8")
cmd.Env = append(cmd.Env, "TZ=Asia/Shanghai")