From a87950e3694805ac8ae04f8c642bbf6a5703a2b4 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 25 Nov 2019 22:07:20 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E7=8E=AF=E5=A2=83=E5=8F=98?= =?UTF-8?q?=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/config/config.go | 2 +- backend/services/task.go | 11 ++++++++++- backend/template/scrapy/config_spider/pipelines.py | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/backend/config/config.go b/backend/config/config.go index 4d83c0f7..e4c4616c 100644 --- a/backend/config/config.go +++ b/backend/config/config.go @@ -28,7 +28,7 @@ func (c *Config) Init() error { } viper.SetConfigType("yaml") // 设置配置文件格式为YAML viper.AutomaticEnv() // 读取匹配的环境变量 - viper.SetEnvPrefix("CRAWLAB") // 读取环境变量的前缀为APISERVER + viper.SetEnvPrefix("CRAWLAB") // 读取环境变量的前缀为CRAWLAB replacer := strings.NewReplacer(".", "_") viper.SetEnvKeyReplacer(replacer) if err := viper.ReadInConfig(); err != nil { // viper解析配置文件 diff --git a/backend/services/task.go b/backend/services/task.go index 7138f0f9..9336e75d 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -217,7 +217,16 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e } // 环境变量配置 - cmd = SetEnv(cmd, s.Envs, t.Id, s.Col) + envs := s.Envs + if s.Type == constants.Configurable { + envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_HOST", Value: viper.GetString("mongo.host")}) + envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_PORT", Value: viper.GetString("mongo.port")}) + envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_DB", Value: viper.GetString("mongo.db")}) + envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_USERNAME", Value: viper.GetString("mongo.username")}) + envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_PASSWORD", Value: viper.GetString("mongo.password")}) + envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_AUTHSOURCE", Value: viper.GetString("mongo.authSource")}) + } + cmd = SetEnv(cmd, envs, t.Id, s.Col) // 起一个goroutine来监控进程 ch := utils.TaskExecChanMap.ChanBlocked(t.Id) diff --git a/backend/template/scrapy/config_spider/pipelines.py b/backend/template/scrapy/config_spider/pipelines.py index 7a3286ff..69af4c85 100644 --- a/backend/template/scrapy/config_spider/pipelines.py +++ b/backend/template/scrapy/config_spider/pipelines.py @@ -16,7 +16,7 @@ mongo = MongoClient( authSource=os.environ.get('CRAWLAB_MONGO_AUTHSOURCE') or 'admin' ) db = mongo[os.environ.get('CRAWLAB_MONGO_DB') or 'test'] -col_name = os.environ.get('CRAWLAB_COLLECTION') or 'test' +col = db[os.environ.get('CRAWLAB_COLLECTION') or 'test'] task_id = os.environ.get('CRAWLAB_TASK_ID') class ConfigSpiderPipeline(object):