git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-28 17:50:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13 from crawlab-team/develop

Browse Source 
Develop
This commit is contained in:
暗音
2019-11-26 15:29:18 +08:00
committed by GitHub
parent 0a94399da2 e2e61c621e
commit 3ef5ddfa07
172 changed files with 19974 additions and 6034 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
backend/services/task.go
View File

@@ -224,7 +224,16 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e
}
// 环境变量配置
cmd = SetEnv(cmd, s.Envs, t.Id, s.Col)
envs := s.Envs
if s.Type == constants.Configurable {
envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_HOST", Value: viper.GetString("mongo.host")})
envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_PORT", Value: viper.GetString("mongo.port")})
envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_DB", Value: viper.GetString("mongo.db")})
envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_USERNAME", Value: viper.GetString("mongo.username")})
envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_PASSWORD", Value: viper.GetString("mongo.password")})
envs = append(envs, model.Env{Name: "CRAWLAB_MONGO_AUTHSOURCE", Value: viper.GetString("mongo.authSource")})
}
cmd = SetEnv(cmd, envs, t.Id, s.Col)
// 起一个goroutine来监控进程
ch := utils.TaskExecChanMap.ChanBlocked(t.Id)
@@ -378,7 +387,14 @@ func ExecuteTask(id int) {
)
// 执行命令
cmd := spider.Cmd
var cmd string
if spider.Type == constants.Configurable {
// 可配置爬虫命令
cmd = "scrapy crawl config_spider"
} else {
// 自定义爬虫命令
cmd = spider.Cmd
}
// 加入参数
if t.Param != "" {