git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

加入检测scrapy配置

Browse Source
This commit is contained in:
marvzhang
2020-03-12 09:21:55 +08:00
parent df95e05878
commit 0d612e92cf
3 changed files with 11 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/conf/config.yml
View File

@@ -44,6 +44,7 @@ setting:
enableTutorial: "N"
runOnMaster: "Y"
demoSpiders: "N"
checkScrapy: "Y"
notification:
mail:
server: ''

14
backend/services/spider_handler/spider.go
View File

@@ -45,13 +45,15 @@ func (s *SpiderSync) CheckIsScrapy() {
if s.Spider.Type == constants.Configurable {
return
}
if viper.GetString("setting.checkScrapy") != "Y" {
return
}
s.Spider.IsScrapy = utils.Exists(path.Join(s.Spider.Src, "scrapy.cfg"))
// TODO: 暂时停用自动检测Scrapy项目功能
//if err := s.Spider.Save(); err != nil {
// log.Errorf(err.Error())
// debug.PrintStack()
// return
//}
if err := s.Spider.Save(); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
}
}
func (s *SpiderSync) AfterRemoveDownCreate() {

3
docker-compose.yml
View File

@@ -19,9 +19,9 @@ services:
# CRAWLAB_LOG_LEVEL: "info" # log level 日志级别. 默认为 info
# CRAWLAB_LOG_ISDELETEPERIODICALLY: "N" # whether to periodically delete log files 是否周期性删除日志文件. 默认不删除
# CRAWLAB_LOG_DELETEFREQUENCY: "@hourly" # frequency of deleting log files 删除日志文件的频率. 默认为每小时
# CRAWLAB_TASK_WORKERS: 8 # number of task executors 任务执行器个数（并行执行任务数）
# CRAWLAB_SERVER_REGISTER_TYPE: "mac" # node register type 节点注册方式. 默认为 mac 地址，也可设置为 ip（防止 mac 地址冲突）
# CRAWLAB_SERVER_REGISTER_IP: "127.0.0.1" # node register ip 节点注册IP. 节点唯一识别号，只有当 CRAWLAB_SERVER_REGISTER_TYPE 为 "ip" 时才生效
# CRAWLAB_TASK_WORKERS: 8 # number of task executors 任务执行器个数（并行执行任务数）
# CRAWLAB_SERVER_LANG_NODE: "Y" # whether to pre-install Node.js 预安装 Node.js 语言环境
# CRAWLAB_SERVER_LANG_JAVA: "Y" # whether to pre-install Java 预安装 Java 语言环境
# CRAWLAB_SERVER_LANG_DOTNET: "Y" # whether to pre-install .Net core 预安装 .Net Core 语言环境
@@ -30,6 +30,7 @@ services:
# CRAWLAB_SETTING_ENABLETUTORIAL: "N" # whether to enable tutorial 是否启用教程
# CRAWLAB_SETTING_RUNONMASTER: "N" # whether to run on master node 是否在主节点上运行任务
# CRAWLAB_SETTING_DEMOSPIDERS: "Y" # whether to init demo spiders 是否使用Demo爬虫
# CRAWLAB_SETTING_CHECKSCRAPY: "Y" # whether to automatically check if the spider is scrapy 是否自动检测爬虫为scrapy
# CRAWLAB_NOTIFICATION_MAIL_SERVER: smtp.exmaple.com # STMP server address STMP 服务器地址
# CRAWLAB_NOTIFICATION_MAIL_PORT: 465 # STMP server port STMP 服务器端口
# CRAWLAB_NOTIFICATION_MAIL_SENDEREMAIL: admin@exmaple.com # sender email 发送者邮箱