diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 6affb507..8c07a507 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -44,6 +44,7 @@ setting: enableTutorial: "N" runOnMaster: "Y" demoSpiders: "N" + checkScrapy: "Y" notification: mail: server: '' diff --git a/backend/services/spider_handler/spider.go b/backend/services/spider_handler/spider.go index ab1d5774..d249c27b 100644 --- a/backend/services/spider_handler/spider.go +++ b/backend/services/spider_handler/spider.go @@ -45,13 +45,15 @@ func (s *SpiderSync) CheckIsScrapy() { if s.Spider.Type == constants.Configurable { return } + if viper.GetString("setting.checkScrapy") != "Y" { + return + } s.Spider.IsScrapy = utils.Exists(path.Join(s.Spider.Src, "scrapy.cfg")) - // TODO: 暂时停用自动检测Scrapy项目功能 - //if err := s.Spider.Save(); err != nil { - // log.Errorf(err.Error()) - // debug.PrintStack() - // return - //} + if err := s.Spider.Save(); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return + } } func (s *SpiderSync) AfterRemoveDownCreate() { diff --git a/docker-compose.yml b/docker-compose.yml index e9cb08cf..7f4f95da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,9 +19,9 @@ services: # CRAWLAB_LOG_LEVEL: "info" # log level 日志级别. 默认为 info # CRAWLAB_LOG_ISDELETEPERIODICALLY: "N" # whether to periodically delete log files 是否周期性删除日志文件. 默认不删除 # CRAWLAB_LOG_DELETEFREQUENCY: "@hourly" # frequency of deleting log files 删除日志文件的频率. 默认为每小时 + # CRAWLAB_TASK_WORKERS: 8 # number of task executors 任务执行器个数(并行执行任务数) # CRAWLAB_SERVER_REGISTER_TYPE: "mac" # node register type 节点注册方式. 默认为 mac 地址,也可设置为 ip(防止 mac 地址冲突) # CRAWLAB_SERVER_REGISTER_IP: "127.0.0.1" # node register ip 节点注册IP. 节点唯一识别号,只有当 CRAWLAB_SERVER_REGISTER_TYPE 为 "ip" 时才生效 - # CRAWLAB_TASK_WORKERS: 8 # number of task executors 任务执行器个数(并行执行任务数) # CRAWLAB_SERVER_LANG_NODE: "Y" # whether to pre-install Node.js 预安装 Node.js 语言环境 # CRAWLAB_SERVER_LANG_JAVA: "Y" # whether to pre-install Java 预安装 Java 语言环境 # CRAWLAB_SERVER_LANG_DOTNET: "Y" # whether to pre-install .Net core 预安装 .Net Core 语言环境 @@ -30,6 +30,7 @@ services: # CRAWLAB_SETTING_ENABLETUTORIAL: "N" # whether to enable tutorial 是否启用教程 # CRAWLAB_SETTING_RUNONMASTER: "N" # whether to run on master node 是否在主节点上运行任务 # CRAWLAB_SETTING_DEMOSPIDERS: "Y" # whether to init demo spiders 是否使用Demo爬虫 + # CRAWLAB_SETTING_CHECKSCRAPY: "Y" # whether to automatically check if the spider is scrapy 是否自动检测爬虫为scrapy # CRAWLAB_NOTIFICATION_MAIL_SERVER: smtp.exmaple.com # STMP server address STMP 服务器地址 # CRAWLAB_NOTIFICATION_MAIL_PORT: 465 # STMP server port STMP 服务器端口 # CRAWLAB_NOTIFICATION_MAIL_SENDEREMAIL: admin@exmaple.com # sender email 发送者邮箱