git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

updated settings

Browse Source
This commit is contained in:
Marvin Zhang
2019-06-03 12:55:46 +08:00
parent 308173072f
commit bc5d394e52
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawlab/spiders/spiders/settings.py
View File

@@ -19,7 +19,7 @@ NEWSPIDER_MODULE = 'spiders.spiders'
# USER_AGENT = 'spiders (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = spider.get('obey_robots_txt') or True
ROBOTSTXT_OBEY = spider.get('obey_robots_txt') if spider.get('obey_robots_txt') is not None else True
# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32