From bc5d394e5227a4dfdef204df1be08a3587c923d8 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 3 Jun 2019 12:55:46 +0800 Subject: [PATCH] updated settings --- crawlab/spiders/spiders/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlab/spiders/spiders/settings.py b/crawlab/spiders/spiders/settings.py index d78d9281..83685fec 100644 --- a/crawlab/spiders/spiders/settings.py +++ b/crawlab/spiders/spiders/settings.py @@ -19,7 +19,7 @@ NEWSPIDER_MODULE = 'spiders.spiders' # USER_AGENT = 'spiders (+http://www.yourdomain.com)' # Obey robots.txt rules -ROBOTSTXT_OBEY = spider.get('obey_robots_txt') or True +ROBOTSTXT_OBEY = spider.get('obey_robots_txt') if spider.get('obey_robots_txt') is not None else True # Configure maximum concurrent requests performed by Scrapy (default: 16) # CONCURRENT_REQUESTS = 32