添加Scrapy设置

This commit is contained in:
marvzhang
2020-02-17 10:30:32 +08:00
parent d80b977f49
commit 25cee2b492
7 changed files with 451 additions and 6 deletions

View File

@@ -1,4 +1,32 @@
export default {
importantSettingParamNames: {
BOT_NAME: String,
SPIDER_MODULES: Array,
NEWSPIDER_MODULE: String,
USER_AGENT: String,
ROBOTSTXT_OBEY: Boolean,
CONCURRENT_REQUESTS: Number,
DOWNLOAD_DELAY: Number,
CONCURRENT_REQUESTS_PER_DOMAIN: Number,
CONCURRENT_REQUESTS_PER_IP: Number,
COOKIES_ENABLED: Boolean,
TELNETCONSOLE_ENABLED: Boolean,
DEFAULT_REQUEST_HEADERS: Object,
SPIDER_MIDDLEWARES: Object,
DOWNLOADER_MIDDLEWARES: Object,
EXTENSIONS: Object,
ITEM_PIPELINES: Object,
AUTOTHROTTLE_ENABLED: Boolean,
AUTOTHROTTLE_START_DELAY: Number,
AUTOTHROTTLE_MAX_DELAY: Number,
AUTOTHROTTLE_TARGET_CONCURRENCY: Number,
AUTOTHROTTLE_DEBUG: Boolean,
HTTPCACHE_ENABLED: Boolean,
HTTPCACHE_EXPIRATION_SECS: Number,
HTTPCACHE_DIR: String,
HTTPCACHE_IGNORE_HTTP_CODES: Array,
HTTPCACHE_STORAGE: String
},
settingParamNames: [
'AWS_ACCESS_KEY_ID',
'AWS_SECRET_ACCESS_KEY',