更新可配置爬虫

This commit is contained in:
marvzhang
2019-12-03 12:14:38 +08:00
parent c3bc20127d
commit 9e849695d1
2 changed files with 1 additions and 3 deletions

View File

@@ -187,7 +187,7 @@ func (g ScrapyGenerator) GetListParserString(stageName string, stage entity.Stag
// 分页
if stage.PageCss != "" || stage.PageXpath != "" {
str += g.PadCode(fmt.Sprintf(`next_url = response.%s.extract_first()`, g.GetExtractStringFromStage(stage)), 2)
str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, next_url), callback=self.parse_%s, meta={'item': item})`, stageName), 2)
str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, next_url), callback=self.parse_%s, meta={'item': prev_item})`, stageName), 2)
}
// 加入末尾换行

View File

@@ -102,7 +102,5 @@ for setting_env_name in [x for x in os.environ.keys() if x.startswith('CRAWLAB_S
setting_value = int(setting_value)
else:
pass
print(setting_name)
print(setting_value)
locals()[setting_name] = setting_value