mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-23 17:31:11 +01:00
更新可配置爬虫
This commit is contained in:
@@ -187,7 +187,7 @@ func (g ScrapyGenerator) GetListParserString(stageName string, stage entity.Stag
|
||||
// 分页
|
||||
if stage.PageCss != "" || stage.PageXpath != "" {
|
||||
str += g.PadCode(fmt.Sprintf(`next_url = response.%s.extract_first()`, g.GetExtractStringFromStage(stage)), 2)
|
||||
str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, next_url), callback=self.parse_%s, meta={'item': item})`, stageName), 2)
|
||||
str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, next_url), callback=self.parse_%s, meta={'item': prev_item})`, stageName), 2)
|
||||
}
|
||||
|
||||
// 加入末尾换行
|
||||
|
||||
@@ -102,7 +102,5 @@ for setting_env_name in [x for x in os.environ.keys() if x.startswith('CRAWLAB_S
|
||||
setting_value = int(setting_value)
|
||||
else:
|
||||
pass
|
||||
print(setting_name)
|
||||
print(setting_value)
|
||||
locals()[setting_name] = setting_value
|
||||
|
||||
|
||||
Reference in New Issue
Block a user