git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

added chinaz spider

Browse Source
This commit is contained in:
Marvin Zhang
2019-05-10 12:39:10 +08:00
parent 93f9612b9b
commit 69e2e2f3c5
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
spiders/chinaz/chinaz/pipelines.py
View File

@@ -23,5 +23,6 @@ class MongoPipeline(object):
def process_item(self, item, spider):
item['task_id'] = os.environ.get('CRAWLAB_TASK_ID')
item['_id'] = item['domain']
self.col.save(item)
if self.col.find_one({'_id': item['_id']}) is not None:
self.col.save(item)
return item