git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-30 18:00:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

添加demo爬虫

Browse Source
This commit is contained in:
marvzhang
2020-02-02 22:56:11 +08:00
parent 19b2dc1086
commit 98e5ead285
10 changed files with 130 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
spiders/chinaz/Spiderfile Normal file
View File

@@ -0,0 +1,4 @@
name: "chinaz"
display_name: "站长之家 (Scrapy)"
col: "results_chinaz"
cmd: "scrapy crawl chinaz_spider"

21
spiders/chinaz/chinaz/pipelines.py
View File

@@ -5,24 +5,3 @@
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import os
from pymongo import MongoClient
MONGO_HOST = os.environ.get('MONGO_HOST') or 'localhost'
MONGO_PORT = int(os.environ.get('MONGO_PORT') or '27017')
MONGO_DB = os.environ.get('MONGO_DB') or 'crawlab_test'
class MongoPipeline(object):
mongo = MongoClient(host=MONGO_HOST, port=MONGO_PORT)
db = mongo[MONGO_DB]
col_name = os.environ.get('CRAWLAB_COLLECTION') or 'sites'
col = db[col_name]
def process_item(self, item, spider):
item['task_id'] = os.environ.get('CRAWLAB_TASK_ID')
item['_id'] = item['domain']
if self.col.find_one({'_id': item['_id']}) is None:
self.col.save(item)
return item

2
spiders/chinaz/chinaz/settings.py
View File

@@ -65,7 +65,7 @@ ROBOTSTXT_OBEY = True
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'chinaz.pipelines.MongoPipeline': 300,
'crawlab.pipelines.CrawlabMongoPipeline': 300,
}
# Enable and configure the AutoThrottle extension (disabled by default)

19
spiders/realestate/realestate/pipelines.py
View File

@@ -4,22 +4,3 @@
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import os
from pymongo import MongoClient
MONGO_HOST = os.environ.get('MONGO_HOST') or 'localhost'
MONGO_PORT = int(os.environ.get('MONGO_PORT') or '27017')
MONGO_DB = os.environ.get('MONGO_DB') or 'crawlab_test'
class MongoPipeline(object):
mongo = MongoClient(host=MONGO_HOST, port=MONGO_PORT)
db = mongo[MONGO_DB]
col_name = os.environ.get('CRAWLAB_COLLECTION')
col = db[col_name]
def process_item(self, item, spider):
item['task_id'] = os.environ.get('CRAWLAB_TASK_ID')
self.col.save(item)
return item

2
spiders/realestate/realestate/settings.py
View File

@@ -64,7 +64,7 @@ ROBOTSTXT_OBEY = True
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'realestate.pipelines.MongoPipeline': 300,
'crawlab.pipelines.CrawlabMongoPipeline': 300,
}
# Enable and configure the AutoThrottle extension (disabled by default)