mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-30 18:00:56 +01:00
18 lines
491 B
Python
18 lines
491 B
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Define your item pipelines here
|
|
#
|
|
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
|
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
|
|
from pymongo import MongoClient
|
|
|
|
|
|
class JdPipeline(object):
|
|
mongo = MongoClient(host=MONGO_HOST, port=MONGO_PORT)
|
|
db = mongo[MONGO_DB]
|
|
col_name = os.environ.get('CRAWLAB_COLLECTION') or 'jd_products'
|
|
col = db[col_name]
|
|
|
|
def process_item(self, item, spider):
|
|
return item
|