diff --git a/crawlab/tasks/spider.py b/crawlab/tasks/spider.py index e03bab66..3bda236c 100644 --- a/crawlab/tasks/spider.py +++ b/crawlab/tasks/spider.py @@ -2,6 +2,7 @@ import os import sys from datetime import datetime from time import sleep +import traceback from bson import ObjectId from pymongo import ASCENDING, DESCENDING @@ -213,8 +214,10 @@ def execute_config_spider(self, id: str, params: str = None): env['MONGO_HOST'] = MONGO_HOST env['MONGO_PORT'] = str(MONGO_PORT) env['MONGO_DB'] = MONGO_DB - env['MONGO_USERNAME'] = MONGO_USERNAME - env['MONGO_PASSWORD'] = MONGO_PASSWORD + if MONGO_USERNAME is not None: + env['MONGO_USERNAME'] = MONGO_USERNAME + if MONGO_PASSWORD: + env['MONGO_PASSWORD'] = MONGO_PASSWORD cmd_arr = [ sys.executable, @@ -246,6 +249,7 @@ def execute_config_spider(self, id: str, params: str = None): else: status = TaskStatus.FAILURE except Exception as err: + traceback.print_exc() logger.error(err) stderr.write(str(err)) status = TaskStatus.FAILURE diff --git a/spiders/xueqiu/xueqiu/pipelines.py b/spiders/xueqiu/xueqiu/pipelines.py index 90a86da2..67173772 100644 --- a/spiders/xueqiu/xueqiu/pipelines.py +++ b/spiders/xueqiu/xueqiu/pipelines.py @@ -12,7 +12,7 @@ from pymongo import MongoClient class XueqiuPipeline(object): mongo = MongoClient( host=os.environ.get('MONGO_HOST') or 'localhost', - port=int(os.environ.get('MONGO_PORT')) or 27017 + port=int(os.environ.get('MONGO_PORT') or 27017) ) db = mongo[os.environ.get('MONGO_DB') or 'crawlab_test'] col = db.get_collection(os.environ.get('CRAWLAB_COLLECTION') or 'results_xueqiu')