git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

download results

Browse Source
This commit is contained in:
Marvin Zhang
2019-05-30 13:22:42 +08:00
parent 0370fe4bc6
commit fb11a14040
2 changed files with 8 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawlab/routes/tasks.py
View File

@@ -224,7 +224,7 @@ class TaskApi(BaseApi):
if not col_name:
return send_csv([], f'results_{col_name}_{round(time())}.csv')
items = db_manager.list(col_name, {'task_id': id}, limit=999999999)
fields = get_spider_col_fields(col_name)
fields = get_spider_col_fields(col_name, task_id=id, limit=999999999)
return send_csv(items,
filename=f'results_{col_name}_{round(time())}.csv',
fields=fields,

9
crawlab/utils/spider.py
View File

@@ -41,12 +41,17 @@ def get_spider_type(path: str) -> SpiderType:
return SpiderType.SCRAPY
def get_spider_col_fields(col_name: str) -> list:
def get_spider_col_fields(col_name: str, task_id: str = None, limit: int = 100) -> list:
"""
Get spider collection fields
:param col_name: collection name
:param task_id: task_id
:param limit: limit
"""
items = db_manager.list(col_name, {}, limit=100, sort_key='_id')
filter_ = {}
if task_id is not None:
filter_['task_id'] = task_id
items = db_manager.list(col_name, filter_, limit=limit, sort_key='_id')
fields = set()
for item in items:
for k in item.keys():