git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

pagination change

Browse Source
This commit is contained in:
Marvin Zhang
2019-06-09 17:01:17 +08:00
parent ceba5b86e3
commit ada606afbb
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
crawlab/spiders/spiders/spiders/config_spider.py
View File

@@ -73,7 +73,10 @@ def get_next_url(response):
if next_url is not None:
if not next_url.startswith('http') and not next_url.startswith('//'):
u = urlparse(response.url)
next_url = f'{u.scheme}://{u.netloc}{next_url}'
if next_url.startswith('/'):
next_url = f'{u.scheme}://{u.netloc}{next_url}'
else:
next_url = f'{u.scheme}://{u.netloc}{u.path}/{next_url}'
return next_url
return None