git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

added generate_urls function

Browse Source
This commit is contained in:
Marvin Zhang
2019-05-28 13:32:00 +08:00
parent 26e7a49a59
commit 7ed9a52f27
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
crawlab/spiders/spiders/utils.py
View File

@@ -27,7 +27,7 @@ def generate_urls(base_url: str) -> str:
url = url.replace('{' + res[0] + ',' + res[1] + '}', '{' + _key + '}', 1)
# string list
for i, res in enumerate(re.findall(r'\[(.+)\]', base_url)):
for i, res in enumerate(re.findall(r'\[([\w\-,]+)\]', base_url)):
# list
_list = res.split(',')
@@ -48,3 +48,8 @@ def generate_urls(base_url: str) -> str:
_list, _key = _arr
_url = _url.replace('{' + _key + '}', str(_rep), 1)
yield _url
#
# base_url = 'http://[baidu,ali].com/page-{1,10}-[1,2,3]'
# for url in generate_urls(base_url):
# print(url)