git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

updated sites_inspector

Browse Source
This commit is contained in:
Marvin Zhang
2019-05-22 20:08:38 +08:00
parent c69c8e5f64
commit 2e01c6a08f
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
spiders/sites_inspector/sites_inspector.py
View File

@@ -63,7 +63,7 @@ async def request_site_home_page(url: str, semophore):
async def run():
semaphore = asyncio.Semaphore(50) # 限制并发量为50
sites = [site for site in col.find({'rank': {'$lte': 100}})]
sites = [site for site in col.find({'rank': {'$lte': 5000}})]
urls = [site['_id'] for site in sites]
to_get = [request_site(url, semaphore) for url in urls]
to_get += [request_site_home_page(url, semaphore) for url in urls]