From 2e01c6a08f12c8b826c487e0d62291c6305a9237 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 22 May 2019 20:08:38 +0800 Subject: [PATCH] updated sites_inspector --- spiders/sites_inspector/sites_inspector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spiders/sites_inspector/sites_inspector.py b/spiders/sites_inspector/sites_inspector.py index ae425abc..72d2184b 100644 --- a/spiders/sites_inspector/sites_inspector.py +++ b/spiders/sites_inspector/sites_inspector.py @@ -63,7 +63,7 @@ async def request_site_home_page(url: str, semophore): async def run(): semaphore = asyncio.Semaphore(50) # 限制并发量为50 - sites = [site for site in col.find({'rank': {'$lte': 100}})] + sites = [site for site in col.find({'rank': {'$lte': 5000}})] urls = [site['_id'] for site in sites] to_get = [request_site(url, semaphore) for url in urls] to_get += [request_site_home_page(url, semaphore) for url in urls]