mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-22 17:31:03 +01:00
Marvin Zhang
2
crawlab/.gitignore
vendored
2
crawlab/.gitignore
vendored
@@ -110,3 +110,5 @@ node_modules/
|
||||
|
||||
# egg-info
|
||||
*.egg-info
|
||||
|
||||
tmp/
|
||||
|
||||
@@ -71,7 +71,8 @@ api.add_resource(ScheduleApi,
|
||||
'/api/schedules/<string:id>')
|
||||
api.add_resource(SiteApi,
|
||||
'/api/sites',
|
||||
'/api/sites/<string:id>')
|
||||
'/api/sites/<string:id>',
|
||||
'/api/sites/get/<string:action>')
|
||||
|
||||
|
||||
def monitor_nodes_status(celery_app):
|
||||
|
||||
@@ -13,7 +13,7 @@ import tasks.spider
|
||||
import tasks.deploy
|
||||
|
||||
if __name__ == '__main__':
|
||||
if sys.platform == 'windows':
|
||||
if 'win' in sys.platform:
|
||||
celery_app.start(argv=['tasks', 'worker', '-P', 'eventlet', '-E', '-l', 'INFO'])
|
||||
else:
|
||||
celery_app.start(argv=['tasks', 'worker', '-E', '-l', 'INFO'])
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
class SpiderType:
|
||||
SCRAPY = 'scrapy'
|
||||
PYSPIDER = 'pyspider'
|
||||
WEBMAGIC = 'webmagic'
|
||||
CONFIGURABLE = 'configurable'
|
||||
CUSTOMIZED = 'customized'
|
||||
|
||||
|
||||
class LangType:
|
||||
@@ -17,6 +16,22 @@ class CronEnabled:
|
||||
OFF = 0
|
||||
|
||||
|
||||
class CrawlType:
|
||||
LIST = 'list'
|
||||
DETAIL = 'detail'
|
||||
LIST_DETAIL = 'list-detail'
|
||||
|
||||
|
||||
class QueryType:
|
||||
CSS = 'css'
|
||||
XPATH = 'xpath'
|
||||
|
||||
|
||||
class ExtractType:
|
||||
TEXT = 'text'
|
||||
ATTRIBUTE = 'attribute'
|
||||
|
||||
|
||||
SUFFIX_IGNORE = [
|
||||
'pyc'
|
||||
]
|
||||
|
||||
@@ -179,5 +179,9 @@ class DbManager(object):
|
||||
col = self.db[col_name]
|
||||
col.create_index(keys=keys, **kwargs)
|
||||
|
||||
def distinct(self, col_name: str, key: str, filter: dict):
|
||||
col = self.db[col_name]
|
||||
return sorted(col.distinct(key, filter))
|
||||
|
||||
|
||||
db_manager = DbManager()
|
||||
|
||||
@@ -1,35 +1,79 @@
|
||||
aiohttp==3.5.4
|
||||
amqp==2.4.2
|
||||
aniso8601==6.0.0
|
||||
Appium-Python-Client==0.40
|
||||
APScheduler==3.6.0
|
||||
asn1crypto==0.24.0
|
||||
async-timeout==3.0.1
|
||||
attrs==19.1.0
|
||||
Automat==0.7.0
|
||||
Babel==2.6.0
|
||||
beautifulsoup4==4.7.1
|
||||
billiard==3.6.0.0
|
||||
bs4==0.0.1
|
||||
cachetools==3.1.0
|
||||
celery==4.3.0
|
||||
certifi==2019.3.9
|
||||
cffi==1.12.3
|
||||
chardet==3.0.4
|
||||
Click==7.0
|
||||
coloredlogs==10.0
|
||||
constantly==15.1.0
|
||||
cryptography==2.6.1
|
||||
cssselect==1.0.3
|
||||
Flask==1.0.2
|
||||
Flask-APScheduler==1.11.0
|
||||
Flask-Cors==3.0.7
|
||||
Flask-RESTful==0.3.7
|
||||
flask-restplus==0.12.1
|
||||
flower==0.9.3
|
||||
gevent==1.4.0
|
||||
greenlet==0.4.15
|
||||
gunicorn==19.9.0
|
||||
html5lib==1.0.1
|
||||
humanfriendly==4.18
|
||||
hyperlink==19.0.0
|
||||
idna==2.8
|
||||
idna-ssl==1.1.0
|
||||
incremental==17.5.0
|
||||
itsdangerous==1.1.0
|
||||
Jinja2==2.10
|
||||
jsonpickle==1.1
|
||||
jsonschema==3.0.1
|
||||
kombu==4.5.0
|
||||
lxml==4.3.3
|
||||
MarkupSafe==1.1.1
|
||||
mongoengine==0.17.0
|
||||
multidict==4.5.2
|
||||
parsel==1.5.1
|
||||
pyasn1==0.4.5
|
||||
pyasn1-modules==0.2.5
|
||||
pycparser==2.19
|
||||
PyDispatcher==2.0.5
|
||||
PyHamcrest==1.9.0
|
||||
pymongo==3.7.2
|
||||
pyOpenSSL==19.0.0
|
||||
pyrsistent==0.14.11
|
||||
python-dateutil==2.8.0
|
||||
pytz==2018.9
|
||||
queuelib==1.5.0
|
||||
redis==3.2.1
|
||||
redisbeat==1.1.4
|
||||
reppy==0.4.12
|
||||
requests==2.21.0
|
||||
Scrapy==1.6.0
|
||||
selenium==3.141.0
|
||||
service-identity==18.1.0
|
||||
six==1.12.0
|
||||
soupsieve==1.9.1
|
||||
tornado==5.1.1
|
||||
Twisted==19.2.0
|
||||
typing-extensions==3.7.2
|
||||
tzlocal==1.5.1
|
||||
urllib3==1.24.1
|
||||
vine==1.3.0
|
||||
w3lib==1.20.0
|
||||
webencodings==0.5.1
|
||||
Werkzeug==0.15.2
|
||||
yarl==1.3.0
|
||||
zope.interface==4.6.0
|
||||
|
||||
@@ -111,7 +111,7 @@ class BaseApi(Resource):
|
||||
|
||||
self.after_update()
|
||||
|
||||
return item
|
||||
return jsonify(item)
|
||||
|
||||
def update(self, id: str = None) -> (dict, tuple):
|
||||
"""
|
||||
@@ -137,7 +137,7 @@ class BaseApi(Resource):
|
||||
# execute after_update hook
|
||||
self.after_update(id)
|
||||
|
||||
return item
|
||||
return jsonify(item)
|
||||
|
||||
def post(self, id: str = None, action: str = None):
|
||||
"""
|
||||
|
||||
@@ -13,6 +13,7 @@ class SiteApi(BaseApi):
|
||||
|
||||
arguments = (
|
||||
('keyword', str),
|
||||
('main_category', str),
|
||||
('category', str),
|
||||
)
|
||||
|
||||
@@ -70,3 +71,20 @@ class SiteApi(BaseApi):
|
||||
'page_size': page_size,
|
||||
'items': jsonify(sites)
|
||||
}
|
||||
|
||||
def get_main_category_list(self, id):
|
||||
return {
|
||||
'status': 'ok',
|
||||
'items': db_manager.distinct(col_name=self.col_name, key='main_category', filter={})
|
||||
}
|
||||
|
||||
def get_category_list(self, id):
|
||||
args = self.parser.parse_args()
|
||||
filter_ = {}
|
||||
if args.get('main_category') is not None:
|
||||
filter_['main_category'] = args.get('main_category')
|
||||
return {
|
||||
'status': 'ok',
|
||||
'items': db_manager.distinct(col_name=self.col_name, key='category',
|
||||
filter=filter_)
|
||||
}
|
||||
|
||||
@@ -5,23 +5,27 @@ import subprocess
|
||||
from datetime import datetime
|
||||
from random import random
|
||||
|
||||
import gevent
|
||||
import requests
|
||||
from bson import ObjectId
|
||||
from flask import current_app, request
|
||||
from flask_restful import reqparse, Resource
|
||||
from lxml import etree
|
||||
from werkzeug.datastructures import FileStorage
|
||||
|
||||
from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_SOURCE_FILE_FOLDER, PROJECT_TMP_FOLDER
|
||||
from constants.node import NodeStatus
|
||||
from constants.spider import SpiderType, CrawlType, QueryType, ExtractType
|
||||
from constants.task import TaskStatus
|
||||
from db.manager import db_manager
|
||||
from routes.base import BaseApi
|
||||
from tasks.scheduler import scheduler
|
||||
from tasks.spider import execute_spider
|
||||
from tasks.spider import execute_spider, execute_config_spider
|
||||
from utils import jsonify
|
||||
from utils.deploy import zip_file, unzip_file
|
||||
from utils.file import get_file_suffix_stats, get_file_suffix
|
||||
from utils.spider import get_lang_by_stats, get_last_n_run_errors_count, get_last_n_day_tasks_count
|
||||
from utils.spider import get_lang_by_stats, get_last_n_run_errors_count, get_last_n_day_tasks_count, get_list_page_data, \
|
||||
get_detail_page_data
|
||||
|
||||
parser = reqparse.RequestParser()
|
||||
parser.add_argument('file', type=FileStorage, location='files')
|
||||
@@ -64,6 +68,37 @@ class SpiderApi(BaseApi):
|
||||
|
||||
# spider site
|
||||
('site', str),
|
||||
|
||||
########################
|
||||
# Configurable Spider
|
||||
########################
|
||||
|
||||
# spider crawl fields for list page
|
||||
('fields', str),
|
||||
|
||||
# spider crawl fields for detail page
|
||||
('detail_fields', str),
|
||||
|
||||
# spider crawl type
|
||||
('crawl_type', str),
|
||||
|
||||
# spider start url
|
||||
('start_url', str),
|
||||
|
||||
# spider item selector
|
||||
('item_selector', str),
|
||||
|
||||
# spider item selector type
|
||||
('item_selector_type', str),
|
||||
|
||||
# spider pagination selector
|
||||
('pagination_selector', str),
|
||||
|
||||
# spider pagination selector type
|
||||
('pagination_selector_type', str),
|
||||
|
||||
# whether to obey robots.txt
|
||||
('obey_robots_txt', str),
|
||||
)
|
||||
|
||||
def get(self, id=None, action=None):
|
||||
@@ -96,6 +131,8 @@ class SpiderApi(BaseApi):
|
||||
# get a list of items
|
||||
else:
|
||||
items = []
|
||||
|
||||
# get customized spiders
|
||||
dirs = os.listdir(PROJECT_SOURCE_FILE_FOLDER)
|
||||
for _dir in dirs:
|
||||
if _dir in IGNORE_DIRS:
|
||||
@@ -114,6 +151,7 @@ class SpiderApi(BaseApi):
|
||||
'src': dir_path,
|
||||
'lang': lang,
|
||||
'suffix_stats': stats,
|
||||
'type': SpiderType.CUSTOMIZED
|
||||
})
|
||||
|
||||
# existing spider
|
||||
@@ -123,39 +161,52 @@ class SpiderApi(BaseApi):
|
||||
if last_deploy is not None:
|
||||
spider['deploy_ts'] = last_deploy['finish_ts']
|
||||
|
||||
# get last task
|
||||
last_task = db_manager.get_last_task(spider_id=spider['_id'])
|
||||
if last_task is not None:
|
||||
spider['task_ts'] = last_task['create_ts']
|
||||
|
||||
# get site
|
||||
if spider.get('site') is not None:
|
||||
site = db_manager.get('sites', spider['site'])
|
||||
if site is not None:
|
||||
spider['site_name'] = site['name']
|
||||
|
||||
# file stats
|
||||
stats = get_file_suffix_stats(dir_path)
|
||||
|
||||
# language
|
||||
lang = get_lang_by_stats(stats)
|
||||
|
||||
# spider type
|
||||
type_ = SpiderType.CUSTOMIZED
|
||||
|
||||
# update spider data
|
||||
db_manager.update_one('spiders', id=str(spider['_id']), values={
|
||||
'lang': lang,
|
||||
'type': type_,
|
||||
'suffix_stats': stats,
|
||||
})
|
||||
|
||||
# ---------
|
||||
# stats
|
||||
# ---------
|
||||
# last 5-run errors
|
||||
spider['last_5_errors'] = get_last_n_run_errors_count(spider_id=spider['_id'], n=5)
|
||||
spider['last_7d_tasks'] = get_last_n_day_tasks_count(spider_id=spider['_id'], n=5)
|
||||
|
||||
# append spider
|
||||
items.append(spider)
|
||||
|
||||
# get configurable spiders
|
||||
for spider in db_manager.list('spiders', {'type': SpiderType.CONFIGURABLE}):
|
||||
# append spider
|
||||
items.append(spider)
|
||||
|
||||
# get other info
|
||||
for i in range(len(items)):
|
||||
spider = items[i]
|
||||
|
||||
# get site
|
||||
if spider.get('site') is not None:
|
||||
site = db_manager.get('sites', spider['site'])
|
||||
if site is not None:
|
||||
items[i]['site_name'] = site['name']
|
||||
|
||||
# get last task
|
||||
last_task = db_manager.get_last_task(spider_id=spider['_id'])
|
||||
if last_task is not None:
|
||||
items[i]['task_ts'] = last_task['create_ts']
|
||||
|
||||
# ---------
|
||||
# stats
|
||||
# ---------
|
||||
# last 5-run errors
|
||||
items[i]['last_5_errors'] = get_last_n_run_errors_count(spider_id=spider['_id'], n=5)
|
||||
items[i]['last_7d_tasks'] = get_last_n_day_tasks_count(spider_id=spider['_id'], n=5)
|
||||
|
||||
return {
|
||||
'status': 'ok',
|
||||
'items': jsonify(items)
|
||||
@@ -214,7 +265,16 @@ class SpiderApi(BaseApi):
|
||||
|
||||
spider = db_manager.get('spiders', id=ObjectId(id))
|
||||
|
||||
job = execute_spider.delay(id, params)
|
||||
# determine execute function
|
||||
if spider['type'] == SpiderType.CONFIGURABLE:
|
||||
# configurable spider
|
||||
exec_func = execute_config_spider
|
||||
else:
|
||||
# customized spider
|
||||
exec_func = execute_spider
|
||||
|
||||
# trigger an asynchronous job
|
||||
job = exec_func.delay(id, params)
|
||||
|
||||
# create a new task
|
||||
db_manager.save('tasks', {
|
||||
@@ -377,10 +437,101 @@ class SpiderApi(BaseApi):
|
||||
scheduler.update()
|
||||
|
||||
def update_envs(self, id: str):
|
||||
"""
|
||||
Update environment variables
|
||||
:param id: spider_id
|
||||
"""
|
||||
args = self.parser.parse_args()
|
||||
envs = json.loads(args.envs)
|
||||
db_manager.update_one(col_name='spiders', id=id, values={'envs': envs})
|
||||
|
||||
def update_fields(self, id: str):
|
||||
"""
|
||||
Update list page fields variables for configurable spiders
|
||||
:param id: spider_id
|
||||
"""
|
||||
args = self.parser.parse_args()
|
||||
fields = json.loads(args.fields)
|
||||
db_manager.update_one(col_name='spiders', id=id, values={'fields': fields})
|
||||
|
||||
def update_detail_fields(self, id: str):
|
||||
"""
|
||||
Update detail page fields variables for configurable spiders
|
||||
:param id: spider_id
|
||||
"""
|
||||
args = self.parser.parse_args()
|
||||
detail_fields = json.loads(args.detail_fields)
|
||||
db_manager.update_one(col_name='spiders', id=id, values={'detail_fields': detail_fields})
|
||||
|
||||
def preview_crawl(self, id: str):
|
||||
spider = db_manager.get(col_name='spiders', id=id)
|
||||
|
||||
if spider['type'] != SpiderType.CONFIGURABLE:
|
||||
return {
|
||||
'status': 'ok',
|
||||
'error': 'type %s is invalid' % spider['type']
|
||||
}, 400
|
||||
|
||||
if spider.get('start_url') is None:
|
||||
return {
|
||||
'status': 'ok',
|
||||
'error': 'start_url should not be empty'
|
||||
}, 400
|
||||
|
||||
try:
|
||||
r = requests.get(spider['start_url'])
|
||||
except Exception as err:
|
||||
return {
|
||||
'status': 'ok',
|
||||
'error': 'connection error'
|
||||
}, 500
|
||||
|
||||
if r.status_code != 200:
|
||||
return {
|
||||
'status': 'ok',
|
||||
'error': 'status code is not 200, but %s' % r.status_code
|
||||
}
|
||||
|
||||
# get html parse tree
|
||||
sel = etree.HTML(r.content)
|
||||
|
||||
# parse fields
|
||||
if spider['crawl_type'] == CrawlType.LIST:
|
||||
if spider.get('item_selector') is None:
|
||||
return {
|
||||
'status': 'ok',
|
||||
'error': 'item_selector should not be empty'
|
||||
}, 400
|
||||
|
||||
data = get_list_page_data(spider, sel)[:10]
|
||||
|
||||
return {
|
||||
'status': 'ok',
|
||||
'items': data
|
||||
}
|
||||
|
||||
elif spider['crawl_type'] == CrawlType.DETAIL:
|
||||
pass
|
||||
|
||||
elif spider['crawl_type'] == CrawlType.LIST_DETAIL:
|
||||
data = get_list_page_data(spider, sel)[:10]
|
||||
|
||||
ev_list = []
|
||||
for idx, d in enumerate(data):
|
||||
for f in spider['fields']:
|
||||
if f.get('is_detail'):
|
||||
url = d.get(f['name'])
|
||||
if url is not None:
|
||||
ev_list.append(gevent.spawn(get_detail_page_data, url, spider, idx, data))
|
||||
break
|
||||
|
||||
gevent.joinall(ev_list)
|
||||
|
||||
return {
|
||||
'status': 'ok',
|
||||
'items': data
|
||||
}
|
||||
|
||||
|
||||
class SpiderImportApi(Resource):
|
||||
__doc__ = """
|
||||
|
||||
11
crawlab/spiders/scrapy.cfg
Normal file
11
crawlab/spiders/scrapy.cfg
Normal file
@@ -0,0 +1,11 @@
|
||||
# Automatically created by: scrapy startproject
|
||||
#
|
||||
# For more information about the [deploy] section see:
|
||||
# https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||
|
||||
[settings]
|
||||
default = spiders.settings
|
||||
|
||||
[deploy]
|
||||
#url = http://localhost:6800/
|
||||
project = spiders
|
||||
0
crawlab/spiders/spiders/__init__.py
Normal file
0
crawlab/spiders/spiders/__init__.py
Normal file
14
crawlab/spiders/spiders/db.py
Normal file
14
crawlab/spiders/spiders/db.py
Normal file
@@ -0,0 +1,14 @@
|
||||
import os
|
||||
|
||||
from pymongo import MongoClient
|
||||
|
||||
MONGO_HOST = os.environ.get('MONGO_HOST')
|
||||
MONGO_PORT = int(os.environ.get('MONGO_PORT'))
|
||||
MONGO_DB = os.environ.get('MONGO_DB')
|
||||
mongo = MongoClient(host=MONGO_HOST,
|
||||
port=MONGO_PORT)
|
||||
db = mongo[MONGO_DB]
|
||||
task_id = os.environ.get('CRAWLAB_TASK_ID')
|
||||
col_name = os.environ.get('CRAWLAB_COLLECTION')
|
||||
task = db['tasks'].find_one({'_id': task_id})
|
||||
spider = db['spiders'].find_one({'_id': task['spider_id']})
|
||||
25
crawlab/spiders/spiders/items.py
Normal file
25
crawlab/spiders/spiders/items.py
Normal file
@@ -0,0 +1,25 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define here the models for your scraped items
|
||||
#
|
||||
# See documentation in:
|
||||
# https://doc.scrapy.org/en/latest/topics/items.html
|
||||
|
||||
import scrapy
|
||||
|
||||
from spiders.db import spider
|
||||
|
||||
|
||||
class SpidersItem(scrapy.Item):
|
||||
if spider['crawl_type'] == 'list':
|
||||
fields = {f['name']: scrapy.Field() for f in spider['fields']}
|
||||
elif spider['crawl_type'] == 'detail':
|
||||
fields = {f['name']: scrapy.Field() for f in spider['detail_fields']}
|
||||
elif spider['crawl_type'] == 'list-detail':
|
||||
fields = {f['name']: scrapy.Field() for f in (spider['fields'] + spider['detail_fields'])}
|
||||
else:
|
||||
fields = {}
|
||||
|
||||
# basic fields
|
||||
fields['_id'] = scrapy.Field()
|
||||
fields['task_id'] = scrapy.Field()
|
||||
103
crawlab/spiders/spiders/middlewares.py
Normal file
103
crawlab/spiders/spiders/middlewares.py
Normal file
@@ -0,0 +1,103 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define here the models for your spider middleware
|
||||
#
|
||||
# See documentation in:
|
||||
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
|
||||
from scrapy import signals
|
||||
|
||||
|
||||
class SpidersSpiderMiddleware(object):
|
||||
# Not all methods need to be defined. If a method is not defined,
|
||||
# scrapy acts as if the spider middleware does not modify the
|
||||
# passed objects.
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
# This method is used by Scrapy to create your spiders.
|
||||
s = cls()
|
||||
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
||||
return s
|
||||
|
||||
def process_spider_input(self, response, spider):
|
||||
# Called for each response that goes through the spider
|
||||
# middleware and into the spider.
|
||||
|
||||
# Should return None or raise an exception.
|
||||
return None
|
||||
|
||||
def process_spider_output(self, response, result, spider):
|
||||
# Called with the results returned from the Spider, after
|
||||
# it has processed the response.
|
||||
|
||||
# Must return an iterable of Request, dict or Item objects.
|
||||
for i in result:
|
||||
yield i
|
||||
|
||||
def process_spider_exception(self, response, exception, spider):
|
||||
# Called when a spider or process_spider_input() method
|
||||
# (from other spider middleware) raises an exception.
|
||||
|
||||
# Should return either None or an iterable of Response, dict
|
||||
# or Item objects.
|
||||
pass
|
||||
|
||||
def process_start_requests(self, start_requests, spider):
|
||||
# Called with the start requests of the spider, and works
|
||||
# similarly to the process_spider_output() method, except
|
||||
# that it doesn’t have a response associated.
|
||||
|
||||
# Must return only requests (not items).
|
||||
for r in start_requests:
|
||||
yield r
|
||||
|
||||
def spider_opened(self, spider):
|
||||
spider.logger.info('Spider opened: %s' % spider.name)
|
||||
|
||||
|
||||
class SpidersDownloaderMiddleware(object):
|
||||
# Not all methods need to be defined. If a method is not defined,
|
||||
# scrapy acts as if the downloader middleware does not modify the
|
||||
# passed objects.
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
# This method is used by Scrapy to create your spiders.
|
||||
s = cls()
|
||||
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
||||
return s
|
||||
|
||||
def process_request(self, request, spider):
|
||||
# Called for each request that goes through the downloader
|
||||
# middleware.
|
||||
|
||||
# Must either:
|
||||
# - return None: continue processing this request
|
||||
# - or return a Response object
|
||||
# - or return a Request object
|
||||
# - or raise IgnoreRequest: process_exception() methods of
|
||||
# installed downloader middleware will be called
|
||||
return None
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
# Called with the response returned from the downloader.
|
||||
|
||||
# Must either;
|
||||
# - return a Response object
|
||||
# - return a Request object
|
||||
# - or raise IgnoreRequest
|
||||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
# Called when a download handler or a process_request()
|
||||
# (from other downloader middleware) raises an exception.
|
||||
|
||||
# Must either:
|
||||
# - return None: continue processing this exception
|
||||
# - return a Response object: stops process_exception() chain
|
||||
# - return a Request object: stops process_exception() chain
|
||||
pass
|
||||
|
||||
def spider_opened(self, spider):
|
||||
spider.logger.info('Spider opened: %s' % spider.name)
|
||||
17
crawlab/spiders/spiders/pipelines.py
Normal file
17
crawlab/spiders/spiders/pipelines.py
Normal file
@@ -0,0 +1,17 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define your item pipelines here
|
||||
#
|
||||
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
||||
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
from spiders.db import db, col_name, task_id
|
||||
|
||||
|
||||
class SpidersPipeline(object):
|
||||
col = db[col_name]
|
||||
|
||||
def process_item(self, item, spider):
|
||||
item['task_id'] = task_id
|
||||
self.col.save(item)
|
||||
|
||||
return item
|
||||
90
crawlab/spiders/spiders/settings.py
Normal file
90
crawlab/spiders/spiders/settings.py
Normal file
@@ -0,0 +1,90 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Scrapy settings for spiders project
|
||||
#
|
||||
# For simplicity, this file contains only settings considered important or
|
||||
# commonly used. You can find more settings consulting the documentation:
|
||||
#
|
||||
# https://doc.scrapy.org/en/latest/topics/settings.html
|
||||
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
from spiders.db import spider
|
||||
|
||||
BOT_NAME = 'Crawlab Spider'
|
||||
|
||||
SPIDER_MODULES = ['spiders.spiders']
|
||||
NEWSPIDER_MODULE = 'spiders.spiders'
|
||||
|
||||
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
||||
# USER_AGENT = 'spiders (+http://www.yourdomain.com)'
|
||||
|
||||
# Obey robots.txt rules
|
||||
ROBOTSTXT_OBEY = spider.get('obey_robots_txt') or True
|
||||
|
||||
# Configure maximum concurrent requests performed by Scrapy (default: 16)
|
||||
# CONCURRENT_REQUESTS = 32
|
||||
|
||||
# Configure a delay for requests for the same website (default: 0)
|
||||
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
|
||||
# See also autothrottle settings and docs
|
||||
# DOWNLOAD_DELAY = 3
|
||||
# The download delay setting will honor only one of:
|
||||
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
|
||||
# CONCURRENT_REQUESTS_PER_IP = 16
|
||||
|
||||
# Disable cookies (enabled by default)
|
||||
# COOKIES_ENABLED = False
|
||||
|
||||
# Disable Telnet Console (enabled by default)
|
||||
# TELNETCONSOLE_ENABLED = False
|
||||
|
||||
# Override the default request headers:
|
||||
# DEFAULT_REQUEST_HEADERS = {
|
||||
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
# 'Accept-Language': 'en',
|
||||
# }
|
||||
|
||||
# Enable or disable spider middlewares
|
||||
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
# SPIDER_MIDDLEWARES = {
|
||||
# 'spiders.middlewares.SpidersSpiderMiddleware': 543,
|
||||
# }
|
||||
|
||||
# Enable or disable downloader middlewares
|
||||
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
# DOWNLOADER_MIDDLEWARES = {
|
||||
# 'spiders.middlewares.SpidersDownloaderMiddleware': 543,
|
||||
# }
|
||||
|
||||
# Enable or disable extensions
|
||||
# See https://doc.scrapy.org/en/latest/topics/extensions.html
|
||||
# EXTENSIONS = {
|
||||
# 'scrapy.extensions.telnet.TelnetConsole': None,
|
||||
# }
|
||||
|
||||
# Configure item pipelines
|
||||
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
ITEM_PIPELINES = {
|
||||
'spiders.pipelines.SpidersPipeline': 300,
|
||||
}
|
||||
|
||||
# Enable and configure the AutoThrottle extension (disabled by default)
|
||||
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
|
||||
# AUTOTHROTTLE_ENABLED = True
|
||||
# The initial download delay
|
||||
# AUTOTHROTTLE_START_DELAY = 5
|
||||
# The maximum download delay to be set in case of high latencies
|
||||
# AUTOTHROTTLE_MAX_DELAY = 60
|
||||
# The average number of requests Scrapy should be sending in parallel to
|
||||
# each remote server
|
||||
# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
|
||||
# Enable showing throttling stats for every response received:
|
||||
# AUTOTHROTTLE_DEBUG = False
|
||||
|
||||
# Enable and configure HTTP caching (disabled by default)
|
||||
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
|
||||
# HTTPCACHE_ENABLED = True
|
||||
# HTTPCACHE_EXPIRATION_SECS = 0
|
||||
# HTTPCACHE_DIR = 'httpcache'
|
||||
# HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
|
||||
4
crawlab/spiders/spiders/spiders/__init__.py
Normal file
4
crawlab/spiders/spiders/spiders/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
# This package will contain the spiders of your Scrapy project
|
||||
#
|
||||
# Please refer to the documentation for information on how to create and manage
|
||||
# your spiders.
|
||||
116
crawlab/spiders/spiders/spiders/config_spider.py
Normal file
116
crawlab/spiders/spiders/spiders/config_spider.py
Normal file
@@ -0,0 +1,116 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import scrapy
|
||||
|
||||
from spiders.db import spider
|
||||
from spiders.items import SpidersItem
|
||||
|
||||
|
||||
def get_detail_url(item):
|
||||
for f in spider['fields']:
|
||||
if f.get('is_detail'):
|
||||
return item.get(f['name'])
|
||||
return None
|
||||
|
||||
|
||||
def get_spiders_item(sel, fields, item=None):
|
||||
if item is None:
|
||||
item = SpidersItem()
|
||||
|
||||
for f in fields:
|
||||
if f['type'] == 'xpath':
|
||||
# xpath selector
|
||||
if f['extract_type'] == 'text':
|
||||
# text content
|
||||
query = f['query'] + '/text()'
|
||||
else:
|
||||
# attribute
|
||||
attribute = f["attribute"]
|
||||
query = f['query'] + f'/@("{attribute}")'
|
||||
item[f['name']] = sel.xpath(query).extract_first()
|
||||
|
||||
else:
|
||||
# css selector
|
||||
if f['extract_type'] == 'text':
|
||||
# text content
|
||||
query = f['query'] + '::text'
|
||||
else:
|
||||
# attribute
|
||||
attribute = f["attribute"]
|
||||
query = f['query'] + f'::attr("{attribute}")'
|
||||
item[f['name']] = sel.css(query).extract_first()
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def get_list_items(response):
|
||||
if spider['item_selector_type'] == 'xpath':
|
||||
# xpath selector
|
||||
items = response.xpath(spider['item_selector'])
|
||||
else:
|
||||
# css selector
|
||||
items = response.css(spider['item_selector'])
|
||||
return items
|
||||
|
||||
|
||||
def get_next_url(response):
|
||||
# pagination
|
||||
if spider.get('pagination_selector') is not None:
|
||||
if spider['pagination_selector_type'] == 'xpath':
|
||||
# xpath selector
|
||||
next_url = response.xpath(spider['pagination_selector'] + '/@href').extract_first()
|
||||
else:
|
||||
# css selector
|
||||
next_url = response.css(spider['pagination_selector'] + '::attr("href")').extract_first()
|
||||
|
||||
# found next url
|
||||
if next_url is not None:
|
||||
if not next_url.startswith('http') and not next_url.startswith('//'):
|
||||
u = urlparse(response.url)
|
||||
next_url = f'{u.scheme}://{u.netloc}{next_url}'
|
||||
return next_url
|
||||
return None
|
||||
|
||||
|
||||
class ConfigSpiderSpider(scrapy.Spider):
|
||||
name = 'config_spider'
|
||||
# allowed_domains = []
|
||||
start_urls = [spider['start_url']]
|
||||
|
||||
def parse(self, response):
|
||||
|
||||
if spider['crawl_type'] == 'list':
|
||||
# list page only
|
||||
items = get_list_items(response)
|
||||
for _item in items:
|
||||
item = get_spiders_item(sel=_item, fields=spider['fields'])
|
||||
yield item
|
||||
next_url = get_next_url(response)
|
||||
if next_url is not None:
|
||||
yield scrapy.Request(url=next_url)
|
||||
|
||||
elif spider['crawl_type'] == 'detail':
|
||||
# TODO: detail page onlny
|
||||
# detail page only
|
||||
pass
|
||||
|
||||
elif spider['crawl_type'] == 'list-detail':
|
||||
# list page + detail page
|
||||
items = get_list_items(response)
|
||||
for _item in items:
|
||||
item = get_spiders_item(sel=_item, fields=spider['fields'])
|
||||
detail_url = get_detail_url(item)
|
||||
if detail_url is not None:
|
||||
yield scrapy.Request(url=detail_url,
|
||||
callback=self.parse_detail,
|
||||
meta={
|
||||
'item': item
|
||||
})
|
||||
next_url = get_next_url(response)
|
||||
if next_url is not None:
|
||||
yield scrapy.Request(url=next_url)
|
||||
|
||||
def parse_detail(self, response):
|
||||
item = get_spiders_item(sel=response, fields=spider['detail_fields'], item=response.meta['item'])
|
||||
yield item
|
||||
@@ -6,13 +6,15 @@ from time import sleep
|
||||
from bson import ObjectId
|
||||
from pymongo import ASCENDING, DESCENDING
|
||||
|
||||
from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_LOGS_FOLDER, PYTHON_ENV_PATH
|
||||
from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_LOGS_FOLDER, PYTHON_ENV_PATH, MONGO_HOST, MONGO_PORT, MONGO_DB
|
||||
from constants.task import TaskStatus
|
||||
from db.manager import db_manager
|
||||
from .celery import celery_app
|
||||
import subprocess
|
||||
from utils.log import other as logger
|
||||
|
||||
BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
|
||||
|
||||
|
||||
def get_task(id: str):
|
||||
i = 0
|
||||
@@ -112,6 +114,125 @@ def execute_spider(self, id: str, params: str = None):
|
||||
env=env,
|
||||
bufsize=1)
|
||||
|
||||
# update pid
|
||||
db_manager.update_one(col_name='tasks', id=task_id, values={
|
||||
'pid': p.pid
|
||||
})
|
||||
|
||||
# get output from the process
|
||||
_stdout, _stderr = p.communicate()
|
||||
|
||||
# get return code
|
||||
code = p.poll()
|
||||
if code == 0:
|
||||
status = TaskStatus.SUCCESS
|
||||
else:
|
||||
status = TaskStatus.FAILURE
|
||||
except Exception as err:
|
||||
logger.error(err)
|
||||
stderr.write(str(err))
|
||||
status = TaskStatus.FAILURE
|
||||
|
||||
# save task when the task is finished
|
||||
finish_ts = datetime.utcnow()
|
||||
db_manager.update_one('tasks', id=task_id, values={
|
||||
'finish_ts': finish_ts,
|
||||
'duration': (finish_ts - task['create_ts']).total_seconds(),
|
||||
'status': status
|
||||
})
|
||||
task = db_manager.get('tasks', id=id)
|
||||
|
||||
# close log file streams
|
||||
stdout.flush()
|
||||
stderr.flush()
|
||||
stdout.close()
|
||||
stderr.close()
|
||||
|
||||
return task
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def execute_config_spider(self, id: str, params: str = None):
|
||||
task_id = self.request.id
|
||||
hostname = self.request.hostname
|
||||
spider = db_manager.get('spiders', id=id)
|
||||
|
||||
# get task object and return if not found
|
||||
task = get_task(task_id)
|
||||
if task is None:
|
||||
return
|
||||
|
||||
# current working directory
|
||||
current_working_directory = os.path.join(BASE_DIR, 'spiders')
|
||||
|
||||
# log info
|
||||
logger.info('task_id: %s' % task_id)
|
||||
logger.info('hostname: %s' % hostname)
|
||||
logger.info('current_working_directory: %s' % current_working_directory)
|
||||
logger.info('spider_id: %s' % id)
|
||||
|
||||
# make sure the log folder exists
|
||||
log_path = os.path.join(PROJECT_LOGS_FOLDER, id)
|
||||
if not os.path.exists(log_path):
|
||||
os.makedirs(log_path)
|
||||
|
||||
# open log file streams
|
||||
log_file_path = os.path.join(log_path, '%s.log' % datetime.now().strftime('%Y%m%d%H%M%S'))
|
||||
stdout = open(log_file_path, 'a')
|
||||
stderr = open(log_file_path, 'a')
|
||||
|
||||
# update task status as started
|
||||
db_manager.update_one('tasks', id=task_id, values={
|
||||
'start_ts': datetime.utcnow(),
|
||||
'node_id': hostname,
|
||||
'hostname': hostname,
|
||||
'log_file_path': log_file_path,
|
||||
'status': TaskStatus.STARTED
|
||||
})
|
||||
|
||||
# pass params as env variables
|
||||
env = os.environ.copy()
|
||||
|
||||
# custom environment variables
|
||||
if spider.get('envs'):
|
||||
for _env in spider.get('envs'):
|
||||
env[_env['name']] = _env['value']
|
||||
|
||||
# task id environment variable
|
||||
env['CRAWLAB_TASK_ID'] = task_id
|
||||
|
||||
# collection environment variable
|
||||
if spider.get('col'):
|
||||
env['CRAWLAB_COLLECTION'] = spider.get('col')
|
||||
|
||||
# create index to speed results data retrieval
|
||||
db_manager.create_index(spider.get('col'), [('task_id', ASCENDING)])
|
||||
|
||||
# mongodb environment variables
|
||||
env['MONGO_HOST'] = MONGO_HOST
|
||||
env['MONGO_PORT'] = str(MONGO_PORT)
|
||||
env['MONGO_DB'] = MONGO_DB
|
||||
|
||||
cmd_arr = [
|
||||
sys.executable,
|
||||
'-m',
|
||||
'scrapy',
|
||||
'crawl',
|
||||
'config_spider'
|
||||
]
|
||||
try:
|
||||
p = subprocess.Popen(cmd_arr,
|
||||
stdout=stdout.fileno(),
|
||||
stderr=stderr.fileno(),
|
||||
cwd=current_working_directory,
|
||||
env=env,
|
||||
bufsize=1)
|
||||
|
||||
# update pid
|
||||
db_manager.update_one(col_name='tasks', id=task_id, values={
|
||||
'pid': p.pid
|
||||
})
|
||||
|
||||
# get output from the process
|
||||
_stdout, _stderr = p.communicate()
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import os
|
||||
import requests
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from bson import ObjectId
|
||||
from lxml import etree
|
||||
|
||||
from constants.spider import FILE_SUFFIX_LANG_MAPPING, LangType, SUFFIX_IGNORE, SpiderType
|
||||
from constants.spider import FILE_SUFFIX_LANG_MAPPING, LangType, SUFFIX_IGNORE, SpiderType, QueryType, ExtractType
|
||||
from constants.task import TaskStatus
|
||||
from db.manager import db_manager
|
||||
|
||||
@@ -69,3 +71,53 @@ def get_last_n_day_tasks_count(spider_id: ObjectId, n: int) -> list:
|
||||
'$gte': (datetime.now() - timedelta(n))
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
def get_list_page_data(spider, sel):
|
||||
data = []
|
||||
if spider['item_selector_type'] == QueryType.XPATH:
|
||||
items = sel.xpath(spider['item_selector'])
|
||||
else:
|
||||
items = sel.cssselect(spider['item_selector'])
|
||||
for item in items:
|
||||
row = {}
|
||||
for f in spider['fields']:
|
||||
if f['type'] == QueryType.CSS:
|
||||
# css selector
|
||||
res = item.cssselect(f['query'])
|
||||
else:
|
||||
# xpath
|
||||
res = item.xpath(f['query'])
|
||||
|
||||
if len(res) > 0:
|
||||
if f['extract_type'] == ExtractType.TEXT:
|
||||
row[f['name']] = res[0].text
|
||||
else:
|
||||
row[f['name']] = res[0].get(f['attribute'])
|
||||
data.append(row)
|
||||
return data
|
||||
|
||||
|
||||
def get_detail_page_data(url, spider, idx, data):
|
||||
r = requests.get(url)
|
||||
|
||||
sel = etree.HTML(r.content)
|
||||
|
||||
row = {}
|
||||
for f in spider['detail_fields']:
|
||||
if f['type'] == QueryType.CSS:
|
||||
# css selector
|
||||
res = sel.cssselect(f['query'])
|
||||
else:
|
||||
# xpath
|
||||
res = sel.xpath(f['query'])
|
||||
|
||||
if len(res) > 0:
|
||||
if f['extract_type'] == ExtractType.TEXT:
|
||||
row[f['name']] = res[0].text
|
||||
else:
|
||||
row[f['name']] = res[0].get(f['attribute'])
|
||||
|
||||
# assign values
|
||||
for k, v in row.items():
|
||||
data[idx][k] = v
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# App
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
# 部署
|
||||
|
||||
所有爬虫在运行前需要被部署当相应当节点中。
|
||||
|
||||
部署时，爬虫会被打包到相应的目录中，方便环境隔离，开发环境的爬虫和生产环境的爬虫需要打包部署来实现隔离。
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# Examples
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
# 安装
|
||||
|
||||
最快安装Crawlab的方式是克隆一份代码到本地
|
||||
|
||||
```bash
|
||||
git clone https://github.com/tikazyq/crawlab
|
||||
```
|
||||
|
||||
安装类库
|
||||
|
||||
```bash
|
||||
# 安装后台类库
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
```bash
|
||||
# 安装前台类库
|
||||
cd frontend
|
||||
npm install
|
||||
```
|
||||
|
||||
|
||||
@@ -1,436 +0,0 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>App · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="../Examples/" />
|
||||
|
||||
|
||||
<link rel="prev" href="Celery.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../QuickStart/">
|
||||
|
||||
<a href="../QuickStart/">
|
||||
|
||||
|
||||
快速开始
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../QuickStart/Installation.html">
|
||||
|
||||
<a href="../QuickStart/Installation.html">
|
||||
|
||||
|
||||
安装
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../QuickStart/Run.html">
|
||||
|
||||
<a href="../QuickStart/Run.html">
|
||||
|
||||
|
||||
运行
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../Concept/">
|
||||
|
||||
<a href="../Concept/">
|
||||
|
||||
|
||||
概念
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Concept/Node.html">
|
||||
|
||||
<a href="../Concept/Node.html">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Concept/Spider.html">
|
||||
|
||||
<a href="../Concept/Spider.html">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Concept/Task.html">
|
||||
|
||||
<a href="../Concept/Task.html">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Concept/Deploy.html">
|
||||
|
||||
<a href="../Concept/Deploy.html">
|
||||
|
||||
|
||||
部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="Celery.html">
|
||||
|
||||
<a href="Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.4.2" data-path="App.html">
|
||||
|
||||
<a href="App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >App</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="app">App</h1>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="Celery.html" class="navigation navigation-prev " aria-label="Previous page: Celery">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../Examples/" class="navigation navigation-next " aria-label="Next page: Examples">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"App","level":"1.4.2","depth":2,"next":{"title":"Examples","level":"1.5","depth":1,"path":"Examples/README.md","ref":"Examples/README.md","articles":[{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},{"title":"与Puppeteer集成","level":"1.5.2","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]}]},"previous":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/App.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -397,7 +397,7 @@
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"App","level":"1.4.2","depth":2,"next":{"title":"Examples","level":"1.5","depth":1,"path":"Examples/README.md","ref":"Examples/README.md","articles":[{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},{"title":"与Puppeteer集成","level":"1.5.2","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]}]},"previous":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/App.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"App","level":"1.4.2","depth":2,"next":{"title":"Examples","level":"1.5","depth":1,"path":"Examples/README.md","ref":"Examples/README.md","articles":[{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},{"title":"与Puppeteer集成","level":"1.5.2","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]}]},"previous":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/App.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
@@ -407,6 +407,10 @@
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-livereload/plugin.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
@@ -397,7 +397,7 @@
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"Celery","level":"1.4.1","depth":2,"next":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"previous":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/Celery.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"Celery","level":"1.4.1","depth":2,"next":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"previous":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/Celery.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
@@ -407,6 +407,10 @@
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-livereload/plugin.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
@@ -397,7 +397,7 @@
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"架构","level":"1.4","depth":1,"next":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"previous":{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"架构","level":"1.4","depth":1,"next":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"previous":{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
@@ -407,6 +407,10 @@
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-livereload/plugin.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
@@ -1,438 +0,0 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>部署 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="../Architecture/" />
|
||||
|
||||
|
||||
<link rel="prev" href="Task.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../QuickStart/">
|
||||
|
||||
<a href="../QuickStart/">
|
||||
|
||||
|
||||
快速开始
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../QuickStart/Installation.html">
|
||||
|
||||
<a href="../QuickStart/Installation.html">
|
||||
|
||||
|
||||
安装
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../QuickStart/Run.html">
|
||||
|
||||
<a href="../QuickStart/Run.html">
|
||||
|
||||
|
||||
运行
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
概念
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="Node.html">
|
||||
|
||||
<a href="Node.html">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="Spider.html">
|
||||
|
||||
<a href="Spider.html">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="Task.html">
|
||||
|
||||
<a href="Task.html">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.4" data-path="Deploy.html">
|
||||
|
||||
<a href="Deploy.html">
|
||||
|
||||
|
||||
部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
||||
|
||||
<a href="../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="../Architecture/Celery.html">
|
||||
|
||||
<a href="../Architecture/Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="../Architecture/App.html">
|
||||
|
||||
<a href="../Architecture/App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >部署</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="部署">部署</h1>
|
||||
<p>所有爬虫在运行前需要被部署当相应当节点中。</p>
|
||||
<p>部署时，爬虫会被打包到相应的目录中，方便环境隔离，开发环境的爬虫和生产环境的爬虫需要打包部署来实现隔离。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="Task.html" class="navigation navigation-prev " aria-label="Previous page: 任务">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../Architecture/" class="navigation navigation-next " aria-label="Next page: 架构">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"部署","level":"1.3.4","depth":2,"next":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"previous":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Deploy.md","mtime":"2019-03-28T12:06:24.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -399,7 +399,7 @@
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"部署","level":"1.3.4","depth":2,"next":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"previous":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Deploy.md","mtime":"2019-03-28T12:06:24.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"部署","level":"1.3.4","depth":2,"next":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"previous":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Deploy.md","mtime":"2019-03-28T12:06:24.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
@@ -409,6 +409,10 @@
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-livereload/plugin.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
@@ -398,7 +398,7 @@
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"节点","level":"1.3.1","depth":2,"next":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},"previous":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Node.md","mtime":"2019-03-28T12:02:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"节点","level":"1.3.1","depth":2,"next":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},"previous":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Node.md","mtime":"2019-03-28T12:02:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
@@ -408,6 +408,10 @@
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-livereload/plugin.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
@@ -403,7 +403,7 @@
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"爬虫","level":"1.3.2","depth":2,"next":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"previous":{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Spider.md","mtime":"2019-03-28T12:03:08.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"爬虫","level":"1.3.2","depth":2,"next":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"previous":{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Spider.md","mtime":"2019-03-28T12:03:08.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
@@ -413,6 +413,10 @@
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-livereload/plugin.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
@@ -398,7 +398,7 @@
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"任务","level":"1.3.3","depth":2,"next":{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]},"previous":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Task.md","mtime":"2019-03-28T12:04:12.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"任务","level":"1.3.3","depth":2,"next":{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]},"previous":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Task.md","mtime":"2019-03-28T12:04:12.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
@@ -408,6 +408,10 @@
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-livereload/plugin.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
@@ -397,7 +397,7 @@
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"概念","level":"1.3","depth":1,"next":{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},"previous":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/README.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"概念","level":"1.3","depth":1,"next":{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},"previous":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/README.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
@@ -407,6 +407,10 @@
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-livereload/plugin.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
@@ -1,436 +0,0 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>Examples · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="./" />
|
||||
|
||||
|
||||
<link rel="prev" href="../Architecture/App.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../QuickStart/">
|
||||
|
||||
<a href="../QuickStart/">
|
||||
|
||||
|
||||
快速开始
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../QuickStart/Installation.html">
|
||||
|
||||
<a href="../QuickStart/Installation.html">
|
||||
|
||||
|
||||
安装
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../QuickStart/Run.html">
|
||||
|
||||
<a href="../QuickStart/Run.html">
|
||||
|
||||
|
||||
运行
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../Concept/">
|
||||
|
||||
<a href="../Concept/">
|
||||
|
||||
|
||||
概念
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Concept/Node.html">
|
||||
|
||||
<a href="../Concept/Node.html">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Concept/Spider.html">
|
||||
|
||||
<a href="../Concept/Spider.html">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Concept/Task.html">
|
||||
|
||||
<a href="../Concept/Task.html">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Concept/Deploy.html">
|
||||
|
||||
<a href="../Concept/Deploy.html">
|
||||
|
||||
|
||||
部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
||||
|
||||
<a href="../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="../Architecture/Celery.html">
|
||||
|
||||
<a href="../Architecture/Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="../Architecture/App.html">
|
||||
|
||||
<a href="../Architecture/App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.5" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
Examples
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter active" data-level="1.5.1" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.5.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >Examples</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="examples">Examples</h1>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="../Architecture/App.html" class="navigation navigation-prev " aria-label="Previous page: App">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="./" class="navigation navigation-next " aria-label="Next page: 与Scrapy集成">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"Examples","level":"1.5","depth":1,"next":{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},"previous":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Examples/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -397,7 +397,7 @@
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"Examples","level":"1.5","depth":1,"next":{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},"previous":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Examples/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"Examples","level":"1.5","depth":1,"next":{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},"previous":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Examples/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
@@ -407,6 +407,10 @@
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-livereload/plugin.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
@@ -1,447 +0,0 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>安装 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Run.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="./" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
快速开始
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter active" data-level="1.2.1" data-path="Installation.html">
|
||||
|
||||
<a href="Installation.html">
|
||||
|
||||
|
||||
安装
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="Run.html">
|
||||
|
||||
<a href="Run.html">
|
||||
|
||||
|
||||
运行
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../Concept/">
|
||||
|
||||
<a href="../Concept/">
|
||||
|
||||
|
||||
概念
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Concept/Node.html">
|
||||
|
||||
<a href="../Concept/Node.html">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Concept/Spider.html">
|
||||
|
||||
<a href="../Concept/Spider.html">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Concept/Task.html">
|
||||
|
||||
<a href="../Concept/Task.html">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Concept/Deploy.html">
|
||||
|
||||
<a href="../Concept/Deploy.html">
|
||||
|
||||
|
||||
部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
||||
|
||||
<a href="../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="../Architecture/Celery.html">
|
||||
|
||||
<a href="../Architecture/Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="../Architecture/App.html">
|
||||
|
||||
<a href="../Architecture/App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >安装</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="安装">安装</h1>
|
||||
<p>最快安装Crawlab的方式是克隆一份代码到本地</p>
|
||||
<pre><code class="lang-bash">git <span class="hljs-built_in">clone</span> https://github.com/tikazyq/crawlab
|
||||
</code></pre>
|
||||
<p>安装类库</p>
|
||||
<pre><code class="lang-bash"><span class="hljs-comment"># 安装后台类库</span>
|
||||
pip install -r requirements.txt
|
||||
</code></pre>
|
||||
<pre><code class="lang-bash"><span class="hljs-comment"># 安装前台类库</span>
|
||||
<span class="hljs-built_in">cd</span> frontend
|
||||
npm install
|
||||
</code></pre>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="./" class="navigation navigation-prev " aria-label="Previous page: 快速开始">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Run.html" class="navigation navigation-next " aria-label="Next page: 运行">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"安装","level":"1.2.1","depth":2,"next":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"previous":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Installation.md","mtime":"2019-03-28T11:55:48.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -408,7 +408,7 @@ npm install
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"安装","level":"1.2.1","depth":2,"next":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"previous":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Installation.md","mtime":"2019-03-28T11:55:48.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"安装","level":"1.2.1","depth":2,"next":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"previous":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Installation.md","mtime":"2019-03-28T11:55:48.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
@@ -418,6 +418,10 @@ npm install
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-livereload/plugin.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
@@ -440,7 +440,7 @@ npm run serve
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"运行","level":"1.2.2","depth":2,"next":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"previous":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Run.md","mtime":"2019-03-28T12:00:11.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"运行","level":"1.2.2","depth":2,"next":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"previous":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Run.md","mtime":"2019-03-28T12:00:11.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
@@ -450,6 +450,10 @@ npm run serve
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-livereload/plugin.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
@@ -401,7 +401,7 @@
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"快速开始","level":"1.2","depth":1,"next":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"previous":{"title":"简介","level":"1.1","depth":1,"path":"README.md","ref":"README.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/README.md","mtime":"2019-03-28T12:02:02.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"快速开始","level":"1.2","depth":1,"next":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"previous":{"title":"简介","level":"1.1","depth":1,"path":"README.md","ref":"README.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/README.md","mtime":"2019-03-28T12:02:02.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
@@ -411,6 +411,10 @@
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-livereload/plugin.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
Binary file not shown.
@@ -1,240 +0,0 @@
|
||||
require(['gitbook', 'jquery'], function(gitbook, $) {
|
||||
// Configuration
|
||||
var MAX_SIZE = 4,
|
||||
MIN_SIZE = 0,
|
||||
BUTTON_ID;
|
||||
|
||||
// Current fontsettings state
|
||||
var fontState;
|
||||
|
||||
// Default themes
|
||||
var THEMES = [
|
||||
{
|
||||
config: 'white',
|
||||
text: 'White',
|
||||
id: 0
|
||||
},
|
||||
{
|
||||
config: 'sepia',
|
||||
text: 'Sepia',
|
||||
id: 1
|
||||
},
|
||||
{
|
||||
config: 'night',
|
||||
text: 'Night',
|
||||
id: 2
|
||||
}
|
||||
];
|
||||
|
||||
// Default font families
|
||||
var FAMILIES = [
|
||||
{
|
||||
config: 'serif',
|
||||
text: 'Serif',
|
||||
id: 0
|
||||
},
|
||||
{
|
||||
config: 'sans',
|
||||
text: 'Sans',
|
||||
id: 1
|
||||
}
|
||||
];
|
||||
|
||||
// Return configured themes
|
||||
function getThemes() {
|
||||
return THEMES;
|
||||
}
|
||||
|
||||
// Modify configured themes
|
||||
function setThemes(themes) {
|
||||
THEMES = themes;
|
||||
updateButtons();
|
||||
}
|
||||
|
||||
// Return configured font families
|
||||
function getFamilies() {
|
||||
return FAMILIES;
|
||||
}
|
||||
|
||||
// Modify configured font families
|
||||
function setFamilies(families) {
|
||||
FAMILIES = families;
|
||||
updateButtons();
|
||||
}
|
||||
|
||||
// Save current font settings
|
||||
function saveFontSettings() {
|
||||
gitbook.storage.set('fontState', fontState);
|
||||
update();
|
||||
}
|
||||
|
||||
// Increase font size
|
||||
function enlargeFontSize(e) {
|
||||
e.preventDefault();
|
||||
if (fontState.size >= MAX_SIZE) return;
|
||||
|
||||
fontState.size++;
|
||||
saveFontSettings();
|
||||
}
|
||||
|
||||
// Decrease font size
|
||||
function reduceFontSize(e) {
|
||||
e.preventDefault();
|
||||
if (fontState.size <= MIN_SIZE) return;
|
||||
|
||||
fontState.size--;
|
||||
saveFontSettings();
|
||||
}
|
||||
|
||||
// Change font family
|
||||
function changeFontFamily(configName, e) {
|
||||
if (e && e instanceof Event) {
|
||||
e.preventDefault();
|
||||
}
|
||||
|
||||
var familyId = getFontFamilyId(configName);
|
||||
fontState.family = familyId;
|
||||
saveFontSettings();
|
||||
}
|
||||
|
||||
// Change type of color theme
|
||||
function changeColorTheme(configName, e) {
|
||||
if (e && e instanceof Event) {
|
||||
e.preventDefault();
|
||||
}
|
||||
|
||||
var $book = gitbook.state.$book;
|
||||
|
||||
// Remove currently applied color theme
|
||||
if (fontState.theme !== 0)
|
||||
$book.removeClass('color-theme-'+fontState.theme);
|
||||
|
||||
// Set new color theme
|
||||
var themeId = getThemeId(configName);
|
||||
fontState.theme = themeId;
|
||||
if (fontState.theme !== 0)
|
||||
$book.addClass('color-theme-'+fontState.theme);
|
||||
|
||||
saveFontSettings();
|
||||
}
|
||||
|
||||
// Return the correct id for a font-family config key
|
||||
// Default to first font-family
|
||||
function getFontFamilyId(configName) {
|
||||
// Search for plugin configured font family
|
||||
var configFamily = $.grep(FAMILIES, function(family) {
|
||||
return family.config == configName;
|
||||
})[0];
|
||||
// Fallback to default font family
|
||||
return (!!configFamily)? configFamily.id : 0;
|
||||
}
|
||||
|
||||
// Return the correct id for a theme config key
|
||||
// Default to first theme
|
||||
function getThemeId(configName) {
|
||||
// Search for plugin configured theme
|
||||
var configTheme = $.grep(THEMES, function(theme) {
|
||||
return theme.config == configName;
|
||||
})[0];
|
||||
// Fallback to default theme
|
||||
return (!!configTheme)? configTheme.id : 0;
|
||||
}
|
||||
|
||||
function update() {
|
||||
var $book = gitbook.state.$book;
|
||||
|
||||
$('.font-settings .font-family-list li').removeClass('active');
|
||||
$('.font-settings .font-family-list li:nth-child('+(fontState.family+1)+')').addClass('active');
|
||||
|
||||
$book[0].className = $book[0].className.replace(/\bfont-\S+/g, '');
|
||||
$book.addClass('font-size-'+fontState.size);
|
||||
$book.addClass('font-family-'+fontState.family);
|
||||
|
||||
if(fontState.theme !== 0) {
|
||||
$book[0].className = $book[0].className.replace(/\bcolor-theme-\S+/g, '');
|
||||
$book.addClass('color-theme-'+fontState.theme);
|
||||
}
|
||||
}
|
||||
|
||||
function init(config) {
|
||||
// Search for plugin configured font family
|
||||
var configFamily = getFontFamilyId(config.family),
|
||||
configTheme = getThemeId(config.theme);
|
||||
|
||||
// Instantiate font state object
|
||||
fontState = gitbook.storage.get('fontState', {
|
||||
size: config.size || 2,
|
||||
family: configFamily,
|
||||
theme: configTheme
|
||||
});
|
||||
|
||||
update();
|
||||
}
|
||||
|
||||
function updateButtons() {
|
||||
// Remove existing fontsettings buttons
|
||||
if (!!BUTTON_ID) {
|
||||
gitbook.toolbar.removeButton(BUTTON_ID);
|
||||
}
|
||||
|
||||
// Create buttons in toolbar
|
||||
BUTTON_ID = gitbook.toolbar.createButton({
|
||||
icon: 'fa fa-font',
|
||||
label: 'Font Settings',
|
||||
className: 'font-settings',
|
||||
dropdown: [
|
||||
[
|
||||
{
|
||||
text: 'A',
|
||||
className: 'font-reduce',
|
||||
onClick: reduceFontSize
|
||||
},
|
||||
{
|
||||
text: 'A',
|
||||
className: 'font-enlarge',
|
||||
onClick: enlargeFontSize
|
||||
}
|
||||
],
|
||||
$.map(FAMILIES, function(family) {
|
||||
family.onClick = function(e) {
|
||||
return changeFontFamily(family.config, e);
|
||||
};
|
||||
|
||||
return family;
|
||||
}),
|
||||
$.map(THEMES, function(theme) {
|
||||
theme.onClick = function(e) {
|
||||
return changeColorTheme(theme.config, e);
|
||||
};
|
||||
|
||||
return theme;
|
||||
})
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
// Init configuration at start
|
||||
gitbook.events.bind('start', function(e, config) {
|
||||
var opts = config.fontsettings;
|
||||
|
||||
// Generate buttons at start
|
||||
updateButtons();
|
||||
|
||||
// Init current settings
|
||||
init(opts);
|
||||
});
|
||||
|
||||
// Expose API
|
||||
gitbook.fontsettings = {
|
||||
enlargeFontSize: enlargeFontSize,
|
||||
reduceFontSize: reduceFontSize,
|
||||
setTheme: changeColorTheme,
|
||||
setFamily: changeFontFamily,
|
||||
getThemes: getThemes,
|
||||
setThemes: setThemes,
|
||||
getFamilies: getFamilies,
|
||||
setFamilies: setFamilies
|
||||
};
|
||||
});
|
||||
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
pre,
|
||||
code {
|
||||
/* http://jmblog.github.io/color-themes-for-highlightjs */
|
||||
/* Tomorrow Comment */
|
||||
/* Tomorrow Red */
|
||||
/* Tomorrow Orange */
|
||||
/* Tomorrow Yellow */
|
||||
/* Tomorrow Green */
|
||||
/* Tomorrow Aqua */
|
||||
/* Tomorrow Blue */
|
||||
/* Tomorrow Purple */
|
||||
}
|
||||
pre .hljs-comment,
|
||||
code .hljs-comment,
|
||||
pre .hljs-title,
|
||||
code .hljs-title {
|
||||
color: #8e908c;
|
||||
}
|
||||
pre .hljs-variable,
|
||||
code .hljs-variable,
|
||||
pre .hljs-attribute,
|
||||
code .hljs-attribute,
|
||||
pre .hljs-tag,
|
||||
code .hljs-tag,
|
||||
pre .hljs-regexp,
|
||||
code .hljs-regexp,
|
||||
pre .hljs-deletion,
|
||||
code .hljs-deletion,
|
||||
pre .ruby .hljs-constant,
|
||||
code .ruby .hljs-constant,
|
||||
pre .xml .hljs-tag .hljs-title,
|
||||
code .xml .hljs-tag .hljs-title,
|
||||
pre .xml .hljs-pi,
|
||||
code .xml .hljs-pi,
|
||||
pre .xml .hljs-doctype,
|
||||
code .xml .hljs-doctype,
|
||||
pre .html .hljs-doctype,
|
||||
code .html .hljs-doctype,
|
||||
pre .css .hljs-id,
|
||||
code .css .hljs-id,
|
||||
pre .css .hljs-class,
|
||||
code .css .hljs-class,
|
||||
pre .css .hljs-pseudo,
|
||||
code .css .hljs-pseudo {
|
||||
color: #c82829;
|
||||
}
|
||||
pre .hljs-number,
|
||||
code .hljs-number,
|
||||
pre .hljs-preprocessor,
|
||||
code .hljs-preprocessor,
|
||||
pre .hljs-pragma,
|
||||
code .hljs-pragma,
|
||||
pre .hljs-built_in,
|
||||
code .hljs-built_in,
|
||||
pre .hljs-literal,
|
||||
code .hljs-literal,
|
||||
pre .hljs-params,
|
||||
code .hljs-params,
|
||||
pre .hljs-constant,
|
||||
code .hljs-constant {
|
||||
color: #f5871f;
|
||||
}
|
||||
pre .ruby .hljs-class .hljs-title,
|
||||
code .ruby .hljs-class .hljs-title,
|
||||
pre .css .hljs-rules .hljs-attribute,
|
||||
code .css .hljs-rules .hljs-attribute {
|
||||
color: #eab700;
|
||||
}
|
||||
pre .hljs-string,
|
||||
code .hljs-string,
|
||||
pre .hljs-value,
|
||||
code .hljs-value,
|
||||
pre .hljs-inheritance,
|
||||
code .hljs-inheritance,
|
||||
pre .hljs-header,
|
||||
code .hljs-header,
|
||||
pre .hljs-addition,
|
||||
code .hljs-addition,
|
||||
pre .ruby .hljs-symbol,
|
||||
code .ruby .hljs-symbol,
|
||||
pre .xml .hljs-cdata,
|
||||
code .xml .hljs-cdata {
|
||||
color: #718c00;
|
||||
}
|
||||
pre .css .hljs-hexcolor,
|
||||
code .css .hljs-hexcolor {
|
||||
color: #3e999f;
|
||||
}
|
||||
pre .hljs-function,
|
||||
code .hljs-function,
|
||||
pre .python .hljs-decorator,
|
||||
code .python .hljs-decorator,
|
||||
pre .python .hljs-title,
|
||||
code .python .hljs-title,
|
||||
pre .ruby .hljs-function .hljs-title,
|
||||
code .ruby .hljs-function .hljs-title,
|
||||
pre .ruby .hljs-title .hljs-keyword,
|
||||
code .ruby .hljs-title .hljs-keyword,
|
||||
pre .perl .hljs-sub,
|
||||
code .perl .hljs-sub,
|
||||
pre .javascript .hljs-title,
|
||||
code .javascript .hljs-title,
|
||||
pre .coffeescript .hljs-title,
|
||||
code .coffeescript .hljs-title {
|
||||
color: #4271ae;
|
||||
}
|
||||
pre .hljs-keyword,
|
||||
code .hljs-keyword,
|
||||
pre .javascript .hljs-function,
|
||||
code .javascript .hljs-function {
|
||||
color: #8959a8;
|
||||
}
|
||||
pre .hljs,
|
||||
code .hljs {
|
||||
display: block;
|
||||
background: white;
|
||||
color: #4d4d4c;
|
||||
padding: 0.5em;
|
||||
}
|
||||
pre .coffeescript .javascript,
|
||||
code .coffeescript .javascript,
|
||||
pre .javascript .xml,
|
||||
code .javascript .xml,
|
||||
pre .tex .hljs-formula,
|
||||
code .tex .hljs-formula,
|
||||
pre .xml .javascript,
|
||||
code .xml .javascript,
|
||||
pre .xml .vbscript,
|
||||
code .xml .vbscript,
|
||||
pre .xml .css,
|
||||
code .xml .css,
|
||||
pre .xml .hljs-cdata,
|
||||
code .xml .hljs-cdata {
|
||||
opacity: 0.5;
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1,90 +0,0 @@
|
||||
require(['gitbook', 'jquery'], function(gitbook, $) {
|
||||
var SITES = {
|
||||
'facebook': {
|
||||
'label': 'Facebook',
|
||||
'icon': 'fa fa-facebook',
|
||||
'onClick': function(e) {
|
||||
e.preventDefault();
|
||||
window.open('http://www.facebook.com/sharer/sharer.php?s=100&p[url]='+encodeURIComponent(location.href));
|
||||
}
|
||||
},
|
||||
'twitter': {
|
||||
'label': 'Twitter',
|
||||
'icon': 'fa fa-twitter',
|
||||
'onClick': function(e) {
|
||||
e.preventDefault();
|
||||
window.open('http://twitter.com/home?status='+encodeURIComponent(document.title+' '+location.href));
|
||||
}
|
||||
},
|
||||
'google': {
|
||||
'label': 'Google+',
|
||||
'icon': 'fa fa-google-plus',
|
||||
'onClick': function(e) {
|
||||
e.preventDefault();
|
||||
window.open('https://plus.google.com/share?url='+encodeURIComponent(location.href));
|
||||
}
|
||||
},
|
||||
'weibo': {
|
||||
'label': 'Weibo',
|
||||
'icon': 'fa fa-weibo',
|
||||
'onClick': function(e) {
|
||||
e.preventDefault();
|
||||
window.open('http://service.weibo.com/share/share.php?content=utf-8&url='+encodeURIComponent(location.href)+'&title='+encodeURIComponent(document.title));
|
||||
}
|
||||
},
|
||||
'instapaper': {
|
||||
'label': 'Instapaper',
|
||||
'icon': 'fa fa-instapaper',
|
||||
'onClick': function(e) {
|
||||
e.preventDefault();
|
||||
window.open('http://www.instapaper.com/text?u='+encodeURIComponent(location.href));
|
||||
}
|
||||
},
|
||||
'vk': {
|
||||
'label': 'VK',
|
||||
'icon': 'fa fa-vk',
|
||||
'onClick': function(e) {
|
||||
e.preventDefault();
|
||||
window.open('http://vkontakte.ru/share.php?url='+encodeURIComponent(location.href));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
gitbook.events.bind('start', function(e, config) {
|
||||
var opts = config.sharing;
|
||||
|
||||
// Create dropdown menu
|
||||
var menu = $.map(opts.all, function(id) {
|
||||
var site = SITES[id];
|
||||
|
||||
return {
|
||||
text: site.label,
|
||||
onClick: site.onClick
|
||||
};
|
||||
});
|
||||
|
||||
// Create main button with dropdown
|
||||
if (menu.length > 0) {
|
||||
gitbook.toolbar.createButton({
|
||||
icon: 'fa fa-share-alt',
|
||||
label: 'Share',
|
||||
position: 'right',
|
||||
dropdown: [menu]
|
||||
});
|
||||
}
|
||||
|
||||
// Direct actions to share
|
||||
$.each(SITES, function(sideId, site) {
|
||||
if (!opts[sideId]) return;
|
||||
|
||||
gitbook.toolbar.createButton({
|
||||
icon: site.icon,
|
||||
label: site.text,
|
||||
position: 'right',
|
||||
onClick: site.onClick
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 4.7 KiB |
@@ -548,7 +548,7 @@ MONGO_DB = <span class="hljs-string">'crawlab_test'</span>
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"简介","level":"1.1","depth":1,"next":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"README.md","mtime":"2019-03-28T11:44:15.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T12:07:05.349Z"},"basePath":".","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"简介","level":"1.1","depth":1,"next":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":["livereload"],"pluginsConfig":{"livereload":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"README.md","mtime":"2019-03-28T11:44:15.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-16T11:37:08.921Z"},"basePath":".","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
@@ -558,6 +558,10 @@ MONGO_DB = <span class="hljs-string">'crawlab_test'</span>
|
||||
<script src="gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="gitbook/gitbook-plugin-livereload/plugin.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 90 KiB |
@@ -1,27 +0,0 @@
|
||||
import request from '@/utils/request'
|
||||
|
||||
export function login (username, password) {
|
||||
return request({
|
||||
url: '/user/login',
|
||||
method: 'post',
|
||||
data: {
|
||||
username,
|
||||
password
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export function getInfo (token) {
|
||||
return request({
|
||||
url: '/user/info',
|
||||
method: 'get',
|
||||
params: { token }
|
||||
})
|
||||
}
|
||||
|
||||
export function logout () {
|
||||
return request({
|
||||
url: '/user/logout',
|
||||
method: 'post'
|
||||
})
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
import request from '@/utils/request'
|
||||
|
||||
export function getList (params) {
|
||||
return request({
|
||||
url: '/table/list',
|
||||
method: 'get',
|
||||
params
|
||||
})
|
||||
}
|
||||
275
frontend/src/components/Config/ConfigList.vue
Normal file
275
frontend/src/components/Config/ConfigList.vue
Normal file
@@ -0,0 +1,275 @@
|
||||
<template>
|
||||
<div class="config-list">
|
||||
<!--preview results-->
|
||||
<el-dialog :visible.sync="dialogVisible"
|
||||
:title="$t('Preview Results')"
|
||||
width="90%"
|
||||
:before-close="onDialogClose">
|
||||
<el-table :data="previewCrawlData"
|
||||
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
|
||||
border>
|
||||
<el-table-column v-for="(f, index) in fields"
|
||||
:label="f.name"
|
||||
:key="index"
|
||||
min-width="100px">
|
||||
<template slot-scope="scope">
|
||||
{{scope.row[f.name]}}
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
</el-dialog>
|
||||
<!--./preview results-->
|
||||
|
||||
<!--config detail-->
|
||||
<el-row>
|
||||
<el-col :span="11" :offset="1">
|
||||
<el-form label-width="150px">
|
||||
<el-form-item :label="$t('Crawl Type')">
|
||||
<el-button-group>
|
||||
<el-button v-for="type in crawlTypeList"
|
||||
:key="type.value"
|
||||
:type="type.value === spiderForm.crawl_type ? 'primary' : ''"
|
||||
@click="onSelectCrawlType(type.value)">
|
||||
{{$t(type.label)}}
|
||||
</el-button>
|
||||
</el-button-group>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Start URL')">
|
||||
<el-input v-model="spiderForm.start_url" :placeholder="$t('Start URL')"></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Obey robots.txt')">
|
||||
<el-switch v-model="spiderForm.obey_robots_txt" :placeholder="$t('Obey robots.txt')"></el-switch>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
</el-col>
|
||||
<el-col :span="11" :offset="1">
|
||||
<el-form label-width="150px">
|
||||
<el-form-item :label="$t('Item Selector')"
|
||||
v-if="['list','list-detail'].includes(spiderForm.crawl_type)">
|
||||
<el-select style="width: 35%;margin-right: 10px;"
|
||||
v-model="spiderForm.item_selector_type"
|
||||
:placeholder="$t('Item Selector Type')">
|
||||
<el-option value="xpath" :label="$t('XPath')"></el-option>
|
||||
<el-option value="css" :label="$t('CSS')"></el-option>
|
||||
</el-select>
|
||||
<el-input style="width: calc(65% - 10px);"
|
||||
v-model="spiderForm.item_selector"
|
||||
:placeholder="$t('Item Selector')">
|
||||
</el-input>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Pagination Selector')"
|
||||
v-if="['list','list-detail'].includes(spiderForm.crawl_type)">
|
||||
<el-select style="width: 35%;margin-right: 10px;"
|
||||
v-model="spiderForm.pagination_selector_type"
|
||||
:placeholder="$t('Pagination Selector Type')">
|
||||
<el-option value="xpath" :label="$t('XPath')"></el-option>
|
||||
<el-option value="css" :label="$t('CSS')"></el-option>
|
||||
</el-select>
|
||||
<el-input style="width: calc(65% - 10px);"
|
||||
v-model="spiderForm.pagination_selector"
|
||||
:placeholder="$t('Pagination Selector')">
|
||||
</el-input>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
</el-col>
|
||||
</el-row>
|
||||
<!--./config detail-->
|
||||
|
||||
<!--button group-->
|
||||
<el-row class="button-group-container">
|
||||
<div class="button-group">
|
||||
<el-button type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
|
||||
<el-button type="warning" @click="onPreview" v-loading="previewLoading">{{$t('Preview')}}</el-button>
|
||||
<el-button type="success" @click="onSave" v-loading="saveLoading">{{$t('Save')}}</el-button>
|
||||
</div>
|
||||
</el-row>
|
||||
<!--./button group-->
|
||||
|
||||
<!--list field list-->
|
||||
<el-row v-if="['list','list-detail'].includes(spiderForm.crawl_type)"
|
||||
class="list-fields-container">
|
||||
<fields-table-view
|
||||
type="list"
|
||||
title="List Page Fields"
|
||||
:fields="spiderForm.fields"
|
||||
/>
|
||||
</el-row>
|
||||
<!--./list field list-->
|
||||
|
||||
<!--detail field list-->
|
||||
<el-row v-if="['detail','list-detail'].includes(spiderForm.crawl_type)"
|
||||
class="detail-fields-container"
|
||||
style="margin-top: 10px;">
|
||||
<fields-table-view
|
||||
type="detail"
|
||||
title="Detail Page Fields"
|
||||
:fields="spiderForm.detail_fields"
|
||||
/>
|
||||
</el-row>
|
||||
<!--./detail field list-->
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
import {
|
||||
mapState
|
||||
} from 'vuex'
|
||||
import FieldsTableView from '../TableView/FieldsTableView'
|
||||
|
||||
export default {
|
||||
name: 'ConfigList',
|
||||
components: { FieldsTableView },
|
||||
data () {
|
||||
return {
|
||||
crawlTypeList: [
|
||||
{ value: 'list', label: 'List Only' },
|
||||
{ value: 'detail', label: 'Detail Only' },
|
||||
{ value: 'list-detail', label: 'List + Detail' }
|
||||
],
|
||||
previewLoading: false,
|
||||
saveLoading: false,
|
||||
dialogVisible: false
|
||||
}
|
||||
},
|
||||
computed: {
|
||||
...mapState('spider', [
|
||||
'spiderForm',
|
||||
'previewCrawlData'
|
||||
]),
|
||||
fields () {
|
||||
if (this.spiderForm.crawl_type === 'list') {
|
||||
return this.spiderForm.fields
|
||||
} else if (this.spiderForm.crawl_type === 'detail') {
|
||||
return this.spiderForm.detail_fields
|
||||
} else if (this.spiderForm.crawl_type === 'list-detail') {
|
||||
return this.spiderForm.fields.concat(this.spiderForm.detail_fields)
|
||||
} else {
|
||||
return []
|
||||
}
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
onSelectCrawlType (value) {
|
||||
this.spiderForm.crawl_type = value
|
||||
},
|
||||
onSave () {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.saveLoading = true
|
||||
this.$store.dispatch('spider/updateSpiderFields')
|
||||
.then(() => {
|
||||
this.$store.dispatch('spider/editSpider')
|
||||
.then(() => {
|
||||
this.$message.success(this.$t('Spider info has been saved successfully'))
|
||||
resolve()
|
||||
})
|
||||
.catch(() => {
|
||||
this.$message.error(this.$t('Something wrong happened'))
|
||||
reject(new Error())
|
||||
})
|
||||
.finally(() => {
|
||||
this.saveLoading = false
|
||||
})
|
||||
})
|
||||
.then(() => {
|
||||
this.$store.dispatch('spider/updateSpiderDetailFields')
|
||||
})
|
||||
.catch(() => {
|
||||
this.$message.error(this.$t('Something wrong happened'))
|
||||
this.saveLoading = false
|
||||
reject(new Error())
|
||||
})
|
||||
})
|
||||
},
|
||||
onDialogClose () {
|
||||
this.dialogVisible = false
|
||||
},
|
||||
onPreview () {
|
||||
this.onSave()
|
||||
.then(() => {
|
||||
this.previewLoading = true
|
||||
this.$store.dispatch('spider/getPreviewCrawlData')
|
||||
.then(() => {
|
||||
this.dialogVisible = true
|
||||
})
|
||||
.catch(() => {
|
||||
this.$message.error(this.$t('Something wrong happened'))
|
||||
})
|
||||
.finally(() => {
|
||||
this.previewLoading = false
|
||||
})
|
||||
})
|
||||
},
|
||||
onCrawl () {
|
||||
this.$confirm(this.$t('Are you sure to run this spider?'), this.$t('Notification'), {
|
||||
confirmButtonText: this.$t('Confirm'),
|
||||
cancelButtonText: this.$t('Cancel')
|
||||
})
|
||||
.then(() => {
|
||||
this.$store.dispatch('spider/crawlSpider', this.spiderForm._id)
|
||||
.then(() => {
|
||||
this.$message.success(this.$t(`Spider task has been scheduled`))
|
||||
})
|
||||
})
|
||||
}
|
||||
},
|
||||
created () {
|
||||
// fields for list page
|
||||
if (!this.spiderForm.fields) {
|
||||
this.spiderForm.fields = []
|
||||
for (let i = 0; i < 3; i++) {
|
||||
this.spiderForm.fields.push({
|
||||
name: `field_${i + 1}`,
|
||||
type: 'css',
|
||||
extract_type: 'text'
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// fields for detail page
|
||||
if (!this.spiderForm.detail_fields) {
|
||||
this.spiderForm.detail_fields = []
|
||||
for (let i = 0; i < 3; i++) {
|
||||
this.spiderForm.detail_fields.push({
|
||||
name: `field_${i + 1}`,
|
||||
type: 'css',
|
||||
extract_type: 'text'
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.spiderForm.crawl_type) this.$set(this.spiderForm, 'crawl_type', 'list')
|
||||
if (!this.spiderForm.start_url) this.$set(this.spiderForm, 'start_url', 'http://example.com')
|
||||
if (!this.spiderForm.item_selector_type) this.$set(this.spiderForm, 'item_selector_type', 'css')
|
||||
if (!this.spiderForm.pagination_selector_type) this.$set(this.spiderForm, 'pagination_selector_type', 'css')
|
||||
if (!this.spiderForm.obey_robots_txt) this.$set(this.spiderForm, 'obey_robots_txt', true)
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
|
||||
.button-group-container {
|
||||
margin-top: 10px;
|
||||
border-bottom: 1px dashed #dcdfe6;
|
||||
padding-bottom: 20px;
|
||||
}
|
||||
|
||||
.button-group {
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.list-fields-container {
|
||||
margin-top: 20px;
|
||||
border-bottom: 1px dashed #dcdfe6;
|
||||
padding-bottom: 20px;
|
||||
}
|
||||
|
||||
.detail-fields-container {
|
||||
margin-top: 20px;
|
||||
}
|
||||
|
||||
.title {
|
||||
color: #606266;
|
||||
font-size: 14px;
|
||||
}
|
||||
</style>
|
||||
@@ -12,10 +12,10 @@
|
||||
<el-form-item :label="$t('Spider Name')">
|
||||
<el-input v-model="spiderForm.name" :placeholder="$t('Spider Name')" :disabled="isView"></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Source Folder')">
|
||||
<el-form-item v-if="isCustomized" :label="$t('Source Folder')">
|
||||
<el-input v-model="spiderForm.src" :placeholder="$t('Source Folder')" disabled></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Execute Command')" prop="cmd" required :inline-message="true">
|
||||
<el-form-item v-if="isCustomized" :label="$t('Execute Command')" prop="cmd" required :inline-message="true">
|
||||
<el-input v-model="spiderForm.cmd" :placeholder="$t('Execute Command')"
|
||||
:disabled="isView"></el-input>
|
||||
</el-form-item>
|
||||
@@ -32,13 +32,12 @@
|
||||
</el-autocomplete>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Spider Type')">
|
||||
<el-select v-model="spiderForm.type" :placeholder="$t('Spider Type')" :disabled="isView" clearable>
|
||||
<el-option value="scrapy" label="Scrapy"></el-option>
|
||||
<el-option value="pyspider" label="PySpider"></el-option>
|
||||
<el-option value="webmagic" label="WebMagic"></el-option>
|
||||
<el-select v-model="spiderForm.type" :placeholder="$t('Spider Type')" :disabled="true" clearable>
|
||||
<el-option value="configurable" :label="$t('Configurable')"></el-option>
|
||||
<el-option value="customized" :label="$t('Customized')"></el-option>
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Language')">
|
||||
<el-form-item v-if="isCustomized" :label="$t('Language')">
|
||||
<el-select v-model="spiderForm.lang" :placeholder="$t('Language')" :disabled="isView" clearable>
|
||||
<el-option value="python" label="Python"></el-option>
|
||||
<el-option value="javascript" label="JavaScript"></el-option>
|
||||
@@ -49,8 +48,8 @@
|
||||
</el-form>
|
||||
</el-row>
|
||||
<el-row class="button-container" v-if="!isView">
|
||||
<el-button v-if="isShowRun" type="danger" @click="onRun">{{$t('Run')}}</el-button>
|
||||
<el-button type="primary" @click="onDeploy">{{$t('Deploy')}}</el-button>
|
||||
<el-button v-if="isShowRun" type="danger" @click="onCrawl">{{$t('Run')}}</el-button>
|
||||
<el-button v-if="isCustomized" type="primary" @click="onDeploy">{{$t('Deploy')}}</el-button>
|
||||
<el-button type="success" @click="onSave">{{$t('Save')}}</el-button>
|
||||
</el-row>
|
||||
</div>
|
||||
@@ -99,17 +98,23 @@ export default {
|
||||
'spiderForm'
|
||||
]),
|
||||
isShowRun () {
|
||||
if (!this.spiderForm.deploy_ts) {
|
||||
return false
|
||||
if (this.isCustomized) {
|
||||
// customized spider
|
||||
if (!this.spiderForm.deploy_ts) {
|
||||
return false
|
||||
}
|
||||
return !!this.spiderForm.cmd
|
||||
} else {
|
||||
// configurable spider
|
||||
return !!this.spiderForm.fields
|
||||
}
|
||||
if (!this.spiderForm.cmd) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
},
|
||||
isCustomized () {
|
||||
return this.spiderForm.type === 'customized'
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
onRun () {
|
||||
onCrawl () {
|
||||
const row = this.spiderForm
|
||||
this.$refs['spiderForm'].validate(res => {
|
||||
if (res) {
|
||||
|
||||
179
frontend/src/components/TableView/FieldsTableView.vue
Normal file
179
frontend/src/components/TableView/FieldsTableView.vue
Normal file
@@ -0,0 +1,179 @@
|
||||
<template>
|
||||
<div class="fields-table-view">
|
||||
<el-row class="button-group-container">
|
||||
<label class="title">{{$t(this.title)}}</label>
|
||||
<div class="button-group">
|
||||
<el-button type="primary" size="small" @click="addField" icon="el-icon-plus">{{$t('Add Field')}}</el-button>
|
||||
</div>
|
||||
</el-row>
|
||||
<el-row>
|
||||
<el-table :data="fields"
|
||||
class="table edit"
|
||||
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
|
||||
border>
|
||||
<el-table-column v-if="type === 'list' && spiderForm.crawl_type === 'list-detail'"
|
||||
:label="$t('Detail Page URL')"
|
||||
align="center">
|
||||
<template slot-scope="scope">
|
||||
<el-checkbox v-model="scope.row.is_detail"
|
||||
@change="onCheck(scope.row)">
|
||||
</el-checkbox>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column :label="$t('Field Name')" width="200px">
|
||||
<template slot-scope="scope">
|
||||
<el-input v-model="scope.row.name" :placeholder="$t('Field Name')"
|
||||
@change="onNameChange(scope.row)"></el-input>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column :label="$t('Query Type')" width="200px">
|
||||
<template slot-scope="scope">
|
||||
<el-select v-model="scope.row.type" :placeholder="$t('Query Type')">
|
||||
<el-option value="css" :label="$t('CSS Selector')"></el-option>
|
||||
<el-option value="xpath" :label="$t('XPath')"></el-option>
|
||||
</el-select>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column :label="$t('Query')" width="250px">
|
||||
<template slot-scope="scope">
|
||||
<el-input v-model="scope.row.query" :placeholder="$t('Query')"></el-input>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column :label="$t('Extract Type')" width="120px">
|
||||
<template slot-scope="scope">
|
||||
<el-select v-model="scope.row.extract_type" :placeholder="$t('Extract Type')">
|
||||
<el-option value="text" :label="$t('Text')"></el-option>
|
||||
<el-option value="attribute" :label="$t('Attribute')"></el-option>
|
||||
</el-select>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column :label="$t('Attribute')" width="250px">
|
||||
<template slot-scope="scope">
|
||||
<template v-if="scope.row.extract_type === 'attribute'">
|
||||
<el-input v-model="scope.row.attribute"
|
||||
:placeholder="$t('Attribute')">
|
||||
</el-input>
|
||||
</template>
|
||||
<template v-else>
|
||||
</template>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column :label="$t('Action')" fixed="right" min-width="100px">
|
||||
<template slot-scope="scope">
|
||||
<div class="action-button-group">
|
||||
<el-button size="mini"
|
||||
style="margin-left:10px"
|
||||
icon="el-icon-delete"
|
||||
type="danger"
|
||||
@click="deleteField(scope.$index)">
|
||||
</el-button>
|
||||
</div>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
</el-row>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
import {
|
||||
mapState
|
||||
} from 'vuex'
|
||||
|
||||
export default {
|
||||
name: 'FieldsTableView',
|
||||
props: {
|
||||
type: {
|
||||
type: String,
|
||||
default: 'list'
|
||||
},
|
||||
title: {
|
||||
type: String,
|
||||
default: ''
|
||||
},
|
||||
fields: {
|
||||
type: Array,
|
||||
default () {
|
||||
return []
|
||||
}
|
||||
}
|
||||
},
|
||||
computed: {
|
||||
...mapState('spider', [
|
||||
'spiderForm'
|
||||
])
|
||||
},
|
||||
methods: {
|
||||
addField () {
|
||||
this.fields.push({
|
||||
type: 'css',
|
||||
extract_type: 'text'
|
||||
})
|
||||
},
|
||||
deleteField (index) {
|
||||
this.fields.splice(index, 1)
|
||||
},
|
||||
onNameChange (row) {
|
||||
if (this.fields.filter(d => d.name === row.name).length > 1) {
|
||||
this.$message.error(this.$t(`Duplicated field names for ${row.name}`))
|
||||
}
|
||||
},
|
||||
onCheck (row) {
|
||||
this.fields.forEach(d => {
|
||||
if (row.name !== d.name) {
|
||||
this.$set(d, 'is_detail', false)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.el-table.edit >>> .el-table__body td {
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.el-table.edit >>> .el-table__body td .cell {
|
||||
padding: 0;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.el-table.edit >>> .el-input__inner:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
.el-table.edit >>> .el-input__inner {
|
||||
height: 36px;
|
||||
border: none;
|
||||
border-radius: 0;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.el-table.edit >>> .el-select .el-input .el-select__caret {
|
||||
line-height: 36px;
|
||||
}
|
||||
|
||||
.button-group-container {
|
||||
/*display: inline-block;*/
|
||||
/*width: 100%;*/
|
||||
}
|
||||
|
||||
.button-group-container .title {
|
||||
float: left;
|
||||
line-height: 32px;
|
||||
}
|
||||
|
||||
.button-group-container .button-group {
|
||||
float: right;
|
||||
}
|
||||
|
||||
.action-button-group {
|
||||
display: flex;
|
||||
margin-left: 10px;
|
||||
}
|
||||
|
||||
.action-button-group >>> .el-checkbox__label {
|
||||
font-size: 12px;
|
||||
}
|
||||
</style>
|
||||
@@ -13,16 +13,18 @@ export default {
|
||||
'Sites': '网站',
|
||||
|
||||
// 标签
|
||||
Overview: '概览',
|
||||
Files: '文件',
|
||||
'Overview': '概览',
|
||||
'Files': '文件',
|
||||
'Deployed Spiders': '已部署爬虫',
|
||||
'Log': '日志',
|
||||
'Results': '结果',
|
||||
'Environment': '环境',
|
||||
'Analytics': '分析',
|
||||
'Rules': '规则',
|
||||
'Config': '配置',
|
||||
|
||||
// 选择
|
||||
Spider: '爬虫',
|
||||
'Spider': '爬虫',
|
||||
|
||||
// 块标题
|
||||
'Latest Tasks': '最近任务',
|
||||
@@ -37,6 +39,7 @@ export default {
|
||||
REVOKED: '已取消',
|
||||
|
||||
// 操作
|
||||
Add: '添加',
|
||||
Run: '运行',
|
||||
Deploy: '部署',
|
||||
Save: '保存',
|
||||
@@ -51,6 +54,7 @@ export default {
|
||||
Remove: '删除',
|
||||
Confirm: '确认',
|
||||
Stop: '停止',
|
||||
Preview: '预览',
|
||||
|
||||
// 主页
|
||||
'Total Tasks': '总任务数',
|
||||
@@ -88,6 +92,10 @@ export default {
|
||||
'Variable': '变量',
|
||||
'Value': '值',
|
||||
'Add Environment Variables': '添加环境变量',
|
||||
'Add Spider': '添加爬虫',
|
||||
'Add Configurable Spider': '添加可配置爬虫',
|
||||
'Add Customized Spider': '添加自定义爬虫',
|
||||
'Add Field': '添加字段',
|
||||
'Last 7-Day Tasks': '最近7天任务数',
|
||||
'Last 5-Run Errors': '最近5次运行错误数',
|
||||
'30-Day Tasks': '最近30天任务数',
|
||||
@@ -98,6 +106,31 @@ export default {
|
||||
'Tasks by Node': '分节点任务数',
|
||||
'Daily Tasks': '每日任务数',
|
||||
'Daily Avg Duration (sec)': '每日平均运行时长(秒)',
|
||||
'Configurable Spider': '可配置爬虫',
|
||||
'Customized Spider': '自定义爬虫',
|
||||
'Configurable': '可配置',
|
||||
'Customized': '自定义',
|
||||
'Text': '文本',
|
||||
'Attribute': '属性',
|
||||
'Field Name': '字段名称',
|
||||
'Query Type': '查询类别',
|
||||
'Query': '查询',
|
||||
'Extract Type': '提取类别',
|
||||
'CSS Selector': 'CSS选择器',
|
||||
'Crawl Type': '抓取类别',
|
||||
'List Only': '仅列表',
|
||||
'Detail Only': '仅详情页',
|
||||
'List + Detail': '列表＋详情页',
|
||||
'Start URL': '开始URL',
|
||||
'Item Selector': '列表项选择器',
|
||||
'Item Selector Type': '列表项选择器类别',
|
||||
'Pagination Selector': '分页选择器',
|
||||
'Pagination Selector Type': '分页项选择器类别',
|
||||
'Preview Results': '预览结果',
|
||||
'Obey robots.txt': '遵守Robots协议',
|
||||
'List Page Fields': '列表页字段',
|
||||
'Detail Page Fields': '详情页字段',
|
||||
'Detail Page URL': '详情页URL',
|
||||
|
||||
// 爬虫列表
|
||||
'Name': '名称',
|
||||
@@ -135,10 +168,15 @@ export default {
|
||||
'Site': '网站',
|
||||
'Rank': '排名',
|
||||
'Domain': '域名',
|
||||
'Main Category': '主类别',
|
||||
'Category': '类别',
|
||||
'Select': '请选择',
|
||||
'Select Main Category': '请选择主类别',
|
||||
'Select Category': '请选择类别',
|
||||
'Spider Count': '爬虫数',
|
||||
'Robots Protocol': 'Robots 协议',
|
||||
'Home Page Response Time (sec)': '首页响应时间(秒)',
|
||||
'Home Page Response Status Code': '首页响应状态码',
|
||||
|
||||
// 文件
|
||||
'Choose Folder': '选择文件',
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import Vue from 'vue'
|
||||
import Vuex from 'vuex'
|
||||
import app from './modules/app'
|
||||
import user from './modules/user'
|
||||
// import user from './modules/user'
|
||||
import tagsView from './modules/tagsView'
|
||||
import dialogView from './modules/dialogView'
|
||||
import node from './modules/node'
|
||||
@@ -19,7 +19,7 @@ Vue.use(Vuex)
|
||||
const store = new Vuex.Store({
|
||||
modules: {
|
||||
app,
|
||||
user,
|
||||
// user,
|
||||
tagsView,
|
||||
dialogView,
|
||||
node,
|
||||
|
||||
@@ -1,10 +1,18 @@
|
||||
import request from '../../api/request'
|
||||
|
||||
const state = {
|
||||
// site list
|
||||
siteList: [],
|
||||
|
||||
// main category list
|
||||
mainCategoryList: [],
|
||||
|
||||
// (sub) category list
|
||||
categoryList: [],
|
||||
|
||||
// filter
|
||||
filter: {
|
||||
mainCategory: undefined,
|
||||
category: undefined
|
||||
},
|
||||
keyword: '',
|
||||
@@ -32,6 +40,12 @@ const mutations = {
|
||||
},
|
||||
SET_TOTAL_COUNT (state, value) {
|
||||
state.totalCount = value
|
||||
},
|
||||
SET_MAIN_CATEGORY_LIST (state, value) {
|
||||
state.mainCategoryList = value
|
||||
},
|
||||
SET_CATEGORY_LIST (state, value) {
|
||||
state.categoryList = value
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,6 +62,7 @@ const actions = {
|
||||
page_size: state.pageSize,
|
||||
keyword: state.keyword || undefined,
|
||||
filter: {
|
||||
main_category: state.filter.mainCategory || undefined,
|
||||
category: state.filter.category || undefined
|
||||
}
|
||||
})
|
||||
@@ -55,6 +70,20 @@ const actions = {
|
||||
commit('SET_SITE_LIST', response.data.items)
|
||||
commit('SET_TOTAL_COUNT', response.data.total_count)
|
||||
})
|
||||
},
|
||||
getMainCategoryList ({ state, commit }) {
|
||||
return request.get('/sites/get/get_main_category_list')
|
||||
.then(response => {
|
||||
commit('SET_MAIN_CATEGORY_LIST', response.data.items)
|
||||
})
|
||||
},
|
||||
getCategoryList ({ state, commit }) {
|
||||
return request.get('/sites/get/get_category_list', {
|
||||
'main_category': state.filter.mainCategory || undefined
|
||||
})
|
||||
.then(response => {
|
||||
commit('SET_CATEGORY_LIST', response.data.items)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -26,7 +26,13 @@ const state = {
|
||||
dailyStats: [],
|
||||
|
||||
// spider node stats
|
||||
nodeStats: []
|
||||
nodeStats: [],
|
||||
|
||||
// filters
|
||||
filterSite: '',
|
||||
|
||||
// preview crawl data
|
||||
previewCrawlData: []
|
||||
}
|
||||
|
||||
const getters = {}
|
||||
@@ -55,12 +61,23 @@ const mutations = {
|
||||
},
|
||||
SET_NODE_STATS (state, value) {
|
||||
state.nodeStats = value
|
||||
},
|
||||
SET_FILTER_SITE (state, value) {
|
||||
state.filterSite = value
|
||||
},
|
||||
SET_PREVIEW_CRAWL_DATA (state, value) {
|
||||
state.previewCrawlData = value
|
||||
}
|
||||
}
|
||||
|
||||
const actions = {
|
||||
getSpiderList ({ state, commit }) {
|
||||
return request.get('/spiders', {})
|
||||
let params = {}
|
||||
if (state.filterSite) {
|
||||
params.site = state.filterSite
|
||||
}
|
||||
console.log(params)
|
||||
return request.get('/spiders', params)
|
||||
.then(response => {
|
||||
commit('SET_SPIDER_LIST', response.data.items)
|
||||
})
|
||||
@@ -68,13 +85,8 @@ const actions = {
|
||||
addSpider ({ state, dispatch }) {
|
||||
return request.put('/spiders', {
|
||||
name: state.spiderForm.name,
|
||||
src: state.spiderForm.src,
|
||||
cmd: state.spiderForm.cmd,
|
||||
type: state.spiderForm.type,
|
||||
lang: state.spiderForm.lang,
|
||||
col: state.spiderForm.col,
|
||||
cron: state.spiderForm.cron,
|
||||
cron_enabled: state.spiderForm.cron_enabled ? 1 : 0,
|
||||
type: 'configurable',
|
||||
site: state.spiderForm.site
|
||||
})
|
||||
.then(() => {
|
||||
@@ -89,9 +101,15 @@ const actions = {
|
||||
type: state.spiderForm.type,
|
||||
lang: state.spiderForm.lang,
|
||||
col: state.spiderForm.col,
|
||||
cron: state.spiderForm.cron,
|
||||
cron_enabled: state.spiderForm.cron_enabled ? 1 : 0,
|
||||
site: state.spiderForm.site
|
||||
site: state.spiderForm.site,
|
||||
// configurable spider
|
||||
crawl_type: state.spiderForm.crawl_type,
|
||||
start_url: state.spiderForm.start_url,
|
||||
item_selector: state.spiderForm.item_selector,
|
||||
item_selector_type: state.spiderForm.item_selector_type,
|
||||
pagination_selector: state.spiderForm.pagination_selector,
|
||||
pagination_selector_type: state.spiderForm.pagination_selector_type,
|
||||
obey_robots_txt: state.spiderForm.obey_robots_txt
|
||||
})
|
||||
.then(() => {
|
||||
dispatch('getSpiderList')
|
||||
@@ -108,6 +126,16 @@ const actions = {
|
||||
envs: JSON.stringify(state.spiderForm.envs)
|
||||
})
|
||||
},
|
||||
updateSpiderFields ({ state }) {
|
||||
return request.post(`/spiders/${state.spiderForm._id}/update_fields`, {
|
||||
fields: JSON.stringify(state.spiderForm.fields)
|
||||
})
|
||||
},
|
||||
updateSpiderDetailFields ({ state }) {
|
||||
return request.post(`/spiders/${state.spiderForm._id}/update_detail_fields`, {
|
||||
detail_fields: JSON.stringify(state.spiderForm.detail_fields)
|
||||
})
|
||||
},
|
||||
getSpiderData ({ state, commit }, id) {
|
||||
return request.get(`/spiders/${id}`)
|
||||
.then(response => {
|
||||
@@ -173,6 +201,12 @@ const actions = {
|
||||
commit('SET_DAILY_STATS', response.data.daily_stats)
|
||||
commit('SET_NODE_STATS', response.data.task_count_by_node)
|
||||
})
|
||||
},
|
||||
getPreviewCrawlData ({ state, commit }) {
|
||||
return request.post(`/spiders/${state.spiderForm._id}/preview_crawl`)
|
||||
.then(response => {
|
||||
commit('SET_PREVIEW_CRAWL_DATA', response.data.items)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<sidebar class="sidebar-container"/>
|
||||
<div class="main-container">
|
||||
<navbar/>
|
||||
<!--<tags-view/>-->
|
||||
<tags-view/>
|
||||
<app-main/>
|
||||
</div>
|
||||
</div>
|
||||
@@ -14,8 +14,8 @@
|
||||
import {
|
||||
Navbar,
|
||||
Sidebar,
|
||||
AppMain
|
||||
// TagsView
|
||||
AppMain,
|
||||
TagsView
|
||||
} from './components'
|
||||
import ResizeMixin from './mixin/ResizeHandler'
|
||||
|
||||
@@ -24,7 +24,7 @@ export default {
|
||||
components: {
|
||||
Navbar,
|
||||
Sidebar,
|
||||
// TagsView,
|
||||
TagsView,
|
||||
AppMain
|
||||
},
|
||||
mixins: [ResizeMixin],
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
class="tags-view-item"
|
||||
@click.middle.native="closeSelectedTag(tag)"
|
||||
@contextmenu.prevent.native="openMenu(tag,$event)">
|
||||
{{ generateTitle(tag.title) }}
|
||||
{{ $t(generateTitle(tag.title)) }}
|
||||
<span v-if="!tag.meta.affix" class="el-icon-close" @click.prevent.stop="closeSelectedTag(tag)"/>
|
||||
</router-link>
|
||||
</scroll-pane>
|
||||
|
||||
@@ -196,7 +196,7 @@ export default {
|
||||
}, 100)
|
||||
})
|
||||
},
|
||||
onRun () {
|
||||
onCrawl () {
|
||||
}
|
||||
},
|
||||
created () {
|
||||
|
||||
@@ -7,11 +7,16 @@
|
||||
class="filter-search"
|
||||
v-model="keyword">
|
||||
</el-input>
|
||||
<el-select v-model="filter.category" class="filter-category" :placeholder="$t('Select Category')" clearable>
|
||||
<el-select v-model="filter.mainCategory" class="filter-category" :placeholder="$t('Select Main Category')"
|
||||
clearable filterable>
|
||||
<el-option v-for="op in mainCategoryList" :key="op" :value="op" :label="op"></el-option>
|
||||
</el-select>
|
||||
<el-select v-model="filter.category" class="filter-category" :placeholder="$t('Select Category')"
|
||||
clearable filterable>
|
||||
<el-option v-for="op in categoryList" :key="op" :value="op" :label="op"></el-option>
|
||||
</el-select>
|
||||
<el-button type="success"
|
||||
icon="el-icon-refresh"
|
||||
icon="el-icon-search"
|
||||
class="btn refresh"
|
||||
@click="onSearch">
|
||||
{{$t('Search')}}
|
||||
@@ -21,27 +26,28 @@
|
||||
<!--table list-->
|
||||
<el-table :data="siteList"
|
||||
class="table"
|
||||
:cell-class-name="getCellClassName"
|
||||
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
|
||||
border>
|
||||
<template v-for="col in columns">
|
||||
<el-table-column v-if="col.name === 'category'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:width="col.width"
|
||||
:align="col.align">
|
||||
<template slot-scope="scope">
|
||||
<el-select v-model="scope.row[col.name]"
|
||||
:placeholder="$t('Select')"
|
||||
@change="onRowChange(scope.row)">
|
||||
<el-option v-for="op in categoryList"
|
||||
:key="op"
|
||||
:value="op"
|
||||
:label="op">
|
||||
</el-option>
|
||||
</el-select>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'domain'"
|
||||
<!--<el-table-column v-if="col.name === 'category'"-->
|
||||
<!--:key="col.name"-->
|
||||
<!--:label="$t(col.label)"-->
|
||||
<!--:width="col.width"-->
|
||||
<!--:align="col.align">-->
|
||||
<!--<template slot-scope="scope">-->
|
||||
<!--<el-select v-model="scope.row[col.name]"-->
|
||||
<!--:placeholder="$t('Select')"-->
|
||||
<!--@change="onRowChange(scope.row)">-->
|
||||
<!--<el-option v-for="op in categoryList"-->
|
||||
<!--:key="op"-->
|
||||
<!--:value="op"-->
|
||||
<!--:label="op">-->
|
||||
<!--</el-option>-->
|
||||
<!--</el-select>-->
|
||||
<!--</template>-->
|
||||
<!--</el-table-column>-->
|
||||
<el-table-column v-if="col.name === 'domain'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:width="col.width"
|
||||
@@ -52,6 +58,60 @@
|
||||
</a>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'spider_count'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:width="col.width"
|
||||
:align="col.align">
|
||||
<template slot-scope="scope">
|
||||
<div>
|
||||
<template v-if="scope.row[col.name] > 0">
|
||||
<a href="javascript:" @click="goToSpiders(scope.row._id)">
|
||||
{{scope.row[col.name]}}
|
||||
</a>
|
||||
</template>
|
||||
<template v-else>
|
||||
{{scope.row[col.name]}}
|
||||
</template>
|
||||
</div>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'has_robots'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:width="col.width"
|
||||
:align="col.align">
|
||||
<template slot-scope="scope">
|
||||
<div>
|
||||
<template v-if="scope.row[col.name]">
|
||||
<a :href="`http://${scope.row._id}/robots.txt`" target="_blank">
|
||||
Y
|
||||
</a>
|
||||
</template>
|
||||
<template v-else>
|
||||
{{scope.row[col.name] === undefined ? 'N/A' : 'N'}}
|
||||
</template>
|
||||
</div>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'home_response_time'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:width="col.width"
|
||||
:align="col.align">
|
||||
<template slot-scope="scope">
|
||||
{{scope.row[col.name] ? scope.row[col.name].toFixed(1) : 'N/A'}}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'home_http_status'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:width="col.width"
|
||||
:align="col.align">
|
||||
<template slot-scope="scope">
|
||||
{{scope.row[col.name] ? scope.row[col.name].toFixed(0) : 'N/A'}}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else
|
||||
:key="col.name"
|
||||
:property="col.name"
|
||||
@@ -61,7 +121,7 @@
|
||||
:width="col.width">
|
||||
</el-table-column>
|
||||
</template>
|
||||
<el-table-column :label="$t('Action')" align="left" width="120">
|
||||
<el-table-column :label="$t('Action')" align="left" width="120" v-if="false">
|
||||
<template slot-scope="scope">
|
||||
<el-tooltip :content="$t('View')" placement="top">
|
||||
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
|
||||
@@ -95,26 +155,30 @@ export default {
|
||||
name: 'SiteList',
|
||||
data () {
|
||||
return {
|
||||
categoryList: [
|
||||
'新闻',
|
||||
'搜索引擎',
|
||||
'综合',
|
||||
'金融',
|
||||
'购物',
|
||||
'社交',
|
||||
'视频',
|
||||
'音乐',
|
||||
'资讯',
|
||||
'政企官网',
|
||||
'其他'
|
||||
],
|
||||
// categoryList: [
|
||||
// '新闻',
|
||||
// '搜索引擎',
|
||||
// '综合',
|
||||
// '金融',
|
||||
// '购物',
|
||||
// '社交',
|
||||
// '视频',
|
||||
// '音乐',
|
||||
// '资讯',
|
||||
// '政企官网',
|
||||
// '其他'
|
||||
// ],
|
||||
columns: [
|
||||
{ name: 'rank', label: 'Rank', align: 'center', width: '80' },
|
||||
{ name: 'name', label: 'Name', align: 'left', width: '120' },
|
||||
{ name: 'name', label: 'Name', align: 'left', width: 'auto' },
|
||||
{ name: 'domain', label: 'Domain', align: 'left', width: '150' },
|
||||
{ name: 'description', label: 'Description', align: 'left' },
|
||||
{ name: 'category', label: 'Category', align: 'center', width: '180' },
|
||||
{ name: 'spider_count', label: 'Spider Count', align: 'center', width: '60' }
|
||||
// { name: 'description', label: 'Description', align: 'left', width: 'auto' },
|
||||
{ name: 'main_category', label: 'Main Category', align: 'center', width: '100' },
|
||||
{ name: 'category', label: 'Category', align: 'center', width: '100' },
|
||||
{ name: 'spider_count', label: 'Spider Count', align: 'center', width: '60' },
|
||||
{ name: 'has_robots', label: 'Robots Protocol', align: 'center', width: '65' },
|
||||
{ name: 'home_response_time', label: 'Home Page Response Time (sec)', align: 'right', width: '80' },
|
||||
{ name: 'home_http_status', label: 'Home Page Response Status Code', align: 'right', width: '80' }
|
||||
]
|
||||
}
|
||||
},
|
||||
@@ -122,6 +186,8 @@ export default {
|
||||
...mapState('site', [
|
||||
'filter',
|
||||
'siteList',
|
||||
'mainCategoryList',
|
||||
'categoryList',
|
||||
'totalCount'
|
||||
]),
|
||||
keyword: {
|
||||
@@ -147,24 +213,103 @@ export default {
|
||||
set (value) {
|
||||
this.$store.commit('site/SET_PAGE_SIZE', value)
|
||||
}
|
||||
},
|
||||
mainCategory () {
|
||||
return this.filter.mainCategory
|
||||
},
|
||||
category () {
|
||||
return this.filter.category
|
||||
}
|
||||
},
|
||||
watch: {
|
||||
mainCategory () {
|
||||
// reset category
|
||||
this.filter.category = undefined
|
||||
|
||||
// get category list
|
||||
this.$store.dispatch('site/getCategoryList')
|
||||
|
||||
// get site list
|
||||
this.$store.dispatch('site/getSiteList')
|
||||
},
|
||||
category () {
|
||||
// get site list
|
||||
this.$store.dispatch('site/getSiteList')
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
onSearch () {
|
||||
this.$store.dispatch('site/getSiteList')
|
||||
setTimeout(() => {
|
||||
this.$store.dispatch('site/getSiteList')
|
||||
}, 0)
|
||||
},
|
||||
onPageChange () {
|
||||
this.$store.dispatch('site/getSiteList')
|
||||
setTimeout(() => {
|
||||
this.$store.dispatch('site/getSiteList')
|
||||
}, 0)
|
||||
},
|
||||
onRowChange (row) {
|
||||
this.$store.dispatch('site/editSite', {
|
||||
id: row.domain,
|
||||
category: row.category
|
||||
})
|
||||
},
|
||||
getCellClassName ({ row, columnIndex }) {
|
||||
let cls = []
|
||||
if (columnIndex === this.getColumnIndex('has_robots')) {
|
||||
cls.push('status')
|
||||
if (row.has_robots === undefined) {
|
||||
cls.push('info')
|
||||
} else if (row.has_robots) {
|
||||
cls.push('danger')
|
||||
} else {
|
||||
cls.push('success')
|
||||
}
|
||||
} else if (columnIndex === this.getColumnIndex('home_response_time')) {
|
||||
cls.push('status')
|
||||
if (row.home_response_time === undefined) {
|
||||
cls.push('info')
|
||||
} else if (row.home_response_time <= 1) {
|
||||
cls.push('success')
|
||||
} else if (row.home_response_time <= 5) {
|
||||
cls.push('primary')
|
||||
} else if (row.home_response_time <= 10) {
|
||||
cls.push('warning')
|
||||
} else {
|
||||
cls.push('danger')
|
||||
}
|
||||
} else if (columnIndex === this.getColumnIndex('home_http_status')) {
|
||||
cls.push('status')
|
||||
if (row.home_http_status === undefined) {
|
||||
cls.push('info')
|
||||
} else if (row.home_http_status >= 200 && row.home_http_status < 300) {
|
||||
cls.push('success')
|
||||
} else {
|
||||
cls.push('danger')
|
||||
}
|
||||
} else if (columnIndex === this.getColumnIndex('spider_count')) {
|
||||
cls.push('status')
|
||||
if (row.spider_count > 0) {
|
||||
cls.push('success')
|
||||
} else {
|
||||
cls.push('info')
|
||||
}
|
||||
}
|
||||
return cls.join(' ')
|
||||
},
|
||||
getColumnIndex (columnName) {
|
||||
return this.columns.map(d => d.name).indexOf(columnName)
|
||||
},
|
||||
goToSpiders (domain) {
|
||||
this.$router.push({ name: 'SpiderList', params: { domain } })
|
||||
}
|
||||
},
|
||||
created () {
|
||||
this.$store.dispatch('site/getSiteList')
|
||||
|
||||
this.$store.dispatch('site/getMainCategoryList')
|
||||
|
||||
this.$store.dispatch('site/getCategoryList')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
@@ -202,4 +347,40 @@ export default {
|
||||
.table >>> .domain {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
.table >>> .status {
|
||||
}
|
||||
|
||||
.table >>> .status.info {
|
||||
color: #909399;
|
||||
background: rgba(144, 147, 153, .1);
|
||||
}
|
||||
|
||||
.table >>> .status.danger {
|
||||
color: #f56c6c;
|
||||
background: rgba(245, 108, 108, .1);
|
||||
}
|
||||
|
||||
.table >>> .status.success {
|
||||
color: #67c23a;
|
||||
background: rgba(103, 194, 58, .1);
|
||||
}
|
||||
|
||||
.table >>> .status.primary {
|
||||
color: #409eff;
|
||||
background: rgba(64, 158, 255, .1);
|
||||
}
|
||||
|
||||
.table >>> .status.warning {
|
||||
color: #e6a23c;
|
||||
background: rgba(230, 162, 60, .1);
|
||||
}
|
||||
|
||||
.table >>> a {
|
||||
text-decoration: underline;
|
||||
display: inline-block;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
@@ -13,7 +13,10 @@
|
||||
<el-tab-pane :label="$t('Overview')" name="overview">
|
||||
<spider-overview/>
|
||||
</el-tab-pane>
|
||||
<el-tab-pane :label="$t('Files')" name="files">
|
||||
<el-tab-pane v-if="isConfigurable" :label="$t('Config')" name="配置">
|
||||
<config-list/>
|
||||
</el-tab-pane>
|
||||
<el-tab-pane v-if="isCustomized" :label="$t('Files')" name="files">
|
||||
<file-list/>
|
||||
</el-tab-pane>
|
||||
<el-tab-pane :label="$t('Environment')" name="environment">
|
||||
@@ -34,10 +37,12 @@ import FileList from '../../components/FileList/FileList'
|
||||
import SpiderOverview from '../../components/Overview/SpiderOverview'
|
||||
import EnvironmentList from '../../components/Environment/EnvironmentList'
|
||||
import SpiderStats from '../../components/Stats/SpiderStats'
|
||||
import ConfigList from '../../components/Config/ConfigList'
|
||||
|
||||
export default {
|
||||
name: 'NodeDetail',
|
||||
components: {
|
||||
ConfigList,
|
||||
SpiderStats,
|
||||
EnvironmentList,
|
||||
FileList,
|
||||
@@ -58,7 +63,13 @@ export default {
|
||||
]),
|
||||
...mapState('deploy', [
|
||||
'deployList'
|
||||
])
|
||||
]),
|
||||
isCustomized () {
|
||||
return this.spiderForm.type === 'customized'
|
||||
},
|
||||
isConfigurable () {
|
||||
return this.spiderForm.type === 'configurable'
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
onTabClick () {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<template>
|
||||
<div class="app-container">
|
||||
<!--add popup-->
|
||||
<!--import popup-->
|
||||
<el-dialog
|
||||
:title="$t('Import Spider')"
|
||||
:visible.sync="dialogVisible"
|
||||
@@ -26,15 +26,83 @@
|
||||
<el-button v-loading="importLoading" type="primary" @click="onImport">{{$t('Import')}}</el-button>
|
||||
</span>
|
||||
</el-dialog>
|
||||
<!--./import popup-->
|
||||
|
||||
<!--add dialog-->
|
||||
<el-dialog :title="$t('Add Spider')"
|
||||
width="40%"
|
||||
:visible.sync="addDialogVisible"
|
||||
:before-close="onAddDialogClose">
|
||||
<div class="add-spider-wrapper">
|
||||
<div @click="onAddConfigurable">
|
||||
<el-card shadow="hover" class="add-spider-item success">
|
||||
{{$t('Configurable Spider')}}
|
||||
</el-card>
|
||||
</div>
|
||||
<div @click="onAddCustomized">
|
||||
<el-card shadow="hover" class="add-spider-item primary">
|
||||
{{$t('Customized Spider')}}
|
||||
</el-card>
|
||||
</div>
|
||||
</div>
|
||||
</el-dialog>
|
||||
<!--./add dialog-->
|
||||
|
||||
<!--configurable spider dialog-->
|
||||
<el-dialog :title="$t('Add Configurable Spider')"
|
||||
width="40%"
|
||||
:visible.sync="addConfigurableDialogVisible"
|
||||
:before-close="onAddConfigurableDialogClose">
|
||||
<el-form :model="spiderForm" ref="addConfigurableForm" inline-message>
|
||||
<el-form-item :label="$t('Spider Name')" label-width="120px" prop="name" required>
|
||||
<el-input :placeholder="$t('Spider Name')" v-model="spiderForm.name"></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Results Collection')" label-width="120px" name="col">
|
||||
<el-input :placeholder="$t('Results Collection')" v-model="spiderForm.col"></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Site')" label-width="120px" name="site">
|
||||
<el-autocomplete v-model="spiderForm.site"
|
||||
:placeholder="$t('Site')"
|
||||
:fetch-suggestions="fetchSiteSuggestions"
|
||||
@select="onAddConfigurableSiteSelect">
|
||||
</el-autocomplete>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
<span slot="footer" class="dialog-footer">
|
||||
<el-button @click="addConfigurableDialogVisible = false">{{$t('Cancel')}}</el-button>
|
||||
<el-button v-loading="addConfigurableLoading" type="primary"
|
||||
@click="onAddConfigurableSpider">{{$t('Add')}}</el-button>
|
||||
</span>
|
||||
</el-dialog>
|
||||
<!--./configurable spider dialog-->
|
||||
|
||||
<!--customized spider dialog-->
|
||||
<el-dialog :title="$t('Add Customized Spider')"
|
||||
width="40%"
|
||||
:visible.sync="addCustomizedDialogVisible"
|
||||
:before-close="onAddCustomizedDialogClose">
|
||||
<p>
|
||||
{{$t('Please go to the source folder of your spiders, create a sub-folder and add your spider codes into it')}}
|
||||
</p>
|
||||
</el-dialog>
|
||||
<!--./customized spider dialog-->
|
||||
|
||||
<!--filter-->
|
||||
<div class="filter">
|
||||
<el-input prefix-icon="el-icon-search"
|
||||
:placeholder="$t('Search')"
|
||||
class="filter-search"
|
||||
v-model="filter.keyword"
|
||||
@change="onSearch">
|
||||
</el-input>
|
||||
<!--<el-input prefix-icon="el-icon-search"-->
|
||||
<!--:placeholder="$t('Search')"-->
|
||||
<!--class="filter-search"-->
|
||||
<!--v-model="filter.keyword"-->
|
||||
<!--@change="onSearch">-->
|
||||
<!--</el-input>-->
|
||||
<div class="left">
|
||||
<el-autocomplete v-model="filterSite"
|
||||
:placeholder="$t('Site')"
|
||||
clearable
|
||||
:fetch-suggestions="fetchSiteSuggestions"
|
||||
@select="onSiteSelect">
|
||||
</el-autocomplete>
|
||||
</div>
|
||||
<div class="right">
|
||||
<el-button type="primary" icon="fa fa-cloud" @click="onDeployAll">
|
||||
{{$t('Deploy All')}}
|
||||
@@ -42,6 +110,12 @@
|
||||
<el-button type="primary" icon="fa fa-download" @click="openImportDialog">
|
||||
{{$t('Import Spiders')}}
|
||||
</el-button>
|
||||
<el-button type="success"
|
||||
icon="el-icon-plus"
|
||||
class="btn add"
|
||||
@click="onAdd">
|
||||
{{$t('Add Spider')}}
|
||||
</el-button>
|
||||
<el-button type="success"
|
||||
icon="el-icon-refresh"
|
||||
class="btn refresh"
|
||||
@@ -60,14 +134,11 @@
|
||||
<el-table-column v-if="col.name === 'type'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
align="center"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
<el-tag v-if="scope.row.type === 'scrapy'">Scrapy</el-tag>
|
||||
<el-tag type="warning" v-else-if="scope.row.type === 'pyspider'">PySpider</el-tag>
|
||||
<el-tag type="info" v-else-if="scope.row.type === 'webmagic'">WebMagic</el-tag>
|
||||
<el-tag type="success" v-else-if="scope.row.type">{{scope.row.type}}</el-tag>
|
||||
<el-tag type="success" v-if="scope.row.type === 'configurable'">{{$t('Configurable')}}</el-tag>
|
||||
<el-tag type="primary" v-else-if="scope.row.type === 'customized'">{{$t('Customized')}}</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'lang'"
|
||||
@@ -104,7 +175,7 @@
|
||||
:width="col.width">
|
||||
</el-table-column>
|
||||
</template>
|
||||
<el-table-column :label="$t('Action')" align="left" width="200">
|
||||
<el-table-column :label="$t('Action')" align="left" width="auto" fixed="right">
|
||||
<template slot-scope="scope">
|
||||
<el-tooltip :content="$t('View')" placement="top">
|
||||
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
|
||||
@@ -115,7 +186,7 @@
|
||||
<el-tooltip :content="$t('Remove')" placement="top">
|
||||
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip :content="$t('Deploy')" placement="top">
|
||||
<el-tooltip v-if="scope.row.type === 'customized'" :content="$t('Deploy')" placement="top">
|
||||
<el-button type="primary" icon="fa fa-cloud" size="mini" @click="onDeploy(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip v-if="isShowRun(scope.row)" :content="$t('Run')" placement="top">
|
||||
@@ -152,16 +223,20 @@ export default {
|
||||
pageSize: 10
|
||||
},
|
||||
importLoading: false,
|
||||
addConfigurableLoading: false,
|
||||
isEditMode: false,
|
||||
dialogVisible: false,
|
||||
addDialogVisible: false,
|
||||
addConfigurableDialogVisible: false,
|
||||
addCustomizedDialogVisible: false,
|
||||
filter: {
|
||||
keyword: ''
|
||||
},
|
||||
// tableData,
|
||||
columns: [
|
||||
{ name: 'name', label: 'Name', width: 'auto' },
|
||||
{ name: 'site_name', label: 'Site', width: '120' },
|
||||
{ name: 'type', label: 'Spider Type', width: '120', sortable: true },
|
||||
{ name: 'name', label: 'Name', width: '180', align: 'left' },
|
||||
{ name: 'site_name', label: 'Site', width: '140', align: 'left' },
|
||||
{ name: 'type', label: 'Spider Type', width: '120' },
|
||||
{ name: 'lang', label: 'Language', width: '120', sortable: true },
|
||||
{ name: 'task_ts', label: 'Last Run', width: '160' },
|
||||
{ name: 'last_7d_tasks', label: 'Last 7-Day Tasks', width: '80' },
|
||||
@@ -179,16 +254,31 @@ export default {
|
||||
'spiderForm'
|
||||
]),
|
||||
filteredTableData () {
|
||||
return this.spiderList.filter(d => {
|
||||
if (!this.filter.keyword) return true
|
||||
for (let i = 0; i < this.columns.length; i++) {
|
||||
const colName = this.columns[i].name
|
||||
if (d[colName] && d[colName].toLowerCase().indexOf(this.filter.keyword.toLowerCase()) > -1) {
|
||||
return true
|
||||
return this.spiderList
|
||||
.filter(d => {
|
||||
if (this.filterSite) {
|
||||
return d.site === this.filterSite
|
||||
}
|
||||
}
|
||||
return false
|
||||
})
|
||||
return true
|
||||
})
|
||||
// .filter(d => {
|
||||
// if (!this.filter.keyword) return true
|
||||
// for (let i = 0; i < this.columns.length; i++) {
|
||||
// const colName = this.columns[i].name
|
||||
// if (d[colName] && d[colName].toLowerCase().indexOf(this.filter.keyword.toLowerCase()) > -1) {
|
||||
// return true
|
||||
// }
|
||||
// }
|
||||
// return false
|
||||
// })
|
||||
},
|
||||
filterSite: {
|
||||
get () {
|
||||
return this.$store.state.spider.filterSite
|
||||
},
|
||||
set (value) {
|
||||
this.$store.commit('spider/SET_FILTER_SITE', value)
|
||||
}
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
@@ -196,9 +286,16 @@ export default {
|
||||
console.log(value)
|
||||
},
|
||||
onAdd () {
|
||||
this.addDialogVisible = true
|
||||
},
|
||||
onAddConfigurable () {
|
||||
this.$store.commit('spider/SET_SPIDER_FORM', {})
|
||||
this.isEditMode = false
|
||||
this.dialogVisible = true
|
||||
this.addDialogVisible = false
|
||||
this.addConfigurableDialogVisible = true
|
||||
},
|
||||
onAddCustomized () {
|
||||
this.addDialogVisible = false
|
||||
this.addCustomizedDialogVisible = true
|
||||
},
|
||||
onRefresh () {
|
||||
this.$store.dispatch('spider/getSpiderList')
|
||||
@@ -223,10 +320,22 @@ export default {
|
||||
this.$store.commit('spider/SET_SPIDER_FORM', {})
|
||||
this.dialogVisible = false
|
||||
},
|
||||
onAddCancel () {
|
||||
this.addDialogVisible = false
|
||||
},
|
||||
onDialogClose () {
|
||||
this.$store.commit('spider/SET_SPIDER_FORM', {})
|
||||
this.dialogVisible = false
|
||||
},
|
||||
onAddDialogClose () {
|
||||
this.addDialogVisible = false
|
||||
},
|
||||
onAddCustomizedDialogClose () {
|
||||
this.addCustomizedDialogVisible = false
|
||||
},
|
||||
onAddConfigurableDialogClose () {
|
||||
this.addConfigurableDialogVisible = false
|
||||
},
|
||||
onEdit (row) {
|
||||
this.isEditMode = true
|
||||
this.$store.commit('spider/SET_SPIDER_FORM', row)
|
||||
@@ -317,17 +426,60 @@ export default {
|
||||
})
|
||||
},
|
||||
isShowRun (row) {
|
||||
if (!row.deploy_ts) {
|
||||
return false
|
||||
if (this.isCustomized(row)) {
|
||||
// customized spider
|
||||
if (!row.deploy_ts) {
|
||||
return false
|
||||
}
|
||||
return !!row.cmd
|
||||
} else {
|
||||
// configurable spider
|
||||
return !!row.fields
|
||||
}
|
||||
if (!row.cmd) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
},
|
||||
isCustomized (row) {
|
||||
return row.type === 'customized'
|
||||
},
|
||||
fetchSiteSuggestions (keyword, callback) {
|
||||
this.$request.get('/sites', {
|
||||
keyword: keyword,
|
||||
page_num: 1,
|
||||
page_size: 100
|
||||
}).then(response => {
|
||||
const data = response.data.items.map(d => {
|
||||
d.value = `${d.name} | ${d.domain}`
|
||||
return d
|
||||
})
|
||||
callback(data)
|
||||
})
|
||||
},
|
||||
onSiteSelect (item) {
|
||||
this.$store.commit('spider/SET_FILTER_SITE', item._id)
|
||||
},
|
||||
onAddConfigurableSiteSelect (item) {
|
||||
this.spiderForm.site = item._id
|
||||
},
|
||||
onAddConfigurableSpider () {
|
||||
this.$refs['addConfigurableForm'].validate(res => {
|
||||
if (res) {
|
||||
this.addConfigurableLoading = true
|
||||
this.$store.dispatch('spider/addSpider')
|
||||
.finally(() => {
|
||||
this.addConfigurableLoading = false
|
||||
this.addConfigurableDialogVisible = false
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
created () {
|
||||
// take site from params to filter
|
||||
this.$store.commit('spider/SET_FILTER_SITE', this.$route.params.domain)
|
||||
|
||||
// fetch spider list
|
||||
this.$store.dispatch('spider/getSpiderList')
|
||||
},
|
||||
mounted () {
|
||||
}
|
||||
}
|
||||
</script>
|
||||
@@ -367,4 +519,37 @@ export default {
|
||||
background-color: red;
|
||||
}
|
||||
|
||||
.add-spider-wrapper {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
|
||||
.add-spider-item {
|
||||
cursor: pointer;
|
||||
width: 180px;
|
||||
font-size: 18px;
|
||||
height: 120px;
|
||||
margin: 0 20px;
|
||||
flex-basis: 40%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.add-spider-item.primary {
|
||||
color: #409eff;
|
||||
background: rgba(64, 158, 255, .1);
|
||||
border: 1px solid rgba(64, 158, 255, .1);
|
||||
}
|
||||
|
||||
.add-spider-item.success {
|
||||
color: #67c23a;
|
||||
background: rgba(103, 194, 58, .1);
|
||||
border: 1px solid rgba(103, 194, 58, .1);
|
||||
}
|
||||
}
|
||||
|
||||
.el-autocomplete {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
@@ -6,11 +6,13 @@
|
||||
<task-overview/>
|
||||
</el-tab-pane>
|
||||
<el-tab-pane :label="$t('Log')" name="log">
|
||||
<div class="log-view">
|
||||
<pre>
|
||||
{{taskLog}}
|
||||
</pre>
|
||||
</div>
|
||||
<el-card>
|
||||
<div class="log-view">
|
||||
<pre>
|
||||
{{taskLog}}
|
||||
</pre>
|
||||
</div>
|
||||
</el-card>
|
||||
</el-tab-pane>
|
||||
<el-tab-pane :label="$t('Results')" name="results">
|
||||
<general-table-view :data="taskResultsData"
|
||||
@@ -101,4 +103,15 @@ export default {
|
||||
.selector .el-select {
|
||||
padding-left: 10px;
|
||||
}
|
||||
|
||||
.log-view {
|
||||
margin: 20px;
|
||||
height: 640px;
|
||||
}
|
||||
|
||||
.log-view pre {
|
||||
height: 100%;
|
||||
overflow-x: auto;
|
||||
overflow-y: auto;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -68,7 +68,7 @@
|
||||
:width="col.width">
|
||||
</el-table-column>
|
||||
</template>
|
||||
<el-table-column :label="$t('Action')" align="center" width="auto">
|
||||
<el-table-column :label="$t('Action')" align="left" width="auto" fixed="right">
|
||||
<template slot-scope="scope">
|
||||
<el-tooltip :content="$t('View')" placement="top">
|
||||
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
|
||||
|
||||
@@ -16,3 +16,6 @@ class ChinazItem(scrapy.Item):
|
||||
domain = scrapy.Field()
|
||||
description = scrapy.Field()
|
||||
rank = scrapy.Field()
|
||||
main_category = scrapy.Field()
|
||||
category = scrapy.Field()
|
||||
location = scrapy.Field()
|
||||
|
||||
@@ -11,19 +11,53 @@ class ChinazSpiderSpider(scrapy.Spider):
|
||||
def parse(self, response):
|
||||
for item in response.css('.listCentent > li'):
|
||||
name = item.css('h3.rightTxtHead > a::text').extract_first()
|
||||
href = item.css('h3.rightTxtHead > a::attr("href")').extract_first()
|
||||
domain = item.css('h3.rightTxtHead > span::text').extract_first()
|
||||
description = item.css('p.RtCInfo::text').extract_first()
|
||||
rank = item.css('.RtCRateCent > strong::text').extract_first()
|
||||
rank = int(rank)
|
||||
yield ChinazItem(
|
||||
item = ChinazItem(
|
||||
_id=domain,
|
||||
name=name,
|
||||
domain=domain,
|
||||
description=description,
|
||||
rank=rank,
|
||||
)
|
||||
yield scrapy.Request(
|
||||
url='http://top.chinaz.com' + href,
|
||||
callback=self.parse_item,
|
||||
meta={
|
||||
'item': item
|
||||
}
|
||||
)
|
||||
|
||||
# pagination
|
||||
a_list = response.css('.ListPageWrap > a::attr("href")').extract()
|
||||
url = 'http://top.chinaz.com/hangye/' + a_list[-1]
|
||||
yield scrapy.Request(url=url)
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
def parse_item(self, response):
|
||||
item = response.meta['item']
|
||||
|
||||
# category info extraction
|
||||
arr = response.css('.TopMainTag-show .SimSun')
|
||||
res1 = arr[0].css('a::text').extract()
|
||||
main_category = res1[0]
|
||||
if len(res1) == 1:
|
||||
category = '其他'
|
||||
else:
|
||||
category = res1[1]
|
||||
|
||||
# location info extraction
|
||||
res2 = arr[1].css('a::text').extract()
|
||||
if len(res2) > 0:
|
||||
location = res2[0]
|
||||
else:
|
||||
location = '其他'
|
||||
|
||||
# assign values to item
|
||||
item['main_category'] = main_category
|
||||
item['category'] = category
|
||||
item['location'] = location
|
||||
|
||||
yield item
|
||||
|
||||
76
spiders/sites_inspector/sites_inspector.py
Normal file
76
spiders/sites_inspector/sites_inspector.py
Normal file
@@ -0,0 +1,76 @@
|
||||
import asyncio
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
import aiohttp
|
||||
import requests
|
||||
|
||||
from pymongo import MongoClient
|
||||
|
||||
# MONGO_HOST = os.environ['MONGO_HOST']
|
||||
# MONGO_PORT = int(os.environ['MONGO_PORT'])
|
||||
# MONGO_DB = os.environ['MONGO_DB']
|
||||
MONGO_HOST = 'localhost'
|
||||
MONGO_PORT = 27017
|
||||
MONGO_DB = 'crawlab_test'
|
||||
|
||||
mongo = MongoClient(host=MONGO_HOST, port=MONGO_PORT)
|
||||
db = mongo[MONGO_DB]
|
||||
col = db['sites']
|
||||
|
||||
|
||||
async def process_response(resp, **kwargs):
|
||||
url = kwargs.get('url')
|
||||
status = resp.status # 读取状态
|
||||
if status == 200 and ('robots.txt' in str(resp.url)):
|
||||
col.update({'_id': url}, {'$set': {'has_robots': True}})
|
||||
else:
|
||||
# 错误状态
|
||||
col.update({'_id': url}, {'$set': {'has_robots': False}})
|
||||
|
||||
|
||||
async def process_home_page_response(resp, **kwargs):
|
||||
url = kwargs.get('url')
|
||||
duration = kwargs.get('duration')
|
||||
status = resp.status # 读取状态
|
||||
col.update({'_id': url}, {'$set': {'home_http_status': status, 'home_response_time': duration}})
|
||||
|
||||
|
||||
async def request_site(url: str, semaphore):
|
||||
_url = 'http://' + url + '/robots.txt'
|
||||
# print('crawling ' + _url)
|
||||
async with semaphore:
|
||||
async with aiohttp.ClientSession() as session: # <1> 开启一个会话
|
||||
async with session.get(_url) as resp: # 发送请求
|
||||
await process_response(resp=resp, url=url)
|
||||
print('crawled ' + _url)
|
||||
# resp = requests.get(_url)
|
||||
return resp
|
||||
|
||||
|
||||
async def request_site_home_page(url: str, semophore):
|
||||
_url = 'http://' + url
|
||||
# print('crawling ' + _url)
|
||||
async with semophore:
|
||||
tic = datetime.now()
|
||||
async with aiohttp.ClientSession() as session: # <1> 开启一个会话
|
||||
async with session.get(_url) as resp: # 发送请求
|
||||
toc = datetime.now()
|
||||
duration = (toc - tic).total_seconds()
|
||||
await process_home_page_response(resp=resp, url=url, duration=duration)
|
||||
print('crawled ' + _url)
|
||||
|
||||
|
||||
async def run():
|
||||
semaphore = asyncio.Semaphore(50) # 限制并发量为50
|
||||
sites = [site for site in col.find({'rank': {'$lte': 5000}})]
|
||||
urls = [site['_id'] for site in sites]
|
||||
to_get = [request_site(url, semaphore) for url in urls]
|
||||
to_get += [request_site_home_page(url, semaphore) for url in urls]
|
||||
await asyncio.wait(to_get)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(run())
|
||||
loop.close()
|
||||
Reference in New Issue
Block a user