git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #55 from tikazyq/develop

Browse Source 
Develop
This commit is contained in:
Marvin Zhang
2019-06-04 23:48:57 +08:00
committed by GitHub
parent de2573af4c 665f957038
commit f602515284
27 changed files with 272 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGELOG.md
View File

@@ -1,6 +1,11 @@
# 0.2.3 (unreleased)
### Features / Enhancement
- **CLI**. Allow user to use command-line interface to execute Crawlab programs.
- **CLI**: Allow user to use command-line interface to execute Crawlab programs.
- **Upload Spider**: Allow user to upload Customized Spider to Crawlab.
- **Edit Fields on Preview**: Allow user to edit fields when previewing data in Configurable Spider.
### Bugs ###
- **Spiders Pagination**
# 0.2.2 (2019-05-30)
### Features / Enhancement

4
Jenkinsfile vendored
View File

@@ -50,9 +50,7 @@ pipeline {
stage('Deploy') {
steps {
echo 'Deploying....'
sh "${NODE_HOME}/bin/node ${NODE_HOME}/bin/pm2 restart app"
sh "${NODE_HOME}/bin/node ${NODE_HOME}/bin/pm2 restart run_flower"
sh "${NODE_HOME}/bin/node ${NODE_HOME}/bin/pm2 restart run_worker"
sh "${NODE_HOME}/bin/node ${NODE_HOME}/bin/pm2 reload all"
}
}
}

2
crawlab/config/config.py
View File

@@ -51,5 +51,5 @@ MONGO_DB = 'crawlab_test'
# Flask 变量
DEBUG = False
FLASK_HOST = '127.0.0.1'
FLASK_HOST = '0.0.0.0'
FLASK_PORT = 8000

12
crawlab/flower.py
View File

@@ -3,14 +3,18 @@ import sys
import subprocess
# make sure the working directory is in system path
file_dir = os.path.dirname(os.path.realpath(__file__))
root_path = os.path.abspath(os.path.join(file_dir, '..'))
sys.path.append(root_path)
FILE_DIR = os.path.dirname(os.path.realpath(__file__))
ROOT_PATH = os.path.abspath(os.path.join(FILE_DIR, '..'))
sys.path.append(ROOT_PATH)
from utils.log import other
from config import BROKER_URL
if __name__ == '__main__':
p = subprocess.Popen(['celery', 'flower', '-b', BROKER_URL], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
p = subprocess.Popen([sys.executable, '-m', 'celery', 'flower', '-b', BROKER_URL],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=ROOT_PATH)
for line in iter(p.stdout.readline, 'b'):
if line.decode('utf-8') != '':
other.info(line.decode('utf-8'))

101
crawlab/routes/spiders.py
View File

@@ -103,6 +103,9 @@ class SpiderApi(BaseApi):
# whether to obey robots.txt
('obey_robots_txt', bool),
# item threshold to filter out non-relevant list items
('item_threshold', int),
)
def get(self, id=None, action=None):
@@ -371,7 +374,6 @@ class SpiderApi(BaseApi):
# make source / destination
src = os.path.join(dir_path, os.listdir(dir_path)[0])
# src = dir_path
dst = os.path.join(PROJECT_DEPLOY_FILE_FOLDER, str(spider.get('_id')))
# logging info
@@ -509,13 +511,24 @@ class SpiderApi(BaseApi):
# get html parse tree
sel = etree.HTML(r.content)
# remove unnecessary tags
unnecessary_tags = [
'script'
]
for t in unnecessary_tags:
etree.strip_tags(sel, t)
return sel
@staticmethod
def _get_children(sel):
return [tag for tag in sel.getchildren() if type(tag) != etree._Comment]
@staticmethod
def _get_text_child_tags(sel):
tags = []
for tag in sel.iter():
if tag.text is not None and tag.text.strip() != '':
if type(tag) != etree._Comment and tag.text is not None and tag.text.strip() != '':
tags.append(tag)
return tags
@@ -530,6 +543,19 @@ class SpiderApi(BaseApi):
return tags
@staticmethod
def _get_next_page_tag(sel):
next_page_text_list = [
'下一页',
'下页',
'next page',
'next',
]
for tag in sel.iter():
if tag.text is not None and tag.text.lower().strip() in next_page_text_list:
return tag
return None
def preview_crawl(self, id: str):
spider = db_manager.get(col_name='spiders', id=id)
@@ -597,11 +623,11 @@ class SpiderApi(BaseApi):
return sel
list_tag_list = []
threshold = 10
threshold = spider.get('item_threshold') or 10
# iterate all child nodes in a top-down direction
for tag in sel.iter():
# get child tags
child_tags = tag.getchildren()
child_tags = self._get_children(tag)
if len(child_tags) < threshold:
# if number of child tags is below threshold, skip
@@ -618,11 +644,10 @@ class SpiderApi(BaseApi):
list_tag_list.append(tag)
# find the list tag with the most child text tags
_tag_list = []
max_tag = None
max_num = 0
for tag in list_tag_list:
_child_text_tags = self._get_text_child_tags(tag[0])
_child_text_tags = self._get_text_child_tags(self._get_children(tag)[0])
if len(_child_text_tags) > max_num:
max_tag = tag
max_num = len(_child_text_tags)
@@ -630,18 +655,26 @@ class SpiderApi(BaseApi):
# get list item selector
item_selector = None
if max_tag.get('id') is not None:
item_selector = f'#{max_tag.get("id")} > {max_tag.getchildren()[0].tag}'
item_selector = f'#{max_tag.get("id")} > {self._get_children(max_tag)[0].tag}'
elif max_tag.get('class') is not None:
if len(sel.cssselect(f'.{max_tag.get("class")}')) == 1:
item_selector = f'.{max_tag.get("class")} > {max_tag.getchildren()[0].tag}'
cls_str = '.'.join([x for x in max_tag.get("class").split(' ') if x != ''])
if len(sel.cssselect(f'.{cls_str}')) == 1:
item_selector = f'.{cls_str} > {self._get_children(max_tag)[0].tag}'
# get list fields
fields = []
if item_selector is not None:
for i, tag in enumerate(self._get_text_child_tags(max_tag[0])):
if tag.get('class') is not None:
first_tag = self._get_children(max_tag)[0]
for i, tag in enumerate(self._get_text_child_tags(first_tag)):
if len(first_tag.cssselect(f'{tag.tag}')) == 1:
fields.append({
'name': f'field{i + 1}',
'type': 'css',
'extract_type': 'text',
'query': f'{tag.tag}',
})
elif tag.get('class') is not None:
cls_str = '.'.join([x for x in tag.get("class").split(' ') if x != ''])
# print(tag.tag + '.' + cls_str)
if len(tag.cssselect(f'{tag.tag}.{cls_str}')) == 1:
fields.append({
'name': f'field{i + 1}',
@@ -650,7 +683,7 @@ class SpiderApi(BaseApi):
'query': f'{tag.tag}.{cls_str}',
})
for i, tag in enumerate(self._get_a_child_tags(max_tag[0])):
for i, tag in enumerate(self._get_a_child_tags(self._get_children(max_tag)[0])):
# if the tag is <a...></a>, extract its href
if tag.get('class') is not None:
cls_str = '.'.join([x for x in tag.get("class").split(' ') if x != ''])
@@ -662,9 +695,19 @@ class SpiderApi(BaseApi):
'query': f'{tag.tag}.{cls_str}',
})
# get pagination tag
pagination_selector = None
pagination_tag = self._get_next_page_tag(sel)
if pagination_tag is not None:
if pagination_tag.get('id') is not None:
pagination_selector = f'#{pagination_tag.get("id")}'
elif pagination_tag.get('class') is not None and len(sel.cssselect(f'.{pagination_tag.get("id")}')) == 1:
pagination_selector = f'.{pagination_tag.get("id")}'
return {
'status': 'ok',
'item_selector': item_selector,
'pagination_selector': pagination_selector,
'fields': fields
}
@@ -801,3 +844,35 @@ class SpiderManageApi(Resource):
'status': 'ok',
'message': 'success'
}
def upload(self):
f = request.files['file']
if get_file_suffix(f.filename) != 'zip':
return {
'status': 'ok',
'error': 'file type mismatch'
}, 400
# save zip file on temp folder
file_path = '%s/%s' % (PROJECT_TMP_FOLDER, f.filename)
with open(file_path, 'wb') as fw:
fw.write(f.stream.read())
# unzip zip file
dir_path = file_path.replace('.zip', '')
if os.path.exists(dir_path):
shutil.rmtree(dir_path)
unzip_file(file_path, dir_path)
# copy to source folder
output_path = os.path.join(PROJECT_SOURCE_FILE_FOLDER, f.filename.replace('.zip', ''))
print(output_path)
if os.path.exists(output_path):
shutil.rmtree(output_path)
shutil.copytree(dir_path, output_path)
return {
'status': 'ok',
'message': 'success'
}

2
crawlab/spiders/spiders/settings.py
View File

@@ -19,7 +19,7 @@ NEWSPIDER_MODULE = 'spiders.spiders'
# USER_AGENT = 'spiders (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = spider.get('obey_robots_txt') or True
ROBOTSTXT_OBEY = spider.get('obey_robots_txt') if spider.get('obey_robots_txt') is not None else True
# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32

2
docs/Architecture/App.html
View File

@@ -397,7 +397,7 @@
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"App","level":"1.4.2","depth":2,"next":{"title":"Examples","level":"1.5","depth":1,"path":"Examples/README.md","ref":"Examples/README.md","articles":[{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},{"title":"与Puppeteer集成","level":"1.5.2","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]}]},"previous":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/App.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-30T13:15:04.840Z"},"basePath":"..","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"App","level":"1.4.2","depth":2,"next":{"title":"Examples","level":"1.5","depth":1,"path":"Examples/README.md","ref":"Examples/README.md","articles":[{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},{"title":"与Puppeteer集成","level":"1.5.2","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]}]},"previous":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/App.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>

2
docs/Architecture/Celery.html
View File

@@ -397,7 +397,7 @@
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"Celery","level":"1.4.1","depth":2,"next":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"previous":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/Celery.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-30T13:15:04.840Z"},"basePath":"..","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"Celery","level":"1.4.1","depth":2,"next":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"previous":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/Celery.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>

2
docs/Architecture/index.html
View File

@@ -397,7 +397,7 @@
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"架构","level":"1.4","depth":1,"next":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"previous":{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-30T13:15:04.840Z"},"basePath":"..","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"架构","level":"1.4","depth":1,"next":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"previous":{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>

2
docs/Concept/Deploy.html
View File

@@ -399,7 +399,7 @@
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"部署","level":"1.3.4","depth":2,"next":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"previous":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Deploy.md","mtime":"2019-03-28T12:06:24.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-30T13:15:04.840Z"},"basePath":"..","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"部署","level":"1.3.4","depth":2,"next":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"previous":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Deploy.md","mtime":"2019-03-28T12:06:24.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>

2
docs/Concept/Node.html
View File

@@ -398,7 +398,7 @@
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"节点","level":"1.3.1","depth":2,"next":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},"previous":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Node.md","mtime":"2019-03-28T12:02:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-30T13:15:04.840Z"},"basePath":"..","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"节点","level":"1.3.1","depth":2,"next":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},"previous":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Node.md","mtime":"2019-03-28T12:02:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>

2
docs/Concept/Spider.html
View File

@@ -403,7 +403,7 @@
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"爬虫","level":"1.3.2","depth":2,"next":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"previous":{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Spider.md","mtime":"2019-03-28T12:03:08.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-30T13:15:04.840Z"},"basePath":"..","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"爬虫","level":"1.3.2","depth":2,"next":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"previous":{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Spider.md","mtime":"2019-03-28T12:03:08.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>

2
docs/Concept/Task.html
View File

@@ -398,7 +398,7 @@
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"任务","level":"1.3.3","depth":2,"next":{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]},"previous":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Task.md","mtime":"2019-03-28T12:04:12.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-30T13:15:04.840Z"},"basePath":"..","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"任务","level":"1.3.3","depth":2,"next":{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]},"previous":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Task.md","mtime":"2019-03-28T12:04:12.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>

2
docs/Concept/index.html
View File

@@ -397,7 +397,7 @@
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"概念","level":"1.3","depth":1,"next":{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},"previous":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/README.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-30T13:15:04.840Z"},"basePath":"..","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"概念","level":"1.3","depth":1,"next":{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},"previous":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/README.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>

2
docs/Examples/index.html
View File

@@ -397,7 +397,7 @@
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"Examples","level":"1.5","depth":1,"next":{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},"previous":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Examples/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-30T13:15:04.840Z"},"basePath":"..","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"Examples","level":"1.5","depth":1,"next":{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},"previous":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Examples/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>

2
docs/QuickStart/Installation.html
View File

@@ -408,7 +408,7 @@ npm install
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"安装","level":"1.2.1","depth":2,"next":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"previous":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Installation.md","mtime":"2019-03-28T11:55:48.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-30T13:15:04.840Z"},"basePath":"..","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"安装","level":"1.2.1","depth":2,"next":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"previous":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Installation.md","mtime":"2019-03-28T11:55:48.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>

11
docs/QuickStart/Run.html
View File

@@ -385,16 +385,13 @@ FLASK_HOST = <span class="hljs-string">&apos;127.0.0.1&apos;</span>
FLASK_PORT = <span class="hljs-number">8000</span>
</code></pre>
<p>&#x542F;&#x52A8;&#x540E;&#x7AEF;API&#xFF0C;&#x4E5F;&#x5C31;&#x662F;&#x4E00;&#x4E2A;Flask App&#xFF0C;&#x53EF;&#x4EE5;&#x76F4;&#x63A5;&#x542F;&#x52A8;&#xFF0C;&#x6216;&#x8005;&#x7528;gunicorn&#x4EE3;&#x66FF;&#x3002;</p>
<pre><code class="lang-bash">python app.py
</code></pre>
<p>&#x542F;&#x52A8;Flower&#x670D;&#x52A1;&#xFF08;&#x62B1;&#x6B49;&#x76EE;&#x524D;&#x96C6;&#x6210;Flower&#x5230;App&#x670D;&#x52A1;&#x4E2D;&#xFF0C;&#x5FC5;&#x987B;&#x5355;&#x72EC;&#x542F;&#x52A8;&#x6765;&#x83B7;&#x53D6;&#x8282;&#x70B9;&#x4FE1;&#x606F;&#xFF0C;&#x540E;&#x9762;&#x7684;&#x7248;&#x672C;&#x4E0D;&#x9700;&#x8981;&#x8FD9;&#x4E2A;&#x64CD;&#x4F5C;&#xFF09;&#x3002;</p>
<pre><code class="lang-bash">python ./bin/run_flower.py
<pre><code class="lang-bash">python manage.py app
</code></pre>
<p>&#x542F;&#x52A8;&#x672C;&#x5730;Worker&#x3002;&#x5728;&#x5176;&#x4ED6;&#x8282;&#x70B9;&#x4E2D;&#x5982;&#x679C;&#x60F3;&#x53EA;&#x662F;&#x60F3;&#x6267;&#x884C;&#x4EFB;&#x52A1;&#x7684;&#x8BDD;&#xFF0C;&#x53EA;&#x9700;&#x8981;&#x542F;&#x52A8;&#x8FD9;&#x4E00;&#x4E2A;&#x670D;&#x52A1;&#x5C31;&#x53EF;&#x4EE5;&#x4E86;&#x3002;</p>
<pre><code class="lang-bash">python ./bin/run_worker.py
<pre><code class="lang-bash">python manage.py worker
</code></pre>
<p>&#x542F;&#x52A8;&#x524D;&#x7AEF;&#x670D;&#x52A1;&#x5668;&#x3002;</p>
<pre><code class="lang-bash"><span class="hljs-built_in">cd</span> ../frontend
<pre><code class="lang-bash"><span class="hljs-built_in">cd</span> frontend
npm run serve
</code></pre>
@@ -440,7 +437,7 @@ npm run serve
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"运行","level":"1.2.2","depth":2,"next":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"previous":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Run.md","mtime":"2019-03-28T12:00:11.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-30T13:15:04.840Z"},"basePath":"..","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"运行","level":"1.2.2","depth":2,"next":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"previous":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Run.md","mtime":"2019-06-04T15:46:17.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>

2
docs/QuickStart/index.html
View File

@@ -401,7 +401,7 @@
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"快速开始","level":"1.2","depth":1,"next":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"previous":{"title":"简介","level":"1.1","depth":1,"path":"README.md","ref":"README.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/README.md","mtime":"2019-03-28T12:02:02.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-30T13:15:04.840Z"},"basePath":"..","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"快速开始","level":"1.2","depth":1,"next":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"previous":{"title":"简介","level":"1.1","depth":1,"path":"README.md","ref":"README.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/README.md","mtime":"2019-03-28T12:02:02.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>

2
docs/index.html
View File

@@ -548,7 +548,7 @@ MONGO_DB = <span class="hljs-string">&apos;crawlab_test&apos;</span>
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"简介","level":"1.1","depth":1,"next":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"README.md","mtime":"2019-03-28T11:44:15.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-30T13:15:04.840Z"},"basePath":".","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"简介","level":"1.1","depth":1,"next":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"README.md","mtime":"2019-03-28T11:44:15.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":".","book":{"language":""}});
});
</script>
</div>

2
docs/search_index.json
View File

File diff suppressed because one or more lines are too long

2
frontend/src/api/request.js
View File

@@ -1,8 +1,6 @@
import axios from 'axios'
let baseUrl = process.env.VUE_APP_BASE_URL ? process.env.VUE_APP_BASE_URL : 'http://localhost:8000/api'
// console.log(process.env)
// const baseUrl = process.env.API_BASE_URL || 'http://localhost:8000/api'
const request = (method, path, params, data) => {
return new Promise((resolve, reject) => {

149
frontend/src/components/Config/ConfigList.vue
View File

@@ -5,15 +5,27 @@
:title="$t('Preview Results')"
width="90%"
:before-close="onDialogClose">
<el-table class="table-header" :data="[{}]" :show-header="false">
<el-table-column v-for="(f, index) in fields"
:key="f.name + '-' + index"
min-width="100px">
<template>
<el-input v-model="columnsDict[f.name]" size="mini" style="width: calc(100% - 15px)"></el-input>
<a href="javascript:" style="margin-left: 2px;" @click="onDeleteField(index)">X</a>
<!--<el-button size="mini" type="danger" icon="el-icon-delete" style="width:45px;margin-left:2px"></el-button>-->
</template>
</el-table-column>
</el-table>
<el-table :data="previewCrawlData"
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
:show-header="false"
border>
<el-table-column v-for="(f, index) in fields"
:key="f.name + '-' + index"
:label="f.name"
:key="index"
min-width="100px">
<template slot-scope="scope">
{{scope.row[f.name]}}
{{getDisplayStr(scope.row[f.name])}}
</template>
</el-table-column>
</el-table>
@@ -22,8 +34,8 @@
<!--config detail-->
<el-row>
<el-col :span="11" :offset="1">
<el-form label-width="150px">
<el-form label-width="150px" ref="form" :model="spiderForm">
<el-col :span="11" :offset="1">
<el-form-item :label="$t('Crawl Type')">
<el-button-group>
<el-button v-for="type in crawlTypeList"
@@ -34,7 +46,7 @@
</el-button>
</el-button-group>
</el-form-item>
<el-form-item :label="$t('Start URL')" required>
<el-form-item :label="$t('Start URL')" prop="start_url" required>
<el-input v-model="spiderForm.start_url" :placeholder="$t('Start URL')"></el-input>
</el-form-item>
<el-form-item :label="$t('Obey robots.txt')">
@@ -43,10 +55,8 @@
<!--<el-form-item :label="$t('URL Pattern')">-->
<!--<el-input v-model="spiderForm.url_pattern" :placeholder="$t('URL Pattern')"></el-input>-->
<!--</el-form-item>-->
</el-form>
</el-col>
<el-col :span="11" :offset="1">
<el-form label-width="150px">
</el-col>
<el-col :span="11" :offset="1">
<el-form-item :label="$t('Item Selector')"
v-if="['list','list-detail'].includes(spiderForm.crawl_type)">
<el-select style="width: 35%;margin-right: 10px;"
@@ -73,8 +83,12 @@
:placeholder="$t('Pagination Selector')">
</el-input>
</el-form-item>
</el-form>
</el-col>
<el-form-item :label="$t('Item Threshold')"
v-if="['list','list-detail'].includes(spiderForm.crawl_type)">
<el-input-number v-model="spiderForm.item_threshold"/>
</el-form-item>
</el-col>
</el-form>
</el-row>
<!--./config detail-->
@@ -134,7 +148,8 @@ export default {
extractFieldsLoading: false,
previewLoading: false,
saveLoading: false,
dialogVisible: false
dialogVisible: false,
columnsDict: {}
}
},
computed: {
@@ -189,23 +204,33 @@ export default {
},
onDialogClose () {
this.dialogVisible = false
this.fields.forEach(f => {
f.name = this.columnsDict[f.name]
})
},
onPreview () {
this.onSave()
.then(() => {
this.previewLoading = true
this.$store.dispatch('spider/getPreviewCrawlData')
this.$refs['form'].validate(res => {
if (res) {
this.onSave()
.then(() => {
this.dialogVisible = true
this.previewLoading = true
this.$store.dispatch('spider/getPreviewCrawlData')
.then(() => {
this.fields.forEach(f => {
this.columnsDict[f.name] = f.name
})
this.dialogVisible = true
})
.catch(() => {
this.$message.error(this.$t('Something wrong happened'))
})
.finally(() => {
this.previewLoading = false
})
this.$st.sendEv('爬虫详情-配置', '预览')
})
.catch(() => {
this.$message.error(this.$t('Something wrong happened'))
})
.finally(() => {
this.previewLoading = false
})
this.$st.sendEv('爬虫详情-配置', '预览')
})
}
})
},
onCrawl () {
this.$confirm(this.$t('Are you sure to run this spider?'), this.$t('Notification'), {
@@ -221,25 +246,42 @@ export default {
})
},
onExtractFields () {
this.onSave()
.then(() => {
this.extractFieldsLoading = true
this.$store.dispatch('spider/extractFields')
.then(response => {
if (response.data.item_selector) {
this.$set(this.spiderForm, 'item_selector', response.data.item_selector)
this.$set(this.spiderForm, 'item_selector_type', 'css')
}
this.$refs['form'].validate(res => {
if (res) {
this.onSave()
.then(() => {
this.extractFieldsLoading = true
this.$store.dispatch('spider/extractFields')
.then(response => {
if (response.data.item_selector) {
this.$set(this.spiderForm, 'item_selector', response.data.item_selector)
this.$set(this.spiderForm, 'item_selector_type', 'css')
}
if (response.data.fields && response.data.fields.length) {
this.spiderForm.fields = response.data.fields
}
if (response.data.fields && response.data.fields.length) {
this.spiderForm.fields = response.data.fields
}
if (response.data.pagination_selector) {
this.spiderForm.pagination_selector = response.data.pagination_selector
}
})
.finally(() => {
this.extractFieldsLoading = false
})
this.$st.sendEv('爬虫详情-配置', '提取字段')
})
.finally(() => {
this.extractFieldsLoading = false
})
this.$st.sendEv('爬虫详情-配置', '提取字段')
})
}
})
},
onDeleteField (index) {
this.fields.splice(index, 1)
},
getDisplayStr (value) {
if (!value) return value
value = value.trim()
if (value.length > 20) return value.substr(0, 20) + '...'
return value
}
},
created () {
@@ -248,7 +290,7 @@ export default {
this.spiderForm.fields = []
for (let i = 0; i < 3; i++) {
this.spiderForm.fields.push({
name: `field_${i + 1}`,
name: 'field_' + (i + 1),
type: 'css',
extract_type: 'text'
})
@@ -260,7 +302,7 @@ export default {
this.spiderForm.detail_fields = []
for (let i = 0; i < 3; i++) {
this.spiderForm.detail_fields.push({
name: `field_${i + 1}`,
name: 'field_' + (i + 1),
type: 'css',
extract_type: 'text'
})
@@ -268,10 +310,11 @@ export default {
}
if (!this.spiderForm.crawl_type) this.$set(this.spiderForm, 'crawl_type', 'list')
if (!this.spiderForm.start_url) this.$set(this.spiderForm, 'start_url', 'http://example.com')
// if (!this.spiderForm.start_url) this.$set(this.spiderForm, 'start_url', 'http://example.com')
if (!this.spiderForm.item_selector_type) this.$set(this.spiderForm, 'item_selector_type', 'css')
if (!this.spiderForm.pagination_selector_type) this.$set(this.spiderForm, 'pagination_selector_type', 'css')
if (this.spiderForm.obey_robots_txt === undefined) this.$set(this.spiderForm, 'obey_robots_txt', true)
if (this.spiderForm.obey_robots_txt == null) this.$set(this.spiderForm, 'obey_robots_txt', true)
if (this.spiderForm.item_threshold == null) this.$set(this.spiderForm, 'item_threshold', 10)
}
}
</script>
@@ -302,4 +345,16 @@ export default {
color: #606266;
font-size: 14px;
}
.el-table.table-header >>> td {
padding: 0;
}
.el-table.table-header >>> .cell {
padding: 0;
}
.el-table.table-header >>> .el-input .el-input__inner {
border-radius: 0;
}
</style>

5
frontend/src/i18n/zh.js
View File

@@ -58,6 +58,9 @@ export default {
'Extract Fields': '提取字段',
'Download': '下载',
'Download CSV': '下载CSV',
'Upload Zip File': '上传Zip文件',
'Upload': '上传',
'Item Threshold': '子项阈值',
// 主页
'Total Tasks': '总任务数',
@@ -120,6 +123,8 @@ export default {
'Query': '查询',
'Extract Type': '提取类别',
'CSS Selector': 'CSS选择器',
'CSS': 'CSS',
'XPath': 'Xpath',
'Crawl Type': '抓取类别',
'List Only': '仅列表',
'Detail Only': '仅详情页',

16
frontend/src/main.js
View File

@@ -33,13 +33,15 @@ Vue.use(ElementUI, { locale })
Vue.config.productionTip = false
// 百度统计
window._hmt = window._hmt || [];
(function () {
let hm = document.createElement('script')
hm.src = 'https://hm.baidu.com/hm.js?c35e3a563a06caee2524902c81975add'
let s = document.getElementsByTagName('script')[0]
s.parentNode.insertBefore(hm, s)
})()
if (localStorage.getItem('useStats') !== '0') {
window._hmt = window._hmt || [];
(function () {
let hm = document.createElement('script')
hm.src = 'https://hm.baidu.com/hm.js?c35e3a563a06caee2524902c81975add'
let s = document.getElementsByTagName('script')[0]
s.parentNode.insertBefore(hm, s)
})()
}
// inject request api
Vue.prototype.$request = request

3
frontend/src/store/modules/spider.js
View File

@@ -110,7 +110,8 @@ const actions = {
item_selector_type: state.spiderForm.item_selector_type,
pagination_selector: state.spiderForm.pagination_selector,
pagination_selector_type: state.spiderForm.pagination_selector_type,
obey_robots_txt: state.spiderForm.obey_robots_txt
obey_robots_txt: state.spiderForm.obey_robots_txt,
item_threshold: state.spiderForm.item_threshold
})
.then(() => {
dispatch('getSpiderList')

35
frontend/src/views/spider/SpiderList.vue
View File

@@ -81,9 +81,16 @@
width="40%"
:visible.sync="addCustomizedDialogVisible"
:before-close="onAddCustomizedDialogClose">
<p>
{{$t('Please go to the source folder of your spiders, create a sub-folder and add your spider codes into it')}}
</p>
<el-form :model="spiderForm" ref="addConfigurableForm" inline-message>
<el-form-item :label="$t('Upload Zip File')" label-width="120px" name="site">
<el-upload
:action="$request.baseUrl + '/spiders/manage/upload'"
:on-success="onUploadSuccess"
:file-list="fileList">
<el-button type="primary" icon="el-icon-upload">{{$t('Upload')}}</el-button>
</el-upload>
</el-form-item>
</el-form>
</el-dialog>
<!--./customized spider dialog-->
@@ -244,7 +251,8 @@ export default {
],
spiderFormRules: {
name: [{ required: true, message: 'Required Field', trigger: 'change' }]
}
},
fileList: []
}
},
computed: {
@@ -261,6 +269,9 @@ export default {
}
return true
})
.filter((d, index) => {
return (this.pagination.pageSize * (this.pagination.pageNum - 1)) <= index && (index < this.pagination.pageSize * this.pagination.pageNum)
})
// .filter(d => {
// if (!this.filter.keyword) return true
// for (let i = 0; i < this.columns.length; i++) {
@@ -390,7 +401,7 @@ export default {
})
},
onView (row) {
this.$router.push(`/spiders/${row._id}`)
this.$router.push('/spiders/' + row._id)
this.$st.sendEv('爬虫', '查看')
},
onPageChange () {
@@ -456,7 +467,7 @@ export default {
page_size: 100
}).then(response => {
const data = response.data.items.map(d => {
d.value = `${d.name} | ${d.domain}`
d.value = d.name + ' | ' + d.domain
return d
})
callback(data)
@@ -480,6 +491,18 @@ export default {
})
}
})
},
onUploadSuccess () {
// clear fileList
this.fileList = []
// fetch spider list
setTimeout(() => {
this.$store.dispatch('spider/getSpiderList')
}, 500)
// close popup
this.addCustomizedDialogVisible = false
}
},
created () {

12
gitbook/QuickStart/Run.md
View File

@@ -36,24 +36,18 @@ FLASK_PORT = 8000
启动后端API，也就是一个Flask App，可以直接启动，或者用gunicorn代替。
```bash
python app.py
```
启动Flower服务（抱歉目前集成Flower到App服务中，必须单独启动来获取节点信息，后面的版本不需要这个操作）。
```bash
python ./bin/run_flower.py
python manage.py app
```
启动本地Worker。在其他节点中如果想只是想执行任务的话，只需要启动这一个服务就可以了。
```bash
python ./bin/run_worker.py
python manage.py worker
```
启动前端服务器。
```bash
cd ../frontend
cd frontend
npm run serve
```