mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-21 17:21:09 +01:00
updated docs
This commit is contained in:
Marvin Zhang
@@ -1,436 +0,0 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>App · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="../Examples/" />
|
||||
|
||||
|
||||
<link rel="prev" href="Celery.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../QuickStart/">
|
||||
|
||||
<a href="../QuickStart/">
|
||||
|
||||
|
||||
快速开始
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../QuickStart/Installation.html">
|
||||
|
||||
<a href="../QuickStart/Installation.html">
|
||||
|
||||
|
||||
安装
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../QuickStart/Run.html">
|
||||
|
||||
<a href="../QuickStart/Run.html">
|
||||
|
||||
|
||||
运行
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../Concept/">
|
||||
|
||||
<a href="../Concept/">
|
||||
|
||||
|
||||
概念
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Concept/Node.html">
|
||||
|
||||
<a href="../Concept/Node.html">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Concept/Spider.html">
|
||||
|
||||
<a href="../Concept/Spider.html">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Concept/Task.html">
|
||||
|
||||
<a href="../Concept/Task.html">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Concept/Deploy.html">
|
||||
|
||||
<a href="../Concept/Deploy.html">
|
||||
|
||||
|
||||
部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="Celery.html">
|
||||
|
||||
<a href="Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.4.2" data-path="App.html">
|
||||
|
||||
<a href="App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >App</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="app">App</h1>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="Celery.html" class="navigation navigation-prev " aria-label="Previous page: Celery">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../Examples/" class="navigation navigation-next " aria-label="Next page: Examples">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"App","level":"1.4.2","depth":2,"next":{"title":"Examples","level":"1.5","depth":1,"path":"Examples/README.md","ref":"Examples/README.md","articles":[{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},{"title":"与Puppeteer集成","level":"1.5.2","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]}]},"previous":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/App.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -1,436 +0,0 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>Celery · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="App.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="./" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../QuickStart/">
|
||||
|
||||
<a href="../QuickStart/">
|
||||
|
||||
|
||||
快速开始
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../QuickStart/Installation.html">
|
||||
|
||||
<a href="../QuickStart/Installation.html">
|
||||
|
||||
|
||||
安装
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../QuickStart/Run.html">
|
||||
|
||||
<a href="../QuickStart/Run.html">
|
||||
|
||||
|
||||
运行
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../Concept/">
|
||||
|
||||
<a href="../Concept/">
|
||||
|
||||
|
||||
概念
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Concept/Node.html">
|
||||
|
||||
<a href="../Concept/Node.html">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Concept/Spider.html">
|
||||
|
||||
<a href="../Concept/Spider.html">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Concept/Task.html">
|
||||
|
||||
<a href="../Concept/Task.html">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Concept/Deploy.html">
|
||||
|
||||
<a href="../Concept/Deploy.html">
|
||||
|
||||
|
||||
部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter active" data-level="1.4.1" data-path="Celery.html">
|
||||
|
||||
<a href="Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="App.html">
|
||||
|
||||
<a href="App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >Celery</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="celery">Celery</h1>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="./" class="navigation navigation-prev " aria-label="Previous page: 架构">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="App.html" class="navigation navigation-next " aria-label="Next page: App">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"Celery","level":"1.4.1","depth":2,"next":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"previous":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/Celery.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -57,10 +57,10 @@
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Celery.html" />
|
||||
<link rel="next" href="../Examples/" />
|
||||
|
||||
|
||||
<link rel="prev" href="../Concept/Deploy.html" />
|
||||
<link rel="prev" href="../Usage/Site/" />
|
||||
|
||||
|
||||
</head>
|
||||
@@ -94,7 +94,7 @@
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
@@ -102,12 +102,12 @@
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../QuickStart/">
|
||||
<li class="chapter " data-level="1.2" data-path="../Installation/">
|
||||
|
||||
<a href="../QuickStart/">
|
||||
<a href="../Installation/">
|
||||
|
||||
|
||||
快速开始
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
@@ -116,12 +116,12 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../QuickStart/Installation.html">
|
||||
<li class="chapter " data-level="1.2.1" data-path="../Installation/Docker.html">
|
||||
|
||||
<a href="../QuickStart/Installation.html">
|
||||
<a href="../Installation/Docker.html">
|
||||
|
||||
|
||||
安装
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
@@ -129,12 +129,25 @@
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../QuickStart/Run.html">
|
||||
<li class="chapter " data-level="1.2.2" data-path="../Installation/Direct.html">
|
||||
|
||||
<a href="../QuickStart/Run.html">
|
||||
<a href="../Installation/Direct.html">
|
||||
|
||||
|
||||
运行
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../Installation/Preview.html">
|
||||
|
||||
<a href="../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
@@ -147,12 +160,12 @@
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../Concept/">
|
||||
<li class="chapter " data-level="1.3" data-path="../Usage/">
|
||||
|
||||
<a href="../Concept/">
|
||||
<a href="../Usage/">
|
||||
|
||||
|
||||
概念
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
@@ -161,9 +174,9 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Concept/Node.html">
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Usage/Node/">
|
||||
|
||||
<a href="../Concept/Node.html">
|
||||
<a href="../Usage/Node/">
|
||||
|
||||
|
||||
节点
|
||||
@@ -172,11 +185,43 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Usage/Node/View.html">
|
||||
|
||||
<a href="../Usage/Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Concept/Spider.html">
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Usage/Node/Edit.html">
|
||||
|
||||
<a href="../Concept/Spider.html">
|
||||
<a href="../Usage/Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Usage/Spider/">
|
||||
|
||||
<a href="../Usage/Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
@@ -185,11 +230,101 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Usage/Spider/Create.html">
|
||||
|
||||
<a href="../Usage/Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Concept/Task.html">
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Concept/Task.html">
|
||||
<a href="../Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Usage/Spider/Deploy.html">
|
||||
|
||||
<a href="../Usage/Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Usage/Spider/Run.html">
|
||||
|
||||
<a href="../Usage/Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Usage/Spider/Analytics.html">
|
||||
|
||||
<a href="../Usage/Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Usage/Task/">
|
||||
|
||||
<a href="../Usage/Task/">
|
||||
|
||||
|
||||
任务
|
||||
@@ -198,14 +333,72 @@
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Concept/Deploy.html">
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Usage/Task/View.html">
|
||||
|
||||
<a href="../Concept/Deploy.html">
|
||||
<a href="../Usage/Task/View.html">
|
||||
|
||||
|
||||
部署
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Usage/Task/Action.html">
|
||||
|
||||
<a href="../Usage/Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Usage/Task/DownloadResults.html">
|
||||
|
||||
<a href="../Usage/Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Usage/Schedule/">
|
||||
|
||||
<a href="../Usage/Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Usage/Site/">
|
||||
|
||||
<a href="../Usage/Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
@@ -229,38 +422,6 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="Celery.html">
|
||||
|
||||
<a href="Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="App.html">
|
||||
|
||||
<a href="App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
@@ -268,7 +429,7 @@
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
@@ -277,9 +438,9 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../Examples/">
|
||||
<a href="../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
@@ -288,19 +449,6 @@
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
@@ -353,7 +501,23 @@
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="架构">架构</h1>
|
||||
<h2 id="架构">架构</h2>
|
||||
<p>Crawlab的架构跟Celery非常相似，但是加入了包括前端、爬虫、Flower在内的额外模块，以支持爬虫管理的功能。架构图如下。</p>
|
||||
<p><img src="https://crawlab.oss-cn-hangzhou.aliyuncs.com/gitbook/architecture.png" alt=""></p>
|
||||
<h3 id="节点-node">节点 Node</h3>
|
||||
<p>节点其实就是Celery中的<code>worker</code>。一个节点运行时会连接到一个任务队列（例如<code>Redis</code>）来接收和运行任务。所有爬虫需要在运行时被部署到节点上，用户在部署前需要定义节点的IP地址和端口。</p>
|
||||
<h3 id="后台应用-backend-app">后台应用 Backend App</h3>
|
||||
<p>这是一个Flask应用，提供了必要的API来支持常规操作，例如CRUD、爬虫部署以及任务运行。每一个节点需要启动Flask应用来支持爬虫部署。运行<code>python app.py</code>来启动应用。</p>
|
||||
<h3 id="爬虫-spider">爬虫 Spider</h3>
|
||||
<p>爬虫源代码或配置规则储存在<code>App</code>上，需要被部署到各个<code>worker</code>节点中。</p>
|
||||
<h3 id="任务-task">任务 Task</h3>
|
||||
<p>任务被触发并被节点执行。用户可以在任务详情页面中看到任务到状态、日志和抓取结果。</p>
|
||||
<h3 id="中间者-broker">中间者 Broker</h3>
|
||||
<p>中间者跟Celery中定义的一样，作为运行异步任务的队列。</p>
|
||||
<h3 id="前端-frontend">前端 Frontend</h3>
|
||||
<p>前端其实就是一个基于<a href="https://github.com/PanJiaChen/vue-element-admin" target="_blank">Vue-Element-Admin</a>的单页应用。其中重用了很多Element-UI的控件来支持相应的展示。</p>
|
||||
<h3 id="flower">Flower</h3>
|
||||
<p>一个Celery的插件，用于监控Celery节点。</p>
|
||||
|
||||
|
||||
</section>
|
||||
@@ -381,12 +545,12 @@
|
||||
|
||||
|
||||
|
||||
<a href="../Concept/Deploy.html" class="navigation navigation-prev " aria-label="Previous page: 部署">
|
||||
<a href="../Usage/Site/" class="navigation navigation-prev " aria-label="Previous page: 网站">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Celery.html" class="navigation navigation-next " aria-label="Next page: Celery">
|
||||
<a href="../Examples/" class="navigation navigation-next " aria-label="Next page: 样例">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
@@ -397,7 +561,7 @@
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"架构","level":"1.4","depth":1,"next":{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},"previous":{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"架构","level":"1.4","depth":1,"next":{"title":"样例","level":"1.5","depth":1,"path":"Examples/README.md","ref":"Examples/README.md","articles":[{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/ScrapyIntegration.md","ref":"Examples/ScrapyIntegration.md","articles":[]}]},"previous":{"title":"网站","level":"1.3.5","depth":2,"path":"Usage/Site/README.md","ref":"Usage/Site/README.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Architecture/README.md","mtime":"2019-06-16T13:48:56.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
@@ -1,438 +0,0 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>部署 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="../Architecture/" />
|
||||
|
||||
|
||||
<link rel="prev" href="Task.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../QuickStart/">
|
||||
|
||||
<a href="../QuickStart/">
|
||||
|
||||
|
||||
快速开始
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../QuickStart/Installation.html">
|
||||
|
||||
<a href="../QuickStart/Installation.html">
|
||||
|
||||
|
||||
安装
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../QuickStart/Run.html">
|
||||
|
||||
<a href="../QuickStart/Run.html">
|
||||
|
||||
|
||||
运行
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
概念
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="Node.html">
|
||||
|
||||
<a href="Node.html">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="Spider.html">
|
||||
|
||||
<a href="Spider.html">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="Task.html">
|
||||
|
||||
<a href="Task.html">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.4" data-path="Deploy.html">
|
||||
|
||||
<a href="Deploy.html">
|
||||
|
||||
|
||||
部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
||||
|
||||
<a href="../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="../Architecture/Celery.html">
|
||||
|
||||
<a href="../Architecture/Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="../Architecture/App.html">
|
||||
|
||||
<a href="../Architecture/App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >部署</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="部署">部署</h1>
|
||||
<p>所有爬虫在运行前需要被部署当相应当节点中。</p>
|
||||
<p>部署时，爬虫会被打包到相应的目录中，方便环境隔离，开发环境的爬虫和生产环境的爬虫需要打包部署来实现隔离。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="Task.html" class="navigation navigation-prev " aria-label="Previous page: 任务">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../Architecture/" class="navigation navigation-next " aria-label="Next page: 架构">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"部署","level":"1.3.4","depth":2,"next":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[{"title":"Celery","level":"1.4.1","depth":2,"path":"Architecture/Celery.md","ref":"Architecture/Celery.md","articles":[]},{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]}]},"previous":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Deploy.md","mtime":"2019-03-28T12:06:24.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -1,437 +0,0 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>节点 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Spider.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="./" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../QuickStart/">
|
||||
|
||||
<a href="../QuickStart/">
|
||||
|
||||
|
||||
快速开始
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../QuickStart/Installation.html">
|
||||
|
||||
<a href="../QuickStart/Installation.html">
|
||||
|
||||
|
||||
安装
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../QuickStart/Run.html">
|
||||
|
||||
<a href="../QuickStart/Run.html">
|
||||
|
||||
|
||||
运行
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
概念
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter active" data-level="1.3.1" data-path="Node.html">
|
||||
|
||||
<a href="Node.html">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="Spider.html">
|
||||
|
||||
<a href="Spider.html">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="Task.html">
|
||||
|
||||
<a href="Task.html">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="Deploy.html">
|
||||
|
||||
<a href="Deploy.html">
|
||||
|
||||
|
||||
部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
||||
|
||||
<a href="../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="../Architecture/Celery.html">
|
||||
|
||||
<a href="../Architecture/Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="../Architecture/App.html">
|
||||
|
||||
<a href="../Architecture/App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >节点</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="节点">节点</h1>
|
||||
<p>节点其实就是Celery中的Worker。一个节点运行时会连接到一个任务队列（例如Redis）来接收和运行任务。所有爬虫需要在运行时被部署到节点上，用户在部署前需要定义节点的IP地址和端口。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="./" class="navigation navigation-prev " aria-label="Previous page: 概念">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Spider.html" class="navigation navigation-next " aria-label="Next page: 爬虫">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"节点","level":"1.3.1","depth":2,"next":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},"previous":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Node.md","mtime":"2019-03-28T12:02:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -1,442 +0,0 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>爬虫 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Task.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="Node.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../QuickStart/">
|
||||
|
||||
<a href="../QuickStart/">
|
||||
|
||||
|
||||
快速开始
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../QuickStart/Installation.html">
|
||||
|
||||
<a href="../QuickStart/Installation.html">
|
||||
|
||||
|
||||
安装
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../QuickStart/Run.html">
|
||||
|
||||
<a href="../QuickStart/Run.html">
|
||||
|
||||
|
||||
运行
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
概念
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="Node.html">
|
||||
|
||||
<a href="Node.html">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.2" data-path="Spider.html">
|
||||
|
||||
<a href="Spider.html">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="Task.html">
|
||||
|
||||
<a href="Task.html">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="Deploy.html">
|
||||
|
||||
<a href="Deploy.html">
|
||||
|
||||
|
||||
部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
||||
|
||||
<a href="../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="../Architecture/Celery.html">
|
||||
|
||||
<a href="../Architecture/Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="../Architecture/App.html">
|
||||
|
||||
<a href="../Architecture/App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >爬虫</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="爬虫">爬虫</h1>
|
||||
<h2 id="自动发现">自动发现</h2>
|
||||
<p>在<code>config.py</code>文件中，修改变量<code>PROJECT_SOURCE_FILE_FOLDER</code>作为爬虫项目所在的目录。Crawlab后台程序会自动发现这些爬虫项目并储存到数据库中。是不是很方便？</p>
|
||||
<h2 id="部署爬虫">部署爬虫</h2>
|
||||
<p>所有爬虫需要在抓取前被部署当相应当节点中。在"爬虫详情"页面点击"Deploy"按钮，爬虫将被部署到所有有效到节点中。</p>
|
||||
<h2 id="运行爬虫">运行爬虫</h2>
|
||||
<p>部署爬虫之后，你可以在"爬虫详情"页面点击"Run"按钮来启动爬虫。一个爬虫任务将被触发，你可以在任务列表页面中看到这个任务。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="Node.html" class="navigation navigation-prev " aria-label="Previous page: 节点">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Task.html" class="navigation navigation-next " aria-label="Next page: 任务">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"爬虫","level":"1.3.2","depth":2,"next":{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},"previous":{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Spider.md","mtime":"2019-03-28T12:03:08.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -1,437 +0,0 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>任务 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Deploy.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="Spider.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../QuickStart/">
|
||||
|
||||
<a href="../QuickStart/">
|
||||
|
||||
|
||||
快速开始
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../QuickStart/Installation.html">
|
||||
|
||||
<a href="../QuickStart/Installation.html">
|
||||
|
||||
|
||||
安装
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../QuickStart/Run.html">
|
||||
|
||||
<a href="../QuickStart/Run.html">
|
||||
|
||||
|
||||
运行
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
概念
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="Node.html">
|
||||
|
||||
<a href="Node.html">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="Spider.html">
|
||||
|
||||
<a href="Spider.html">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.3" data-path="Task.html">
|
||||
|
||||
<a href="Task.html">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="Deploy.html">
|
||||
|
||||
<a href="Deploy.html">
|
||||
|
||||
|
||||
部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
||||
|
||||
<a href="../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="../Architecture/Celery.html">
|
||||
|
||||
<a href="../Architecture/Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="../Architecture/App.html">
|
||||
|
||||
<a href="../Architecture/App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >任务</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="任务">任务</h1>
|
||||
<p>任务被触发并被节点执行。用户可以在任务详情页面中看到任务到状态、日志和抓取结果。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="Spider.html" class="navigation navigation-prev " aria-label="Previous page: 爬虫">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Deploy.html" class="navigation navigation-next " aria-label="Next page: 部署">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"任务","level":"1.3.3","depth":2,"next":{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]},"previous":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/Task.md","mtime":"2019-03-28T12:04:12.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
0
docs/Examples/PuppeteerIntegration.md
Normal file
0
docs/Examples/PuppeteerIntegration.md
Normal file
600
docs/Examples/ScrapyIntegration.html
Normal file
600
docs/Examples/ScrapyIntegration.html
Normal file
@@ -0,0 +1,600 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>与Scrapy集成 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
|
||||
<link rel="prev" href="./" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../Installation/">
|
||||
|
||||
<a href="../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../Installation/Docker.html">
|
||||
|
||||
<a href="../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../Installation/Direct.html">
|
||||
|
||||
<a href="../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../Installation/Preview.html">
|
||||
|
||||
<a href="../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../Usage/">
|
||||
|
||||
<a href="../Usage/">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Usage/Node/">
|
||||
|
||||
<a href="../Usage/Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Usage/Node/View.html">
|
||||
|
||||
<a href="../Usage/Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Usage/Node/Edit.html">
|
||||
|
||||
<a href="../Usage/Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Usage/Spider/">
|
||||
|
||||
<a href="../Usage/Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Usage/Spider/Create.html">
|
||||
|
||||
<a href="../Usage/Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Usage/Spider/Deploy.html">
|
||||
|
||||
<a href="../Usage/Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Usage/Spider/Run.html">
|
||||
|
||||
<a href="../Usage/Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Usage/Spider/Analytics.html">
|
||||
|
||||
<a href="../Usage/Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Usage/Task/">
|
||||
|
||||
<a href="../Usage/Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Usage/Task/View.html">
|
||||
|
||||
<a href="../Usage/Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Usage/Task/Action.html">
|
||||
|
||||
<a href="../Usage/Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Usage/Task/DownloadResults.html">
|
||||
|
||||
<a href="../Usage/Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Usage/Schedule/">
|
||||
|
||||
<a href="../Usage/Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Usage/Site/">
|
||||
|
||||
<a href="../Usage/Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
||||
|
||||
<a href="../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter active" data-level="1.5.1" data-path="ScrapyIntegration.html">
|
||||
|
||||
<a href="ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >与Scrapy集成</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h3 id="与scrapy集成">与Scrapy集成</h3>
|
||||
<p>以下是Crawlab跟<code>Scrapy</code>集成的例子，利用了Crawlab传过来的<code>task_id</code>和<code>collection_name</code>。</p>
|
||||
<pre><code class="lang-python"><span class="hljs-keyword">import</span> os
|
||||
<span class="hljs-keyword">from</span> pymongo <span class="hljs-keyword">import</span> MongoClient
|
||||
|
||||
MONGO_HOST = <span class="hljs-string">'192.168.99.100'</span>
|
||||
MONGO_PORT = <span class="hljs-number">27017</span>
|
||||
MONGO_DB = <span class="hljs-string">'crawlab_test'</span>
|
||||
|
||||
<span class="hljs-comment"># scrapy example in the pipeline</span>
|
||||
<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">JuejinPipeline</span><span class="hljs-params">(object)</span>:</span>
|
||||
mongo = MongoClient(host=MONGO_HOST, port=MONGO_PORT)
|
||||
db = mongo[MONGO_DB]
|
||||
col_name = os.environ.get(<span class="hljs-string">'CRAWLAB_COLLECTION'</span>)
|
||||
<span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> col_name:
|
||||
col_name = <span class="hljs-string">'test'</span>
|
||||
col = db[col_name]
|
||||
|
||||
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">process_item</span><span class="hljs-params">(self, item, spider)</span>:</span>
|
||||
item[<span class="hljs-string">'task_id'</span>] = os.environ.get(<span class="hljs-string">'CRAWLAB_TASK_ID'</span>)
|
||||
self.col.save(item)
|
||||
<span class="hljs-keyword">return</span> item
|
||||
</code></pre>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="./" class="navigation navigation-prev navigation-unique" aria-label="Previous page: 样例">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"与Scrapy集成","level":"1.5.1","depth":2,"previous":{"title":"样例","level":"1.5","depth":1,"path":"Examples/README.md","ref":"Examples/README.md","articles":[{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/ScrapyIntegration.md","ref":"Examples/ScrapyIntegration.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Examples/ScrapyIntegration.md","mtime":"2019-06-16T13:46:53.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>Examples · GitBook</title>
|
||||
<title>样例 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
@@ -57,10 +57,10 @@
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="./" />
|
||||
<link rel="next" href="ScrapyIntegration.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="../Architecture/App.html" />
|
||||
<link rel="prev" href="../Architecture/" />
|
||||
|
||||
|
||||
</head>
|
||||
@@ -94,7 +94,7 @@
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
@@ -102,12 +102,12 @@
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../QuickStart/">
|
||||
<li class="chapter " data-level="1.2" data-path="../Installation/">
|
||||
|
||||
<a href="../QuickStart/">
|
||||
<a href="../Installation/">
|
||||
|
||||
|
||||
快速开始
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
@@ -116,12 +116,12 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../QuickStart/Installation.html">
|
||||
<li class="chapter " data-level="1.2.1" data-path="../Installation/Docker.html">
|
||||
|
||||
<a href="../QuickStart/Installation.html">
|
||||
<a href="../Installation/Docker.html">
|
||||
|
||||
|
||||
安装
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
@@ -129,12 +129,25 @@
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../QuickStart/Run.html">
|
||||
<li class="chapter " data-level="1.2.2" data-path="../Installation/Direct.html">
|
||||
|
||||
<a href="../QuickStart/Run.html">
|
||||
<a href="../Installation/Direct.html">
|
||||
|
||||
|
||||
运行
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../Installation/Preview.html">
|
||||
|
||||
<a href="../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
@@ -147,12 +160,12 @@
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../Concept/">
|
||||
<li class="chapter " data-level="1.3" data-path="../Usage/">
|
||||
|
||||
<a href="../Concept/">
|
||||
<a href="../Usage/">
|
||||
|
||||
|
||||
概念
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
@@ -161,9 +174,9 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Concept/Node.html">
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Usage/Node/">
|
||||
|
||||
<a href="../Concept/Node.html">
|
||||
<a href="../Usage/Node/">
|
||||
|
||||
|
||||
节点
|
||||
@@ -172,11 +185,43 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Usage/Node/View.html">
|
||||
|
||||
<a href="../Usage/Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Concept/Spider.html">
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Usage/Node/Edit.html">
|
||||
|
||||
<a href="../Concept/Spider.html">
|
||||
<a href="../Usage/Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Usage/Spider/">
|
||||
|
||||
<a href="../Usage/Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
@@ -185,11 +230,101 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Usage/Spider/Create.html">
|
||||
|
||||
<a href="../Usage/Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Concept/Task.html">
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Concept/Task.html">
|
||||
<a href="../Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Usage/Spider/Deploy.html">
|
||||
|
||||
<a href="../Usage/Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Usage/Spider/Run.html">
|
||||
|
||||
<a href="../Usage/Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Usage/Spider/Analytics.html">
|
||||
|
||||
<a href="../Usage/Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Usage/Task/">
|
||||
|
||||
<a href="../Usage/Task/">
|
||||
|
||||
|
||||
任务
|
||||
@@ -198,14 +333,72 @@
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Concept/Deploy.html">
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Usage/Task/View.html">
|
||||
|
||||
<a href="../Concept/Deploy.html">
|
||||
<a href="../Usage/Task/View.html">
|
||||
|
||||
|
||||
部署
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Usage/Task/Action.html">
|
||||
|
||||
<a href="../Usage/Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Usage/Task/DownloadResults.html">
|
||||
|
||||
<a href="../Usage/Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Usage/Schedule/">
|
||||
|
||||
<a href="../Usage/Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Usage/Site/">
|
||||
|
||||
<a href="../Usage/Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
@@ -229,38 +422,6 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="../Architecture/Celery.html">
|
||||
|
||||
<a href="../Architecture/Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="../Architecture/App.html">
|
||||
|
||||
<a href="../Architecture/App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.5" data-path="./">
|
||||
@@ -268,7 +429,7 @@
|
||||
<a href="./">
|
||||
|
||||
|
||||
Examples
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
@@ -277,9 +438,9 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter active" data-level="1.5.1" data-path="./">
|
||||
<li class="chapter " data-level="1.5.1" data-path="ScrapyIntegration.html">
|
||||
|
||||
<a href="./">
|
||||
<a href="ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
@@ -288,19 +449,6 @@
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.5.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
@@ -338,7 +486,7 @@
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >Examples</a>
|
||||
<a href=".." >样例</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
@@ -353,7 +501,10 @@
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="examples">Examples</h1>
|
||||
<h2 id="样例">样例</h2>
|
||||
<ol>
|
||||
<li><a href="ScrapyIntegration.html">与Scrapy集成</a></li>
|
||||
</ol>
|
||||
|
||||
|
||||
</section>
|
||||
@@ -381,12 +532,12 @@
|
||||
|
||||
|
||||
|
||||
<a href="../Architecture/App.html" class="navigation navigation-prev " aria-label="Previous page: App">
|
||||
<a href="../Architecture/" class="navigation navigation-prev " aria-label="Previous page: 架构">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="./" class="navigation navigation-next " aria-label="Next page: 与Scrapy集成">
|
||||
<a href="ScrapyIntegration.html" class="navigation navigation-next " aria-label="Next page: 与Scrapy集成">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
@@ -397,7 +548,7 @@
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"Examples","level":"1.5","depth":1,"next":{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},"previous":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Examples/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"样例","level":"1.5","depth":1,"next":{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/ScrapyIntegration.md","ref":"Examples/ScrapyIntegration.md","articles":[]},"previous":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Examples/README.md","mtime":"2019-06-16T13:48:53.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
# 功能列表
|
||||
|
||||
类别 | 功能名称 | 已统计 | 备注
|
||||
--- | --- | --- | ---
|
||||
全局 | 打开页面 | Y | _trackPageview
|
||||
全局 | 切换中英文 | Y
|
||||
全局 | 允许/禁止统计 | Y
|
||||
节点 | 刷新 | Y
|
||||
节点 | 查看 | Y
|
||||
节点 | 删除 | Y
|
||||
节点详情 | 保存 | Y
|
||||
节点详情 | 切换节点 | Y
|
||||
爬虫 | 部署所有爬虫 | Y
|
||||
爬虫 | 导入爬虫 | Y
|
||||
爬虫 | 添加爬虫-可配置爬虫 | Y
|
||||
爬虫 | 添加爬虫-自定义爬虫 | Y
|
||||
爬虫 | 刷新 | Y
|
||||
爬虫 | 查看 | Y
|
||||
爬虫 | 删除 | Y
|
||||
爬虫 | 部署 | Y
|
||||
爬虫 | 运行 | Y
|
||||
爬虫 | 搜索网站 | Y
|
||||
爬虫详情 | 切换爬虫 | Y
|
||||
爬虫详情 | 切换标签 | Y
|
||||
爬虫详情-概览 | 保存 | Y
|
||||
爬虫详情-概览 | 部署 | Y
|
||||
爬虫详情-概览 | 运行 | Y
|
||||
爬虫详情-环境 | 添加 | Y
|
||||
爬虫详情-环境 | 删除 | Y
|
||||
爬虫详情-环境 | 保存 | Y
|
||||
爬虫详情-配置 | 保存 | Y
|
||||
爬虫详情-配置 | 预览 | Y
|
||||
爬虫详情-配置 | 提取字段 | Y
|
||||
爬虫详情-配置 | 运行 | Y
|
||||
爬虫详情-配置 | 添加字段 | Y
|
||||
爬虫详情-配置 | 更改字段 | Y
|
||||
爬虫详情-配置 | 删除字段 | Y
|
||||
爬虫详情-配置 | 设置详情页URL | Y
|
||||
任务 | 选择节点 | Y
|
||||
任务 | 选择爬虫 | Y
|
||||
任务 | 点击爬虫详情 | Y
|
||||
任务 | 点击节点详情 | Y
|
||||
任务 | 搜索 | Y
|
||||
任务 | 查看 | Y
|
||||
任务 | 删除 | Y
|
||||
任务详情 | 切换标签 | Y
|
||||
任务详情-概览 | 点击爬虫详情 | Y
|
||||
任务详情-概览 | 点击节点详情 | Y
|
||||
任务详情-结果 | 下载CSV | Y
|
||||
定时任务 | 添加 | Y
|
||||
定时任务 | 修改 | Y
|
||||
定时任务 | 删除 | Y
|
||||
定时任务 | 提交 | Y
|
||||
部署 | 刷新 | Y
|
||||
网站 | 搜索 | Y
|
||||
网站 | 选择主类别 | Y
|
||||
网站 | 选择类别 | Y
|
||||
网站 | 点击域名 | Y
|
||||
网站 | 点击爬虫数 | Y
|
||||
网站 | 点击Robots协议 | N
|
||||
|
||||
632
docs/Installation/Direct.html
Normal file
632
docs/Installation/Direct.html
Normal file
@@ -0,0 +1,632 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>直接部署 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Preview.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="Docker.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="Docker.html">
|
||||
|
||||
<a href="Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.2.2" data-path="Direct.html">
|
||||
|
||||
<a href="Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="Preview.html">
|
||||
|
||||
<a href="Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../Usage/">
|
||||
|
||||
<a href="../Usage/">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Usage/Node/">
|
||||
|
||||
<a href="../Usage/Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Usage/Node/View.html">
|
||||
|
||||
<a href="../Usage/Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Usage/Node/Edit.html">
|
||||
|
||||
<a href="../Usage/Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Usage/Spider/">
|
||||
|
||||
<a href="../Usage/Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Usage/Spider/Create.html">
|
||||
|
||||
<a href="../Usage/Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Usage/Spider/Deploy.html">
|
||||
|
||||
<a href="../Usage/Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Usage/Spider/Run.html">
|
||||
|
||||
<a href="../Usage/Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Usage/Spider/Analytics.html">
|
||||
|
||||
<a href="../Usage/Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Usage/Task/">
|
||||
|
||||
<a href="../Usage/Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Usage/Task/View.html">
|
||||
|
||||
<a href="../Usage/Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Usage/Task/Action.html">
|
||||
|
||||
<a href="../Usage/Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Usage/Task/DownloadResults.html">
|
||||
|
||||
<a href="../Usage/Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Usage/Schedule/">
|
||||
|
||||
<a href="../Usage/Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Usage/Site/">
|
||||
|
||||
<a href="../Usage/Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
||||
|
||||
<a href="../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >直接部署</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="直接部署">直接部署</h2>
|
||||
<p>直接部署是之前没有Docker时的部署方式，相对于Docker部署来说有些繁琐。但了解如何直接部署可以帮助更深入地理解Docker是如何构建Crawlab镜像的。这里简单介绍一下。</p>
|
||||
<h3 id="拉取代码">拉取代码</h3>
|
||||
<p>首先是将github上的代码拉取到本地。</p>
|
||||
<pre><code class="lang-bash">git <span class="hljs-built_in">clone</span> https://github.com/tikazyq/crawlab
|
||||
</code></pre>
|
||||
<h3 id="安装">安装</h3>
|
||||
<p>安装前端所需库。</p>
|
||||
<pre><code class="lang-bash">npm install -g yarn pm2
|
||||
<span class="hljs-built_in">cd</span> frontend
|
||||
yarn install
|
||||
</code></pre>
|
||||
<p>安装后端所需库。</p>
|
||||
<pre><code class="lang-bash"><span class="hljs-built_in">cd</span> ../crawlab
|
||||
pip install -r requirements
|
||||
</code></pre>
|
||||
<h3 id="配置">配置</h3>
|
||||
<p>分别配置前端配置文件<code>./frontend/.env.production</code>和后端配置文件<code>./crawlab/config/config.py</code>。分别需要对部署后API地址以及数据库地址进行配置。</p>
|
||||
<h3 id="构建">构建</h3>
|
||||
<p>这里的构建是指前端构建，需要执行以下命令。</p>
|
||||
<pre><code class="lang-bash"><span class="hljs-built_in">cd</span> ../frontend
|
||||
npm run build:prod
|
||||
</code></pre>
|
||||
<p>构建完成后，会在<code>./frontend</code>目录下创建一个<code>dist</code>文件夹，里面是打包好后的静态文件。</p>
|
||||
<h3 id="nginx">Nginx</h3>
|
||||
<p>安装<code>nginx</code>，在<code>ubuntu 16.04</code>是以下命令。</p>
|
||||
<pre><code class="lang-bash">sudo apt-get install nginx
|
||||
</code></pre>
|
||||
<p>添加<code>/etc/nginx/conf.d/crawlab.conf</code>文件，输入以下内容。</p>
|
||||
<pre><code>server {
|
||||
listen 8080;
|
||||
server_name dev.crawlab.com;
|
||||
root /home/yeqing/jenkins_home/workspace/crawlab_develop/frontend/dist;
|
||||
index index.html;
|
||||
}
|
||||
</code></pre><p>其中，<code>root</code>是静态文件的根目录，这里是<code>npm</code>打包好后的静态文件。</p>
|
||||
<p>现在，只需要启动<code>nginx</code>服务就完成了启动前端服务。</p>
|
||||
<pre><code class="lang-bash">nginx reload
|
||||
</code></pre>
|
||||
<h3 id="启动服务">启动服务</h3>
|
||||
<p>这里是指启动后端服务。我们用<code>pm2</code>来管理进程。执行以下命令。</p>
|
||||
<pre><code class="lang-bash">pm2 start app.py <span class="hljs-comment"># API服务</span>
|
||||
pm2 start worker.py <span class="hljs-comment"># Worker</span>
|
||||
pm2 start flower.py <span class="hljs-comment"># Flower</span>
|
||||
</code></pre>
|
||||
<p>这样，<code>pm2</code>会启动3个守护进程来管理这3个服务。我们如果想看后端服务的日志的话，可以执行以下命令。</p>
|
||||
<pre><code class="lang-bash">pm2 logs [app]
|
||||
</code></pre>
|
||||
<p>然后在浏览器中输入<code>http://localhost:8080</code>就可以看到界面了。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="Docker.html" class="navigation navigation-prev " aria-label="Previous page: Docker">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Preview.html" class="navigation navigation-next " aria-label="Next page: 预览模式">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"直接部署","level":"1.2.2","depth":2,"next":{"title":"预览模式","level":"1.2.3","depth":2,"path":"Installation/Preview.md","ref":"Installation/Preview.md","articles":[]},"previous":{"title":"Docker","level":"1.2.1","depth":2,"path":"Installation/Docker.md","ref":"Installation/Docker.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Installation/Direct.md","mtime":"2019-06-16T03:18:40.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
707
docs/Installation/Docker.html
Normal file
707
docs/Installation/Docker.html
Normal file
@@ -0,0 +1,707 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>Docker · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Direct.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="./" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter active" data-level="1.2.1" data-path="Docker.html">
|
||||
|
||||
<a href="Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="Direct.html">
|
||||
|
||||
<a href="Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="Preview.html">
|
||||
|
||||
<a href="Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../Usage/">
|
||||
|
||||
<a href="../Usage/">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Usage/Node/">
|
||||
|
||||
<a href="../Usage/Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Usage/Node/View.html">
|
||||
|
||||
<a href="../Usage/Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Usage/Node/Edit.html">
|
||||
|
||||
<a href="../Usage/Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Usage/Spider/">
|
||||
|
||||
<a href="../Usage/Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Usage/Spider/Create.html">
|
||||
|
||||
<a href="../Usage/Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Usage/Spider/Deploy.html">
|
||||
|
||||
<a href="../Usage/Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Usage/Spider/Run.html">
|
||||
|
||||
<a href="../Usage/Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Usage/Spider/Analytics.html">
|
||||
|
||||
<a href="../Usage/Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Usage/Task/">
|
||||
|
||||
<a href="../Usage/Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Usage/Task/View.html">
|
||||
|
||||
<a href="../Usage/Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Usage/Task/Action.html">
|
||||
|
||||
<a href="../Usage/Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Usage/Task/DownloadResults.html">
|
||||
|
||||
<a href="../Usage/Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Usage/Schedule/">
|
||||
|
||||
<a href="../Usage/Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Usage/Site/">
|
||||
|
||||
<a href="../Usage/Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
||||
|
||||
<a href="../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >Docker</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="docker安装部署">Docker安装部署</h2>
|
||||
<p>这应该是部署应用的最方便也是最节省时间的方式了。在最近的一次版本更新<a href="https://github.com/tikazyq/crawlab/releases/tag/v0.2.3" target="_blank">v0.2.3</a>中，我们发布了Docker功能，让大家可以利用Docker来轻松部署Crawlab。下面将一步一步介绍如何使用Docker来部署Crawlab。</p>
|
||||
<p>对Docker不了解的开发者，可以参考一下这篇文章（<a href="https://juejin.im/post/5c2c69cee51d450d9707236e" target="_blank">9102 年了，学点 Docker 知识</a>）做进一步了解。简单来说，Docker可以利用已存在的镜像帮助构建一些常用的服务和应用，例如Nginx、MongoDB、Redis等等。用Docker运行一个MongoDB服务仅需<code>docker run -d --name mongo -p 27017:27017 mongo</code>一行命令。如何安装Docker跟操作系统有关，这里就不展开讲了，需要的同学自行百度一下相关教程。</p>
|
||||
<h3 id="下载镜像">下载镜像</h3>
|
||||
<p>我们已经在<a href="https://hub.docker.com/r/tikazyq/crawlab" target="_blank">DockerHub</a>上构建了Crawlab的镜像，开发者只需要将其pull下来使用。在pull 镜像之前，我们需要配置一下镜像源。因为我们在墙内，使用原有的镜像源速度非常感人，因此将使用DockerHub在国内的加速器。创建<code>/etc/docker/daemon.json</code>文件，在其中输入如下内容。</p>
|
||||
<pre><code class="lang-json">{
|
||||
<span class="hljs-string">"registry-mirrors"</span>: [<span class="hljs-string">"https://registry.docker-cn.com"</span>]
|
||||
}
|
||||
</code></pre>
|
||||
<p>这样的话，pull镜像的速度会比不改变镜像源的速度快很多。</p>
|
||||
<p>执行以下命令将Crawlab的镜像下载下来。镜像大小大概在几百兆，因此下载需要几分钟时间。</p>
|
||||
<pre><code class="lang-bash">docker pull tikazyq/crawlab:latest
|
||||
</code></pre>
|
||||
<h3 id="更改配置文件">更改配置文件</h3>
|
||||
<p>拷贝一份后端配置文件<code>./crawlab/config/config.py</code>以及前端配置文件<code>./frontend/.env.production</code>到某一个地方。例如我的例子，分别为<code>/home/yeqing/config.py</code>和<code>/home/yeqing/.env.production</code>。</p>
|
||||
<p>更改后端配置文件<code>config.py</code>，将MongoDB、Redis的指向IP更改为自己数据的值。注意，容器中对应的宿主机的IP地址不是<code>localhost</code>，而是<code>172.17.0.1</code>（当然也可以用network来做，只是稍微麻烦一些）。更改前端配置文件<code>.env.production</code>，将API地址<code>VUE_APP_BASE_URL</code>更改为宿主机所在的IP地址，例如<code>http://192.168.0.8:8000</code>，这将是前端调用API会用到的URL。</p>
|
||||
<h3 id="运行docker容器">运行Docker容器</h3>
|
||||
<p>更改好配置文件之后，接下来就是运行容器了。执行以下命令来启动容器。</p>
|
||||
<pre><code class="lang-bash">docker run <span class="hljs-_">-d</span> --rm --name crawlab \
|
||||
-p 8080:8080 \
|
||||
-p 8000:8000 \
|
||||
-v /home/yeqing/.env.production:/opt/crawlab/frontend/.env.production \
|
||||
-v /home/yeqing/config.py:/opt/crawlab/crawlab/config/config.py \
|
||||
tikazyq/crawlab master
|
||||
</code></pre>
|
||||
<p>其中，我们映射了8080端口（Nginx前端静态文件）以及8000端口（后端API）到宿主机。另外还将前端配置文件<code>/home/yeqing/.env.production</code>和后端配置文件<code>/home/yeqing/config.py</code>映射到了容器相应的目录下。传入参数<code>master</code>是代表该启动方式为主机启动模式，也就是所有服务（前端、Api、Flower、Worker）都会启动。另外一个模式是<code>worker</code>模式，只会启动必要的Api和Worker服务，这个对于分布式部署比较有用。等待大约20-30秒的时间来build前端静态文件，之后就可以打开Crawlab界面地址地址看到界面了。界面地址默认为<code>http://localhost:8080</code>。</p>
|
||||
<p><img src="https://user-gold-cdn.xitu.io/2019/6/12/16b4c3ed5dcd6cfc?w=2532&h=1300&f=png&s=146531" alt=""></p>
|
||||
<h3 id="docker-compose">Docker-Compose</h3>
|
||||
<p>当然，也可以用<code>docker-compose</code>的方式来部署。<code>docker-compose</code>是一个集群管理方式，可以利用名为<code>docker-compose.yml</code>的<code>yaml</code>文件来定义需要启动的容器，可以是单个，也可以（通常）是多个的。Crawlab的<code>docker-compose.yml</code>定义如下。</p>
|
||||
<pre><code class="lang-yaml"><span class="hljs-attr">version:</span> <span class="hljs-string">'3.3'</span>
|
||||
<span class="hljs-attr">services:</span>
|
||||
<span class="hljs-attr"> master:</span>
|
||||
<span class="hljs-attr"> image:</span> tikazyq/crawlab:latest
|
||||
<span class="hljs-attr"> container_name:</span> crawlab
|
||||
<span class="hljs-attr"> volumns:</span>
|
||||
<span class="hljs-bullet"> -</span> /home/yeqing/config.py:/opt/crawlab/crawlab/config/config.py <span class="hljs-comment"># 后端配置文件</span>
|
||||
<span class="hljs-bullet"> -</span> /home/yeqing/.env.production:/opt/crawlab/frontend/.env.production <span class="hljs-comment"># 前端配置文件</span>
|
||||
<span class="hljs-attr"> ports:</span>
|
||||
<span class="hljs-bullet"> -</span> <span class="hljs-string">"8080:8080"</span> <span class="hljs-comment"># nginx</span>
|
||||
<span class="hljs-bullet"> -</span> <span class="hljs-string">"8000:8000"</span> <span class="hljs-comment"># app</span>
|
||||
<span class="hljs-attr"> depends_on:</span>
|
||||
<span class="hljs-bullet"> -</span> mongo
|
||||
<span class="hljs-bullet"> -</span> redis
|
||||
<span class="hljs-attr"> entrypoint:</span>
|
||||
<span class="hljs-bullet"> -</span> /bin/sh
|
||||
<span class="hljs-bullet"> -</span> /opt/crawlab/docker_init.sh
|
||||
<span class="hljs-bullet"> -</span> master
|
||||
<span class="hljs-attr"> mongo:</span>
|
||||
<span class="hljs-attr"> image:</span> mongo:latest
|
||||
<span class="hljs-attr"> restart:</span> always
|
||||
<span class="hljs-attr"> ports:</span>
|
||||
<span class="hljs-bullet"> -</span> <span class="hljs-string">"27017:27017"</span>
|
||||
<span class="hljs-attr"> redis:</span>
|
||||
<span class="hljs-attr"> image:</span> redis:latest
|
||||
<span class="hljs-attr"> restart:</span> always
|
||||
<span class="hljs-attr"> ports:</span>
|
||||
<span class="hljs-bullet"> -</span> <span class="hljs-string">"6379:6379"</span>
|
||||
</code></pre>
|
||||
<p>这里先定义了<code>master</code>节点，也就是Crawlab的主节点。<code>master</code>依赖于<code>mongo</code>和<code>redis</code>容器，因此在启动之前会同时启动<code>mongo</code>和<code>redis</code>容器。这样就不需要单独配置<code>mongo</code>和<code>redis</code>服务了，大大节省了环境配置的时间。</p>
|
||||
<p>安装<code>docker-compose</code>也很简单，大家去网上百度一下就可以了。</p>
|
||||
<p>安装完<code>docker-compose</code>和定义好<code>docker-compose.yml</code>后，只需要运行以下命令就可以启动Crawlab。</p>
|
||||
<pre><code class="lang-bash">docker-compose up
|
||||
</code></pre>
|
||||
<p>同样，在浏览器中输入<code>http://localhost:8080</code>就可以看到界面。</p>
|
||||
<h3 id="多节点模式">多节点模式</h3>
|
||||
<p><code>docker-compose</code>的方式很适合多节点部署，在原有的<code>master</code>基础上增加几个<code>worker</code>节点，达到多节点部署的目的。将<code>docker-compose.yml</code>更改为如下内容。</p>
|
||||
<pre><code class="lang-yaml"><span class="hljs-attr">version:</span> <span class="hljs-string">'3.3'</span>
|
||||
<span class="hljs-attr">services:</span>
|
||||
<span class="hljs-attr"> master:</span>
|
||||
<span class="hljs-attr"> image:</span> tikazyq/crawlab:latest
|
||||
<span class="hljs-attr"> container_name:</span> crawlab
|
||||
<span class="hljs-attr"> volumns:</span>
|
||||
<span class="hljs-bullet"> -</span> /home/yeqing/config.master.py:/opt/crawlab/crawlab/config/config.py <span class="hljs-comment"># 后端配置文件</span>
|
||||
<span class="hljs-bullet"> -</span> /home/yeqing/.env.production.master:/opt/crawlab/frontend/.env.production <span class="hljs-comment"># 前端配置文件</span>
|
||||
<span class="hljs-attr"> ports:</span>
|
||||
<span class="hljs-bullet"> -</span> <span class="hljs-string">"8080:8080"</span> <span class="hljs-comment"># nginx</span>
|
||||
<span class="hljs-bullet"> -</span> <span class="hljs-string">"8000:8000"</span> <span class="hljs-comment"># app</span>
|
||||
<span class="hljs-attr"> depends_on:</span>
|
||||
<span class="hljs-bullet"> -</span> mongo
|
||||
<span class="hljs-bullet"> -</span> redis
|
||||
<span class="hljs-attr"> entrypoint:</span>
|
||||
<span class="hljs-bullet"> -</span> /bin/sh
|
||||
<span class="hljs-bullet"> -</span> /opt/crawlab/docker_init.sh
|
||||
<span class="hljs-bullet"> -</span> master
|
||||
<span class="hljs-attr"> worker1:</span>
|
||||
<span class="hljs-attr"> image:</span> tikazyq/crawlab:latest
|
||||
<span class="hljs-attr"> volumns:</span>
|
||||
<span class="hljs-bullet"> -</span> /home/yeqing/config.worker.py:/opt/crawlab/crawlab/config/config.py <span class="hljs-comment"># 后端配置文件</span>
|
||||
<span class="hljs-bullet"> -</span> /home/yeqing/.env.production.worker:/opt/crawlab/frontend/.env.production <span class="hljs-comment"># 前端配置文件</span>
|
||||
<span class="hljs-attr"> ports:</span>
|
||||
<span class="hljs-bullet"> -</span> <span class="hljs-string">"8001:8000"</span> <span class="hljs-comment"># app</span>
|
||||
<span class="hljs-attr"> depends_on:</span>
|
||||
<span class="hljs-bullet"> -</span> mongo
|
||||
<span class="hljs-bullet"> -</span> redis
|
||||
<span class="hljs-attr"> entrypoint:</span>
|
||||
<span class="hljs-bullet"> -</span> /bin/sh
|
||||
<span class="hljs-bullet"> -</span> /opt/crawlab/docker_init.sh
|
||||
<span class="hljs-bullet"> -</span> worker
|
||||
<span class="hljs-attr"> worker2:</span>
|
||||
<span class="hljs-attr"> image:</span> tikazyq/crawlab:latest
|
||||
<span class="hljs-attr"> volumns:</span>
|
||||
<span class="hljs-bullet"> -</span> /home/yeqing/config.worker.py:/opt/crawlab/crawlab/config/config.py <span class="hljs-comment"># 后端配置文件</span>
|
||||
<span class="hljs-bullet"> -</span> /home/yeqing/.env.production.worker:/opt/crawlab/frontend/.env.production <span class="hljs-comment"># 前端配置文件</span>
|
||||
<span class="hljs-attr"> ports:</span>
|
||||
<span class="hljs-bullet"> -</span> <span class="hljs-string">"8002:8000"</span> <span class="hljs-comment"># app</span>
|
||||
<span class="hljs-attr"> depends_on:</span>
|
||||
<span class="hljs-bullet"> -</span> mongo
|
||||
<span class="hljs-bullet"> -</span> redis
|
||||
<span class="hljs-attr"> entrypoint:</span>
|
||||
<span class="hljs-bullet"> -</span> /bin/sh
|
||||
<span class="hljs-bullet"> -</span> /opt/crawlab/docker_init.sh
|
||||
<span class="hljs-bullet"> -</span> worker
|
||||
<span class="hljs-attr"> mongo:</span>
|
||||
<span class="hljs-attr"> image:</span> mongo:latest
|
||||
<span class="hljs-attr"> restart:</span> always
|
||||
<span class="hljs-attr"> ports:</span>
|
||||
<span class="hljs-bullet"> -</span> <span class="hljs-string">"27017:27017"</span>
|
||||
<span class="hljs-attr"> redis:</span>
|
||||
<span class="hljs-attr"> image:</span> redis:latest
|
||||
<span class="hljs-attr"> restart:</span> always
|
||||
<span class="hljs-attr"> ports:</span>
|
||||
<span class="hljs-bullet"> -</span> <span class="hljs-string">"6379:6379"</span>
|
||||
</code></pre>
|
||||
<p>这里启动了多增加了两个<code>worker</code>节点，以<code>worker</code>模式启动。这样，多节点部署，也就是分布式部署就完成了。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="./" class="navigation navigation-prev " aria-label="Previous page: 安装Crawlab">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Direct.html" class="navigation navigation-next " aria-label="Next page: 直接部署">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"Docker","level":"1.2.1","depth":2,"next":{"title":"直接部署","level":"1.2.2","depth":2,"path":"Installation/Direct.md","ref":"Installation/Direct.md","articles":[]},"previous":{"title":"安装Crawlab","level":"1.2","depth":1,"path":"Installation/README.md","ref":"Installation/README.md","articles":[{"title":"Docker","level":"1.2.1","depth":2,"path":"Installation/Docker.md","ref":"Installation/Docker.md","articles":[]},{"title":"直接部署","level":"1.2.2","depth":2,"path":"Installation/Direct.md","ref":"Installation/Direct.md","articles":[]},{"title":"预览模式","level":"1.2.3","depth":2,"path":"Installation/Preview.md","ref":"Installation/Preview.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Installation/Docker.md","mtime":"2019-06-16T03:07:59.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
588
docs/Installation/Preview.html
Normal file
588
docs/Installation/Preview.html
Normal file
@@ -0,0 +1,588 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>预览模式 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="../Usage/" />
|
||||
|
||||
|
||||
<link rel="prev" href="Direct.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="Docker.html">
|
||||
|
||||
<a href="Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="Direct.html">
|
||||
|
||||
<a href="Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.2.3" data-path="Preview.html">
|
||||
|
||||
<a href="Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../Usage/">
|
||||
|
||||
<a href="../Usage/">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Usage/Node/">
|
||||
|
||||
<a href="../Usage/Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Usage/Node/View.html">
|
||||
|
||||
<a href="../Usage/Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Usage/Node/Edit.html">
|
||||
|
||||
<a href="../Usage/Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Usage/Spider/">
|
||||
|
||||
<a href="../Usage/Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Usage/Spider/Create.html">
|
||||
|
||||
<a href="../Usage/Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Usage/Spider/Deploy.html">
|
||||
|
||||
<a href="../Usage/Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Usage/Spider/Run.html">
|
||||
|
||||
<a href="../Usage/Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Usage/Spider/Analytics.html">
|
||||
|
||||
<a href="../Usage/Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Usage/Task/">
|
||||
|
||||
<a href="../Usage/Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Usage/Task/View.html">
|
||||
|
||||
<a href="../Usage/Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Usage/Task/Action.html">
|
||||
|
||||
<a href="../Usage/Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Usage/Task/DownloadResults.html">
|
||||
|
||||
<a href="../Usage/Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Usage/Schedule/">
|
||||
|
||||
<a href="../Usage/Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Usage/Site/">
|
||||
|
||||
<a href="../Usage/Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
||||
|
||||
<a href="../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >预览模式</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="预览模式">预览模式</h2>
|
||||
<p><strong>预览模式</strong>是一种让用户比较快的上手的一种部署模式。跟<strong>直接部署</strong>类似，但不用经过<code>构建</code>、<code>nginx</code>和<code>启动服务</code>的步骤。在启动时只需要执行以下命令就可以了。相较于直接部署来说方便一些。</p>
|
||||
<pre><code class="lang-bash">python manage.py serve
|
||||
</code></pre>
|
||||
<p>该模式同样会启动3个后端服务和1个前端服务。前端服务是通过<code>npm run serve</code>来进行的，因此是开发者模式。<strong>注意：强烈不建议在生产环境中用预览模式</strong>。预览模式只是让开发者快速体验Crawlab以及调试代码问题的一种方式，而不是用作生产环境部署的。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="Direct.html" class="navigation navigation-prev " aria-label="Previous page: 直接部署">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../Usage/" class="navigation navigation-next " aria-label="Next page: 使用Crawlab">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"预览模式","level":"1.2.3","depth":2,"next":{"title":"使用Crawlab","level":"1.3","depth":1,"path":"Usage/README.md","ref":"Usage/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Usage/Node/README.md","ref":"Usage/Node/README.md","articles":[{"title":"查看节点列表","level":"1.3.1.1","depth":3,"path":"Usage/Node/View.md","ref":"Usage/Node/View.md","articles":[]},{"title":"修改节点信息","level":"1.3.1.2","depth":3,"path":"Usage/Node/Edit.md","ref":"Usage/Node/Edit.md","articles":[]}]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Usage/Spider/README.md","ref":"Usage/Spider/README.md","articles":[{"title":"创建爬虫","level":"1.3.2.1","depth":3,"path":"Usage/Spider/Create.md","ref":"Usage/Spider/Create.md","articles":[{"title":"自定义爬虫","level":"1.3.2.1.1","depth":4,"path":"Usage/Spider/CustomizedSpider.md","ref":"Usage/Spider/CustomizedSpider.md","articles":[]},{"title":"可配置爬虫","level":"1.3.2.1.2","depth":4,"path":"Usage/Spider/ConfigurableSpider.md","ref":"Usage/Spider/ConfigurableSpider.md","articles":[]}]},{"title":"部署爬虫","level":"1.3.2.2","depth":3,"path":"Usage/Spider/Deploy.md","ref":"Usage/Spider/Deploy.md","articles":[]},{"title":"运行爬虫","level":"1.3.2.3","depth":3,"path":"Usage/Spider/Run.md","ref":"Usage/Spider/Run.md","articles":[]},{"title":"统计数据","level":"1.3.2.4","depth":3,"path":"Usage/Spider/Analytics.md","ref":"Usage/Spider/Analytics.md","articles":[]}]},{"title":"任务","level":"1.3.3","depth":2,"path":"Usage/Task/README.md","ref":"Usage/Task/README.md","articles":[{"title":"查看任务","level":"1.3.3.1","depth":3,"path":"Usage/Task/View.md","ref":"Usage/Task/View.md","articles":[]},{"title":"操作任务","level":"1.3.3.2","depth":3,"path":"Usage/Task/Action.md","ref":"Usage/Task/Action.md","articles":[]},{"title":"下载结果","level":"1.3.3.3","depth":3,"path":"Usage/Task/DownloadResults.md","ref":"Usage/Task/DownloadResults.md","articles":[]}]},{"title":"定时任务","level":"1.3.4","depth":2,"path":"Usage/Schedule/README.md","ref":"Usage/Schedule/README.md","articles":[]},{"title":"网站","level":"1.3.5","depth":2,"path":"Usage/Site/README.md","ref":"Usage/Site/README.md","articles":[]}]},"previous":{"title":"直接部署","level":"1.2.2","depth":2,"path":"Installation/Direct.md","ref":"Installation/Direct.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Installation/Preview.md","mtime":"2019-06-16T03:19:51.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>快速开始 · GitBook</title>
|
||||
<title>安装Crawlab · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
@@ -57,7 +57,7 @@
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Installation.html" />
|
||||
<link rel="next" href="Docker.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="../" />
|
||||
@@ -94,7 +94,7 @@
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
@@ -107,7 +107,7 @@
|
||||
<a href="./">
|
||||
|
||||
|
||||
快速开始
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
@@ -116,12 +116,12 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="Installation.html">
|
||||
<li class="chapter " data-level="1.2.1" data-path="Docker.html">
|
||||
|
||||
<a href="Installation.html">
|
||||
<a href="Docker.html">
|
||||
|
||||
|
||||
安装
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
@@ -129,12 +129,25 @@
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="Run.html">
|
||||
<li class="chapter " data-level="1.2.2" data-path="Direct.html">
|
||||
|
||||
<a href="Run.html">
|
||||
<a href="Direct.html">
|
||||
|
||||
|
||||
运行
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="Preview.html">
|
||||
|
||||
<a href="Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
@@ -147,12 +160,12 @@
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../Concept/">
|
||||
<li class="chapter " data-level="1.3" data-path="../Usage/">
|
||||
|
||||
<a href="../Concept/">
|
||||
<a href="../Usage/">
|
||||
|
||||
|
||||
概念
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
@@ -161,9 +174,9 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Concept/Node.html">
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Usage/Node/">
|
||||
|
||||
<a href="../Concept/Node.html">
|
||||
<a href="../Usage/Node/">
|
||||
|
||||
|
||||
节点
|
||||
@@ -172,11 +185,43 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Usage/Node/View.html">
|
||||
|
||||
<a href="../Usage/Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Concept/Spider.html">
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Usage/Node/Edit.html">
|
||||
|
||||
<a href="../Concept/Spider.html">
|
||||
<a href="../Usage/Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Usage/Spider/">
|
||||
|
||||
<a href="../Usage/Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
@@ -185,11 +230,101 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Usage/Spider/Create.html">
|
||||
|
||||
<a href="../Usage/Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Concept/Task.html">
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Concept/Task.html">
|
||||
<a href="../Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Usage/Spider/Deploy.html">
|
||||
|
||||
<a href="../Usage/Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Usage/Spider/Run.html">
|
||||
|
||||
<a href="../Usage/Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Usage/Spider/Analytics.html">
|
||||
|
||||
<a href="../Usage/Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Usage/Task/">
|
||||
|
||||
<a href="../Usage/Task/">
|
||||
|
||||
|
||||
任务
|
||||
@@ -198,14 +333,72 @@
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Concept/Deploy.html">
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Usage/Task/View.html">
|
||||
|
||||
<a href="../Concept/Deploy.html">
|
||||
<a href="../Usage/Task/View.html">
|
||||
|
||||
|
||||
部署
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Usage/Task/Action.html">
|
||||
|
||||
<a href="../Usage/Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Usage/Task/DownloadResults.html">
|
||||
|
||||
<a href="../Usage/Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Usage/Schedule/">
|
||||
|
||||
<a href="../Usage/Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Usage/Site/">
|
||||
|
||||
<a href="../Usage/Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
@@ -229,38 +422,6 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="../Architecture/Celery.html">
|
||||
|
||||
<a href="../Architecture/Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="../Architecture/App.html">
|
||||
|
||||
<a href="../Architecture/App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
@@ -268,7 +429,7 @@
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
@@ -277,9 +438,9 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../Examples/">
|
||||
<a href="../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
@@ -288,19 +449,6 @@
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
@@ -338,7 +486,7 @@
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >快速开始</a>
|
||||
<a href=".." >安装Crawlab</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
@@ -353,11 +501,12 @@
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="快速开始">快速开始</h1>
|
||||
<ul>
|
||||
<li><a href="Installation.html">安装</a></li>
|
||||
<li><a href="Run.html">运行</a></li>
|
||||
</ul>
|
||||
<p>本小节将介绍三种安装Docker的方式:</p>
|
||||
<ol>
|
||||
<li><a href="Docker.html">Docker</a></li>
|
||||
<li><a href="Direct.html">直接部署</a></li>
|
||||
<li><a href="Preview.html">预览模式</a></li>
|
||||
</ol>
|
||||
|
||||
|
||||
</section>
|
||||
@@ -385,12 +534,12 @@
|
||||
|
||||
|
||||
|
||||
<a href="../" class="navigation navigation-prev " aria-label="Previous page: 简介">
|
||||
<a href="../" class="navigation navigation-prev " aria-label="Previous page: Crawlab简介">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Installation.html" class="navigation navigation-next " aria-label="Next page: 安装">
|
||||
<a href="Docker.html" class="navigation navigation-next " aria-label="Next page: Docker">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
@@ -401,7 +550,7 @@
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"快速开始","level":"1.2","depth":1,"next":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"previous":{"title":"简介","level":"1.1","depth":1,"path":"README.md","ref":"README.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/README.md","mtime":"2019-03-28T12:02:02.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"安装Crawlab","level":"1.2","depth":1,"next":{"title":"Docker","level":"1.2.1","depth":2,"path":"Installation/Docker.md","ref":"Installation/Docker.md","articles":[]},"previous":{"title":"Crawlab简介","level":"1.1","depth":1,"path":"README.md","ref":"README.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Installation/README.md","mtime":"2019-06-16T03:12:38.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
@@ -1,447 +0,0 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>安装 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Run.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="./" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
快速开始
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter active" data-level="1.2.1" data-path="Installation.html">
|
||||
|
||||
<a href="Installation.html">
|
||||
|
||||
|
||||
安装
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="Run.html">
|
||||
|
||||
<a href="Run.html">
|
||||
|
||||
|
||||
运行
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../Concept/">
|
||||
|
||||
<a href="../Concept/">
|
||||
|
||||
|
||||
概念
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Concept/Node.html">
|
||||
|
||||
<a href="../Concept/Node.html">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Concept/Spider.html">
|
||||
|
||||
<a href="../Concept/Spider.html">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Concept/Task.html">
|
||||
|
||||
<a href="../Concept/Task.html">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Concept/Deploy.html">
|
||||
|
||||
<a href="../Concept/Deploy.html">
|
||||
|
||||
|
||||
部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
||||
|
||||
<a href="../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="../Architecture/Celery.html">
|
||||
|
||||
<a href="../Architecture/Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="../Architecture/App.html">
|
||||
|
||||
<a href="../Architecture/App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >安装</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="安装">安装</h1>
|
||||
<p>最快安装Crawlab的方式是克隆一份代码到本地</p>
|
||||
<pre><code class="lang-bash">git <span class="hljs-built_in">clone</span> https://github.com/tikazyq/crawlab
|
||||
</code></pre>
|
||||
<p>安装类库</p>
|
||||
<pre><code class="lang-bash"><span class="hljs-comment"># 安装后台类库</span>
|
||||
pip install -r requirements.txt
|
||||
</code></pre>
|
||||
<pre><code class="lang-bash"><span class="hljs-comment"># 安装前台类库</span>
|
||||
<span class="hljs-built_in">cd</span> frontend
|
||||
npm install
|
||||
</code></pre>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="./" class="navigation navigation-prev " aria-label="Previous page: 快速开始">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Run.html" class="navigation navigation-next " aria-label="Next page: 运行">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"安装","level":"1.2.1","depth":2,"next":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"previous":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Installation.md","mtime":"2019-03-28T11:55:48.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -1,476 +0,0 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>运行 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="../Concept/" />
|
||||
|
||||
|
||||
<link rel="prev" href="Installation.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
快速开始
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="Installation.html">
|
||||
|
||||
<a href="Installation.html">
|
||||
|
||||
|
||||
安装
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.2.2" data-path="Run.html">
|
||||
|
||||
<a href="Run.html">
|
||||
|
||||
|
||||
运行
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../Concept/">
|
||||
|
||||
<a href="../Concept/">
|
||||
|
||||
|
||||
概念
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Concept/Node.html">
|
||||
|
||||
<a href="../Concept/Node.html">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Concept/Spider.html">
|
||||
|
||||
<a href="../Concept/Spider.html">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Concept/Task.html">
|
||||
|
||||
<a href="../Concept/Task.html">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Concept/Deploy.html">
|
||||
|
||||
<a href="../Concept/Deploy.html">
|
||||
|
||||
|
||||
部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
||||
|
||||
<a href="../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="../Architecture/Celery.html">
|
||||
|
||||
<a href="../Architecture/Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="../Architecture/App.html">
|
||||
|
||||
<a href="../Architecture/App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >运行</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="运行">运行</h1>
|
||||
<p>在运行之前需要对Crawlab进行一些配置，配置文件为<code>config.py</code>。</p>
|
||||
<pre><code class="lang-python"><span class="hljs-comment"># project variables</span>
|
||||
PROJECT_SOURCE_FILE_FOLDER = <span class="hljs-string">'/Users/yeqing/projects/crawlab/spiders'</span> <span class="hljs-comment"># 爬虫源码根目录</span>
|
||||
PROJECT_DEPLOY_FILE_FOLDER = <span class="hljs-string">'/var/crawlab'</span> <span class="hljs-comment"># 爬虫部署根目录</span>
|
||||
PROJECT_LOGS_FOLDER = <span class="hljs-string">'/var/logs/crawlab'</span> <span class="hljs-comment"># 日志目录</span>
|
||||
PROJECT_TMP_FOLDER = <span class="hljs-string">'/tmp'</span> <span class="hljs-comment"># 临时文件目录</span>
|
||||
|
||||
<span class="hljs-comment"># celery variables</span>
|
||||
BROKER_URL = <span class="hljs-string">'redis://192.168.99.100:6379/0'</span> <span class="hljs-comment"># 中间者URL，连接redis</span>
|
||||
CELERY_RESULT_BACKEND = <span class="hljs-string">'mongodb://192.168.99.100:27017/'</span> <span class="hljs-comment"># CELERY后台URL</span>
|
||||
CELERY_MONGODB_BACKEND_SETTINGS = {
|
||||
<span class="hljs-string">'database'</span>: <span class="hljs-string">'crawlab_test'</span>,
|
||||
<span class="hljs-string">'taskmeta_collection'</span>: <span class="hljs-string">'tasks_celery'</span>,
|
||||
}
|
||||
CELERY_TIMEZONE = <span class="hljs-string">'Asia/Shanghai'</span>
|
||||
CELERY_ENABLE_UTC = <span class="hljs-keyword">True</span>
|
||||
|
||||
<span class="hljs-comment"># flower variables</span>
|
||||
FLOWER_API_ENDPOINT = <span class="hljs-string">'http://localhost:5555/api'</span> <span class="hljs-comment"># Flower服务地址</span>
|
||||
|
||||
<span class="hljs-comment"># database variables</span>
|
||||
MONGO_HOST = <span class="hljs-string">'192.168.99.100'</span>
|
||||
MONGO_PORT = <span class="hljs-number">27017</span>
|
||||
MONGO_DB = <span class="hljs-string">'crawlab_test'</span>
|
||||
|
||||
<span class="hljs-comment"># flask variables</span>
|
||||
DEBUG = <span class="hljs-keyword">True</span>
|
||||
FLASK_HOST = <span class="hljs-string">'127.0.0.1'</span>
|
||||
FLASK_PORT = <span class="hljs-number">8000</span>
|
||||
</code></pre>
|
||||
<p>启动后端API，也就是一个Flask App，可以直接启动，或者用gunicorn代替。</p>
|
||||
<pre><code class="lang-bash">python manage.py app
|
||||
</code></pre>
|
||||
<p>启动本地Worker。在其他节点中如果想只是想执行任务的话，只需要启动这一个服务就可以了。</p>
|
||||
<pre><code class="lang-bash">python manage.py worker
|
||||
</code></pre>
|
||||
<p>启动前端服务器。</p>
|
||||
<pre><code class="lang-bash"><span class="hljs-built_in">cd</span> frontend
|
||||
npm run serve
|
||||
</code></pre>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="Installation.html" class="navigation navigation-prev " aria-label="Previous page: 安装">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../Concept/" class="navigation navigation-next " aria-label="Next page: 概念">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"运行","level":"1.2.2","depth":2,"next":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"previous":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Run.md","mtime":"2019-06-04T15:46:17.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook.js"></script>
|
||||
<script src="../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
588
docs/Usage/Node/Edit.html
Normal file
588
docs/Usage/Node/Edit.html
Normal file
@@ -0,0 +1,588 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>修改节点信息 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="../Spider/" />
|
||||
|
||||
|
||||
<link rel="prev" href="View.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="View.html">
|
||||
|
||||
<a href="View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.1.2" data-path="Edit.html">
|
||||
|
||||
<a href="Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Spider/">
|
||||
|
||||
<a href="../Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Spider/Create.html">
|
||||
|
||||
<a href="../Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Spider/Deploy.html">
|
||||
|
||||
<a href="../Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Spider/Run.html">
|
||||
|
||||
<a href="../Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Spider/Analytics.html">
|
||||
|
||||
<a href="../Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Task/">
|
||||
|
||||
<a href="../Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Task/View.html">
|
||||
|
||||
<a href="../Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Task/Action.html">
|
||||
|
||||
<a href="../Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Task/DownloadResults.html">
|
||||
|
||||
<a href="../Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >修改节点信息</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="修改节点信息">修改节点信息</h2>
|
||||
<p>后面我们需要让爬虫运行在各个节点上，需要让主机与节点进行通信，因此需要知道节点的IP地址和端口。我们需要手动配置一下节点的IP和端口。在<code>节点列表</code>中点击<code>操作</code>列里的蓝色查看按钮进入到节点详情。节点详情样子如下。</p>
|
||||
<p><img src="https://crawlab.oss-cn-hangzhou.aliyuncs.com/gitbook/node-detail.png" alt=""></p>
|
||||
<p>在右侧分别输入该节点对应的<code>节点IP</code>和<code>节点端口</code>，然后点击<code>保存</code>按钮，保存该节点信息。</p>
|
||||
<p>这样，我们就完成了节点的配置工作。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="View.html" class="navigation navigation-prev " aria-label="Previous page: 查看节点列表">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../Spider/" class="navigation navigation-next " aria-label="Next page: 爬虫">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"修改节点信息","level":"1.3.1.2","depth":3,"next":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Usage/Spider/README.md","ref":"Usage/Spider/README.md","articles":[{"title":"创建爬虫","level":"1.3.2.1","depth":3,"path":"Usage/Spider/Create.md","ref":"Usage/Spider/Create.md","articles":[{"title":"自定义爬虫","level":"1.3.2.1.1","depth":4,"path":"Usage/Spider/CustomizedSpider.md","ref":"Usage/Spider/CustomizedSpider.md","articles":[]},{"title":"可配置爬虫","level":"1.3.2.1.2","depth":4,"path":"Usage/Spider/ConfigurableSpider.md","ref":"Usage/Spider/ConfigurableSpider.md","articles":[]}]},{"title":"部署爬虫","level":"1.3.2.2","depth":3,"path":"Usage/Spider/Deploy.md","ref":"Usage/Spider/Deploy.md","articles":[]},{"title":"运行爬虫","level":"1.3.2.3","depth":3,"path":"Usage/Spider/Run.md","ref":"Usage/Spider/Run.md","articles":[]},{"title":"统计数据","level":"1.3.2.4","depth":3,"path":"Usage/Spider/Analytics.md","ref":"Usage/Spider/Analytics.md","articles":[]}]},"previous":{"title":"查看节点列表","level":"1.3.1.1","depth":3,"path":"Usage/Node/View.md","ref":"Usage/Node/View.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Node/Edit.md","mtime":"2019-06-16T03:50:16.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
586
docs/Usage/Node/View.html
Normal file
586
docs/Usage/Node/View.html
Normal file
@@ -0,0 +1,586 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>查看节点列表 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Edit.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="./" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter active" data-level="1.3.1.1" data-path="View.html">
|
||||
|
||||
<a href="View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="Edit.html">
|
||||
|
||||
<a href="Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Spider/">
|
||||
|
||||
<a href="../Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Spider/Create.html">
|
||||
|
||||
<a href="../Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Spider/Deploy.html">
|
||||
|
||||
<a href="../Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Spider/Run.html">
|
||||
|
||||
<a href="../Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Spider/Analytics.html">
|
||||
|
||||
<a href="../Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Task/">
|
||||
|
||||
<a href="../Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Task/View.html">
|
||||
|
||||
<a href="../Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Task/Action.html">
|
||||
|
||||
<a href="../Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Task/DownloadResults.html">
|
||||
|
||||
<a href="../Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >查看节点列表</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="查看节点列表">查看节点列表</h2>
|
||||
<p>点击<code>侧边栏</code>的<code>节点</code>导航至<code>节点列表</code>，可以看到已上线的节点。这里的节点其实就是已经运行起来的<code>celery worker</code>，他们通过连接到配置好的<code>broker</code>（通常是<code>redis</code>）来进行与主机的通信。</p>
|
||||
<p><img src="https://crawlab.oss-cn-hangzhou.aliyuncs.com/gitbook/node-list.png" alt=""></p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="./" class="navigation navigation-prev " aria-label="Previous page: 节点">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Edit.html" class="navigation navigation-next " aria-label="Next page: 修改节点信息">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"查看节点列表","level":"1.3.1.1","depth":3,"next":{"title":"修改节点信息","level":"1.3.1.2","depth":3,"path":"Usage/Node/Edit.md","ref":"Usage/Node/Edit.md","articles":[]},"previous":{"title":"节点","level":"1.3.1","depth":2,"path":"Usage/Node/README.md","ref":"Usage/Node/README.md","articles":[{"title":"查看节点列表","level":"1.3.1.1","depth":3,"path":"Usage/Node/View.md","ref":"Usage/Node/View.md","articles":[]},{"title":"修改节点信息","level":"1.3.1.2","depth":3,"path":"Usage/Node/Edit.md","ref":"Usage/Node/Edit.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Node/View.md","mtime":"2019-06-16T03:54:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
589
docs/Usage/Node/index.html
Normal file
589
docs/Usage/Node/index.html
Normal file
@@ -0,0 +1,589 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>节点 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="View.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="../" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter active" data-level="1.3.1" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="View.html">
|
||||
|
||||
<a href="View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="Edit.html">
|
||||
|
||||
<a href="Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Spider/">
|
||||
|
||||
<a href="../Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Spider/Create.html">
|
||||
|
||||
<a href="../Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Spider/Deploy.html">
|
||||
|
||||
<a href="../Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Spider/Run.html">
|
||||
|
||||
<a href="../Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Spider/Analytics.html">
|
||||
|
||||
<a href="../Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Task/">
|
||||
|
||||
<a href="../Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Task/View.html">
|
||||
|
||||
<a href="../Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Task/Action.html">
|
||||
|
||||
<a href="../Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Task/DownloadResults.html">
|
||||
|
||||
<a href="../Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >节点</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="节点">节点</h2>
|
||||
<p>节点其实就是Celery中的Worker。一个节点运行时会连接到一个任务队列（例如Redis）来接收和运行任务。所有爬虫需要在运行时被部署到节点上，用户在部署前需要定义节点的IP地址和端口（默认为<code>localhost:8000</code>）。</p>
|
||||
<ol>
|
||||
<li><a href="View.html">查看节点</a></li>
|
||||
<li><a href="Edit.html">修改节点信息</a></li>
|
||||
</ol>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="../" class="navigation navigation-prev " aria-label="Previous page: 使用Crawlab">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="View.html" class="navigation navigation-next " aria-label="Next page: 查看节点列表">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"节点","level":"1.3.1","depth":2,"next":{"title":"查看节点列表","level":"1.3.1.1","depth":3,"path":"Usage/Node/View.md","ref":"Usage/Node/View.md","articles":[]},"previous":{"title":"使用Crawlab","level":"1.3","depth":1,"path":"Usage/README.md","ref":"Usage/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Usage/Node/README.md","ref":"Usage/Node/README.md","articles":[{"title":"查看节点列表","level":"1.3.1.1","depth":3,"path":"Usage/Node/View.md","ref":"Usage/Node/View.md","articles":[]},{"title":"修改节点信息","level":"1.3.1.2","depth":3,"path":"Usage/Node/Edit.md","ref":"Usage/Node/Edit.md","articles":[]}]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Usage/Spider/README.md","ref":"Usage/Spider/README.md","articles":[{"title":"创建爬虫","level":"1.3.2.1","depth":3,"path":"Usage/Spider/Create.md","ref":"Usage/Spider/Create.md","articles":[{"title":"自定义爬虫","level":"1.3.2.1.1","depth":4,"path":"Usage/Spider/CustomizedSpider.md","ref":"Usage/Spider/CustomizedSpider.md","articles":[]},{"title":"可配置爬虫","level":"1.3.2.1.2","depth":4,"path":"Usage/Spider/ConfigurableSpider.md","ref":"Usage/Spider/ConfigurableSpider.md","articles":[]}]},{"title":"部署爬虫","level":"1.3.2.2","depth":3,"path":"Usage/Spider/Deploy.md","ref":"Usage/Spider/Deploy.md","articles":[]},{"title":"运行爬虫","level":"1.3.2.3","depth":3,"path":"Usage/Spider/Run.md","ref":"Usage/Spider/Run.md","articles":[]},{"title":"统计数据","level":"1.3.2.4","depth":3,"path":"Usage/Spider/Analytics.md","ref":"Usage/Spider/Analytics.md","articles":[]}]},{"title":"任务","level":"1.3.3","depth":2,"path":"Usage/Task/README.md","ref":"Usage/Task/README.md","articles":[{"title":"查看任务","level":"1.3.3.1","depth":3,"path":"Usage/Task/View.md","ref":"Usage/Task/View.md","articles":[]},{"title":"操作任务","level":"1.3.3.2","depth":3,"path":"Usage/Task/Action.md","ref":"Usage/Task/Action.md","articles":[]},{"title":"下载结果","level":"1.3.3.3","depth":3,"path":"Usage/Task/DownloadResults.md","ref":"Usage/Task/DownloadResults.md","articles":[]}]},{"title":"定时任务","level":"1.3.4","depth":2,"path":"Usage/Schedule/README.md","ref":"Usage/Schedule/README.md","articles":[]},{"title":"网站","level":"1.3.5","depth":2,"path":"Usage/Site/README.md","ref":"Usage/Site/README.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Node/README.md","mtime":"2019-06-16T03:56:26.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
596
docs/Usage/Schedule/index.html
Normal file
596
docs/Usage/Schedule/index.html
Normal file
@@ -0,0 +1,596 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>定时任务 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="../Site/" />
|
||||
|
||||
|
||||
<link rel="prev" href="../Task/DownloadResults.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Node/">
|
||||
|
||||
<a href="../Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Node/View.html">
|
||||
|
||||
<a href="../Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Node/Edit.html">
|
||||
|
||||
<a href="../Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Spider/">
|
||||
|
||||
<a href="../Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Spider/Create.html">
|
||||
|
||||
<a href="../Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Spider/Deploy.html">
|
||||
|
||||
<a href="../Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Spider/Run.html">
|
||||
|
||||
<a href="../Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Spider/Analytics.html">
|
||||
|
||||
<a href="../Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Task/">
|
||||
|
||||
<a href="../Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Task/View.html">
|
||||
|
||||
<a href="../Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Task/Action.html">
|
||||
|
||||
<a href="../Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Task/DownloadResults.html">
|
||||
|
||||
<a href="../Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.4" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >定时任务</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="定时任务">定时任务</h2>
|
||||
<p>定时任务是指定某个时刻，重复性地执行的任务，英文叫做<code>Periodical Tasks</code>，在<code>Linux</code>中也被称为<code>Crontab</code>。定时任务可以让任务可以被执行多次，而用户则不用手动的操作来执行任务。在生产环境中，这非常常见。定时任务对于对增量抓取或对数据实时性有要求的用户来说非常有用。</p>
|
||||
<p>在Crawlab中，定时任务是通过<code>apscheduler</code>来实现的。创建一个定时任务之后，会在名为<code>mongo</code>的<code>jobstore</code>中创建一个<code>periodical job</code>，<code>apscheduler</code>调度引擎将会不断的去数据库中匹配任务的执行时间，如果执行时间满足要求，则会在后台触发一次任务运行。</p>
|
||||
<p>定时任务列表会进行更新。每一次爬虫更新、删除、创建，以及定时任务的更新、删除、创建，都会触发定时任务列表的更新。</p>
|
||||
<h3 id="创建定时任务">创建定时任务</h3>
|
||||
<p>导航至<code>定时任务</code>页面，可以看到定时任务的列表。</p>
|
||||
<p>点击<code>添加定时任务</code>，弹出创建定时任务的弹框。填写相应的内容，点击<code>提交</code>按钮创建定时任务。</p>
|
||||
<p><img src="https://crawlab.oss-cn-hangzhou.aliyuncs.com/gitbook/schedule-list-add.png" alt=""></p>
|
||||
<p>这里的<code>Cron</code>跟<code>Linux</code>中的<code>crontab</code>是一致的。如果对<code>crontab</code>不了解，可以参考<a href="https://www.cnblogs.com/longjshz/p/5779215.html" target="_blank">这篇文章</a>。</p>
|
||||
<h3 id="修改定时任务">修改定时任务</h3>
|
||||
<p>导航至<code>定时任务</code>页面，点击<code>操作</code>列的<code>修改</code>按钮，弹出修改定时任务的弹框。填写相应的内容，点击<code>提交</code>按钮修改定时任务。</p>
|
||||
<h3 id="删除定时任务">删除定时任务</h3>
|
||||
<p>导航至<code>定时任务</code>页面，点击<code>操作</code>列的<code>删除</code>按钮，确认删除该任务。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="../Task/DownloadResults.html" class="navigation navigation-prev " aria-label="Previous page: 下载结果">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../Site/" class="navigation navigation-next " aria-label="Next page: 网站">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"定时任务","level":"1.3.4","depth":2,"next":{"title":"网站","level":"1.3.5","depth":2,"path":"Usage/Site/README.md","ref":"Usage/Site/README.md","articles":[]},"previous":{"title":"下载结果","level":"1.3.3.3","depth":3,"path":"Usage/Task/DownloadResults.md","ref":"Usage/Task/DownloadResults.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Schedule/README.md","mtime":"2019-06-16T13:33:59.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
586
docs/Usage/Site/index.html
Normal file
586
docs/Usage/Site/index.html
Normal file
@@ -0,0 +1,586 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>网站 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="../../Architecture/" />
|
||||
|
||||
|
||||
<link rel="prev" href="../Schedule/" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Node/">
|
||||
|
||||
<a href="../Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Node/View.html">
|
||||
|
||||
<a href="../Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Node/Edit.html">
|
||||
|
||||
<a href="../Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Spider/">
|
||||
|
||||
<a href="../Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Spider/Create.html">
|
||||
|
||||
<a href="../Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Spider/Deploy.html">
|
||||
|
||||
<a href="../Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Spider/Run.html">
|
||||
|
||||
<a href="../Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Spider/Analytics.html">
|
||||
|
||||
<a href="../Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Task/">
|
||||
|
||||
<a href="../Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Task/View.html">
|
||||
|
||||
<a href="../Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Task/Action.html">
|
||||
|
||||
<a href="../Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Task/DownloadResults.html">
|
||||
|
||||
<a href="../Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.5" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >网站</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="网站">网站</h2>
|
||||
<p>网站信息是帮助用户查看<a href="http://top.chinaz.com/hangye/" target="_blank">站长之家</a>收录网站的信息的，包含<code>Robots协议</code>、<code>首页响应</code>等信息。</p>
|
||||
<p><img src="https://crawlab.oss-cn-hangzhou.aliyuncs.com/gitbook/site-list.png" alt=""></p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="../Schedule/" class="navigation navigation-prev " aria-label="Previous page: 定时任务">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../../Architecture/" class="navigation navigation-next " aria-label="Next page: 架构">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"网站","level":"1.3.5","depth":2,"next":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[]},"previous":{"title":"定时任务","level":"1.3.4","depth":2,"path":"Usage/Schedule/README.md","ref":"Usage/Schedule/README.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Site/README.md","mtime":"2019-06-16T13:37:35.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
587
docs/Usage/Spider/Analytics.html
Normal file
587
docs/Usage/Spider/Analytics.html
Normal file
@@ -0,0 +1,587 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>统计数据 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="../Task/" />
|
||||
|
||||
|
||||
<link rel="prev" href="Run.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Node/">
|
||||
|
||||
<a href="../Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Node/View.html">
|
||||
|
||||
<a href="../Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Node/Edit.html">
|
||||
|
||||
<a href="../Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="Create.html">
|
||||
|
||||
<a href="Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="CustomizedSpider.html">
|
||||
|
||||
<a href="CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="ConfigurableSpider.html">
|
||||
|
||||
<a href="ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="Deploy.html">
|
||||
|
||||
<a href="Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="Run.html">
|
||||
|
||||
<a href="Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.2.4" data-path="Analytics.html">
|
||||
|
||||
<a href="Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Task/">
|
||||
|
||||
<a href="../Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Task/View.html">
|
||||
|
||||
<a href="../Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Task/Action.html">
|
||||
|
||||
<a href="../Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Task/DownloadResults.html">
|
||||
|
||||
<a href="../Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >统计数据</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="统计数据">统计数据</h2>
|
||||
<p>在运行了一段时间之后，爬虫会积累一些统计数据，例如<code>运行成功率</code>、<code>任务数</code>、<code>运行时长</code>等指标。Crawlab将这些指标汇总并呈现给开发者。</p>
|
||||
<p>要查看统计数据的话，只需要在<code>爬虫详情</code>中，点击<code>分析</code>标签，就可以看到爬虫的统计数据了。</p>
|
||||
<p><img src="https://crawlab.oss-cn-hangzhou.aliyuncs.com/gitbook/spider-detail-analytics.png" alt=""></p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="Run.html" class="navigation navigation-prev " aria-label="Previous page: 运行爬虫">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../Task/" class="navigation navigation-next " aria-label="Next page: 任务">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"统计数据","level":"1.3.2.4","depth":3,"next":{"title":"任务","level":"1.3.3","depth":2,"path":"Usage/Task/README.md","ref":"Usage/Task/README.md","articles":[{"title":"查看任务","level":"1.3.3.1","depth":3,"path":"Usage/Task/View.md","ref":"Usage/Task/View.md","articles":[]},{"title":"操作任务","level":"1.3.3.2","depth":3,"path":"Usage/Task/Action.md","ref":"Usage/Task/Action.md","articles":[]},{"title":"下载结果","level":"1.3.3.3","depth":3,"path":"Usage/Task/DownloadResults.md","ref":"Usage/Task/DownloadResults.md","articles":[]}]},"previous":{"title":"运行爬虫","level":"1.3.2.3","depth":3,"path":"Usage/Spider/Run.md","ref":"Usage/Spider/Run.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Spider/Analytics.md","mtime":"2019-06-16T12:56:36.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
619
docs/Usage/Spider/ConfigurableSpider.html
Normal file
619
docs/Usage/Spider/ConfigurableSpider.html
Normal file
@@ -0,0 +1,619 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>可配置爬虫 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Deploy.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="CustomizedSpider.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Node/">
|
||||
|
||||
<a href="../Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Node/View.html">
|
||||
|
||||
<a href="../Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Node/Edit.html">
|
||||
|
||||
<a href="../Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="Create.html">
|
||||
|
||||
<a href="Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="CustomizedSpider.html">
|
||||
|
||||
<a href="CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.2.1.2" data-path="ConfigurableSpider.html">
|
||||
|
||||
<a href="ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="Deploy.html">
|
||||
|
||||
<a href="Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="Run.html">
|
||||
|
||||
<a href="Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="Analytics.html">
|
||||
|
||||
<a href="Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Task/">
|
||||
|
||||
<a href="../Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Task/View.html">
|
||||
|
||||
<a href="../Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Task/Action.html">
|
||||
|
||||
<a href="../Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Task/DownloadResults.html">
|
||||
|
||||
<a href="../Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >可配置爬虫</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="可配置爬虫">可配置爬虫</h2>
|
||||
<p>可配置爬虫是版本<a href="https://github.com/tikazyq/crawlab/releases/tag/v0.2.1" target="_blank">v0.2.1</a>开发的功能。目的是将具有相似网站结构的爬虫项目可配置化，将开发爬虫的过程流程化，大大提高爬虫开发效率。</p>
|
||||
<p>Crawlab的可配置爬虫是基于Scrapy的，因此天生支持并发。而且，可配置爬虫完全支持<a href="CustomizedSpider">自定义爬虫</a>的一般功能，因此也支持任务调度、任务监控、日志监控、数据分析。</p>
|
||||
<h3 id="添加爬虫">添加爬虫</h3>
|
||||
<p>在<code>侧边栏</code>点击<code>爬虫</code>导航至<code>爬虫列表</code>，点击<strong>添加爬虫</strong>按钮。</p>
|
||||
<p><img src="https://user-gold-cdn.xitu.io/2019/5/27/16af74ec408111a7?w=1662&h=702&f=png&s=98898" alt="爬虫列表"></p>
|
||||
<p>点击<strong>可配置爬虫</strong>。</p>
|
||||
<p><img src="https://user-gold-cdn.xitu.io/2019/5/27/16af74f4c75346da?w=1667&h=703&f=png&s=92067" alt="爬虫列表-添加爬虫"></p>
|
||||
<p>输入完基本信息，点击<strong>添加</strong>。</p>
|
||||
<p><img src="https://user-gold-cdn.xitu.io/2019/5/27/16af751c5d8d984d?w=1666&h=688&f=png&s=90926" alt="爬虫列表-爬虫信息"></p>
|
||||
<h3 id="配置爬虫">配置爬虫</h3>
|
||||
<p>添加完成后，可以看到刚刚添加的可配置爬虫出现了在最下方，点击<strong>查看</strong>进入到<strong>爬虫详情</strong>。</p>
|
||||
<p><img src="https://user-gold-cdn.xitu.io/2019/5/27/16af754c6f000698?w=1645&h=739&f=png&s=103908" alt=""></p>
|
||||
<p>点击<strong>配置</strong>标签进入到配置页面。接下来，我们需要对爬虫规则进行配置。</p>
|
||||
<p><img src="https://user-gold-cdn.xitu.io/2019/5/27/16af756d003eae66?w=1659&h=726&f=png&s=92224" alt=""></p>
|
||||
<p>这里已经有一些配置好的初始输入项。我们简单介绍一下各自的含义。</p>
|
||||
<h4 id="抓取类别">抓取类别</h4>
|
||||
<p>这也是爬虫抓取采用的策略，也就是爬虫遍历网页是如何进行的。作为第一个版本，我们有<strong>仅列表</strong>、<strong>仅详情页</strong>、<strong>列表+详情页</strong>。</p>
|
||||
<ul>
|
||||
<li>仅列表页。这也是最简单的形式，爬虫遍历列表上的列表项，将数据抓取下来。</li>
|
||||
<li>仅详情页。爬虫只抓取详情页。</li>
|
||||
<li>列表+详情页。爬虫先遍历列表页，将列表项中的详情页地址提取出来并跟进抓取详情页。</li>
|
||||
</ul>
|
||||
<p>这里我们选择<strong>列表+详情页</strong>。</p>
|
||||
<h4 id="列表项选择器--分页选择器">列表项选择器 & 分页选择器</h4>
|
||||
<p>列表项的匹和分页按钮的匹配查询，由CSS或XPath来进行匹配。</p>
|
||||
<h4 id="开始url">开始URL</h4>
|
||||
<p>爬虫最开始遍历的网址。</p>
|
||||
<h4 id="遵守robots协议">遵守Robots协议</h4>
|
||||
<p>这个默认是开启的。如果开启，爬虫将先抓取网站的robots.txt并判断页面是否可抓；否则，不会对此进行验证。用户可以选择将其关闭。请注意，任何无视Robots协议的行为都有法律风险。</p>
|
||||
<h4 id="列表页字段--详情页字段">列表页字段 & 详情页字段</h4>
|
||||
<p>这些都是再列表页或详情页中需要提取的字段。字段由CSS选择器或者XPath来匹配提取。可以选择文本或者属性。</p>
|
||||
<p>在检查完目标网页的元素CSS选择器之后，我们输入列表项选择器、开始URL、列表页/详情页等信息。注意勾选url为详情页URL。</p>
|
||||
<p><img src="https://user-gold-cdn.xitu.io/2019/5/27/16af7685423c7d57?w=1653&h=873&f=png&s=117230" alt=""></p>
|
||||
<p>点击保存、预览，查看预览内容。</p>
|
||||
<p><img src="https://user-gold-cdn.xitu.io/2019/5/27/16af769811d7bd0c?w=1720&h=663&f=png&s=123762" alt=""></p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="CustomizedSpider.html" class="navigation navigation-prev " aria-label="Previous page: 自定义爬虫">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Deploy.html" class="navigation navigation-next " aria-label="Next page: 部署爬虫">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"可配置爬虫","level":"1.3.2.1.2","depth":4,"next":{"title":"部署爬虫","level":"1.3.2.2","depth":3,"path":"Usage/Spider/Deploy.md","ref":"Usage/Spider/Deploy.md","articles":[]},"previous":{"title":"自定义爬虫","level":"1.3.2.1.1","depth":4,"path":"Usage/Spider/CustomizedSpider.md","ref":"Usage/Spider/CustomizedSpider.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Spider/ConfigurableSpider.md","mtime":"2019-06-16T04:28:54.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
590
docs/Usage/Spider/Create.html
Normal file
590
docs/Usage/Spider/Create.html
Normal file
@@ -0,0 +1,590 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>创建爬虫 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="CustomizedSpider.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="./" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Node/">
|
||||
|
||||
<a href="../Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Node/View.html">
|
||||
|
||||
<a href="../Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Node/Edit.html">
|
||||
|
||||
<a href="../Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter active" data-level="1.3.2.1" data-path="Create.html">
|
||||
|
||||
<a href="Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="CustomizedSpider.html">
|
||||
|
||||
<a href="CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="ConfigurableSpider.html">
|
||||
|
||||
<a href="ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="Deploy.html">
|
||||
|
||||
<a href="Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="Run.html">
|
||||
|
||||
<a href="Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="Analytics.html">
|
||||
|
||||
<a href="Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Task/">
|
||||
|
||||
<a href="../Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Task/View.html">
|
||||
|
||||
<a href="../Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Task/Action.html">
|
||||
|
||||
<a href="../Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Task/DownloadResults.html">
|
||||
|
||||
<a href="../Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >创建爬虫</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="创建爬虫">创建爬虫</h2>
|
||||
<p>Crawlab允许用户创建两种爬虫：</p>
|
||||
<ol>
|
||||
<li><a href="CustomizedSpider.html">自定义爬虫</a></li>
|
||||
<li><a href="ConfigurableSpider.html">可配置爬虫</a></li>
|
||||
</ol>
|
||||
<p>前者可以通过Web界面和创建项目目录的方式来添加，后者由于没有源代码，只能通过Web界面来添加。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="./" class="navigation navigation-prev " aria-label="Previous page: 爬虫">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="CustomizedSpider.html" class="navigation navigation-next " aria-label="Next page: 自定义爬虫">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"创建爬虫","level":"1.3.2.1","depth":3,"next":{"title":"自定义爬虫","level":"1.3.2.1.1","depth":4,"path":"Usage/Spider/CustomizedSpider.md","ref":"Usage/Spider/CustomizedSpider.md","articles":[]},"previous":{"title":"爬虫","level":"1.3.2","depth":2,"path":"Usage/Spider/README.md","ref":"Usage/Spider/README.md","articles":[{"title":"创建爬虫","level":"1.3.2.1","depth":3,"path":"Usage/Spider/Create.md","ref":"Usage/Spider/Create.md","articles":[{"title":"自定义爬虫","level":"1.3.2.1.1","depth":4,"path":"Usage/Spider/CustomizedSpider.md","ref":"Usage/Spider/CustomizedSpider.md","articles":[]},{"title":"可配置爬虫","level":"1.3.2.1.2","depth":4,"path":"Usage/Spider/ConfigurableSpider.md","ref":"Usage/Spider/ConfigurableSpider.md","articles":[]}]},{"title":"部署爬虫","level":"1.3.2.2","depth":3,"path":"Usage/Spider/Deploy.md","ref":"Usage/Spider/Deploy.md","articles":[]},{"title":"运行爬虫","level":"1.3.2.3","depth":3,"path":"Usage/Spider/Run.md","ref":"Usage/Spider/Run.md","articles":[]},{"title":"统计数据","level":"1.3.2.4","depth":3,"path":"Usage/Spider/Analytics.md","ref":"Usage/Spider/Analytics.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Spider/Create.md","mtime":"2019-06-16T04:13:44.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
602
docs/Usage/Spider/CustomizedSpider.html
Normal file
602
docs/Usage/Spider/CustomizedSpider.html
Normal file
@@ -0,0 +1,602 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>自定义爬虫 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="ConfigurableSpider.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="Create.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Node/">
|
||||
|
||||
<a href="../Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Node/View.html">
|
||||
|
||||
<a href="../Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Node/Edit.html">
|
||||
|
||||
<a href="../Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="Create.html">
|
||||
|
||||
<a href="Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter active" data-level="1.3.2.1.1" data-path="CustomizedSpider.html">
|
||||
|
||||
<a href="CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="ConfigurableSpider.html">
|
||||
|
||||
<a href="ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="Deploy.html">
|
||||
|
||||
<a href="Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="Run.html">
|
||||
|
||||
<a href="Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="Analytics.html">
|
||||
|
||||
<a href="Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Task/">
|
||||
|
||||
<a href="../Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Task/View.html">
|
||||
|
||||
<a href="../Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Task/Action.html">
|
||||
|
||||
<a href="../Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Task/DownloadResults.html">
|
||||
|
||||
<a href="../Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >自定义爬虫</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="自定义爬虫">自定义爬虫</h2>
|
||||
<p>自定义爬虫是指用户可以添加的任何语言任何框架的爬虫，高度自定义化。当用户添加好自定义爬虫之后，Crawlab就可以将其集成到爬虫管理的系统中来。</p>
|
||||
<p>自定义爬虫的添加有两种方式：</p>
|
||||
<ol>
|
||||
<li>通过Web界面上传爬虫</li>
|
||||
<li>通过创建项目目录</li>
|
||||
</ol>
|
||||
<h3 id="通过web界面上传">通过Web界面上传</h3>
|
||||
<p>在通过Web界面上传之前，需要将爬虫项目文件打包成<code>zip</code>格式。</p>
|
||||
<p><img src="https://crawlab.oss-cn-hangzhou.aliyuncs.com/gitbook/spider-list.png" alt=""></p>
|
||||
<p>然后，在<code>侧边栏</code>点击<code>爬虫</code>导航至<code>爬虫列表</code>，点击<code>添加爬虫</code>按钮，选择<code>自定义爬虫</code>，点击<code>上传</code>按钮，选择刚刚打包好的<code>zip</code>文件。上传成功后，<code>爬虫列表</code>中会出现新添加的自定义爬虫。这样就算添加好了。</p>
|
||||
<p>这个方式稍微有些繁琐，但是对于无法轻松获取服务器的读写权限时是非常有用的，适合在生产环境上使用。</p>
|
||||
<h3 id="通过添加项目目录">通过添加项目目录</h3>
|
||||
<p>Crawlab会自动发现<code>PROJECT_SOURCE_FILE_FOLDER</code>目录下的所有爬虫目录，并将这些目录生成自定义爬虫并集成到Crawlab中。因此，将爬虫项目目录拷贝到<code>PROJECT_SOURCE_FILE_FOLDER</code>目录下，就可以添加一个爬虫了。</p>
|
||||
<p>这种方式非常方便，但是需要获得主机服务器的读写权限，因而比较适合在开发环境上采用。</p>
|
||||
<h3 id="配置爬虫">配置爬虫</h3>
|
||||
<p>在定义爬虫中，我们需要配置一下<code>执行命令</code>（运行爬虫时后台执行的<code>shell</code>命令）和<code>结果集</code>（通过<code>CRAWLAB_COLLECTION</code>传递给爬虫程序，爬虫程序存储结果的地方），然后点击<code>保存</code>按钮保存爬虫信息。</p>
|
||||
<p><img src="https://crawlab.oss-cn-hangzhou.aliyuncs.com/gitbook/spider-detail-overview.png" alt=""></p>
|
||||
<p>接下来，我们就可以部署、运行自定义爬虫了。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="Create.html" class="navigation navigation-prev " aria-label="Previous page: 创建爬虫">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="ConfigurableSpider.html" class="navigation navigation-next " aria-label="Next page: 可配置爬虫">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"自定义爬虫","level":"1.3.2.1.1","depth":4,"next":{"title":"可配置爬虫","level":"1.3.2.1.2","depth":4,"path":"Usage/Spider/ConfigurableSpider.md","ref":"Usage/Spider/ConfigurableSpider.md","articles":[]},"previous":{"title":"创建爬虫","level":"1.3.2.1","depth":3,"path":"Usage/Spider/Create.md","ref":"Usage/Spider/Create.md","articles":[{"title":"自定义爬虫","level":"1.3.2.1.1","depth":4,"path":"Usage/Spider/CustomizedSpider.md","ref":"Usage/Spider/CustomizedSpider.md","articles":[]},{"title":"可配置爬虫","level":"1.3.2.1.2","depth":4,"path":"Usage/Spider/ConfigurableSpider.md","ref":"Usage/Spider/ConfigurableSpider.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Spider/CustomizedSpider.md","mtime":"2019-06-16T04:40:31.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
592
docs/Usage/Spider/Deploy.html
Normal file
592
docs/Usage/Spider/Deploy.html
Normal file
@@ -0,0 +1,592 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>部署爬虫 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Run.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="ConfigurableSpider.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Node/">
|
||||
|
||||
<a href="../Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Node/View.html">
|
||||
|
||||
<a href="../Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Node/Edit.html">
|
||||
|
||||
<a href="../Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="Create.html">
|
||||
|
||||
<a href="Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="CustomizedSpider.html">
|
||||
|
||||
<a href="CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="ConfigurableSpider.html">
|
||||
|
||||
<a href="ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.2.2" data-path="Deploy.html">
|
||||
|
||||
<a href="Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="Run.html">
|
||||
|
||||
<a href="Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="Analytics.html">
|
||||
|
||||
<a href="Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Task/">
|
||||
|
||||
<a href="../Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Task/View.html">
|
||||
|
||||
<a href="../Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Task/Action.html">
|
||||
|
||||
<a href="../Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Task/DownloadResults.html">
|
||||
|
||||
<a href="../Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >部署爬虫</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="部署爬虫">部署爬虫</h2>
|
||||
<p>这里的爬虫部署是指<a href="CustomizedSpider">自定义爬虫</a>的部署，因为<a href="ConfigurableSpider">可配置爬虫</a>已经内嵌到Crawlab中了，所有节点都可以使用，不需要额外部署。简单来说，就是将主机上的爬虫源代码通过<code>HTTP</code>的方式打包传输至<code>worker</code>节点上，因此节点就可以运行传输过来的爬虫了。</p>
|
||||
<p>部署爬虫很简单，有三种方式：</p>
|
||||
<ol>
|
||||
<li>在<code>爬虫列表</code>中点击<code>部署所有爬虫</code>，将所有爬虫部署到所有在线节点中；</li>
|
||||
<li>在<code>爬虫列表</code>中点击<code>操作</code>列的<code>部署</code>按钮，将指定爬虫部署到所有在线节点中；</li>
|
||||
<li>在<code>爬虫详情</code>的<code>概览</code>标签中，点击<code>部署</code>按钮，将指定爬虫部署到所有在线节点中。</li>
|
||||
</ol>
|
||||
<p>部署好之后，我们就可以运行爬虫了。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="ConfigurableSpider.html" class="navigation navigation-prev " aria-label="Previous page: 可配置爬虫">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Run.html" class="navigation navigation-next " aria-label="Next page: 运行爬虫">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"部署爬虫","level":"1.3.2.2","depth":3,"next":{"title":"运行爬虫","level":"1.3.2.3","depth":3,"path":"Usage/Spider/Run.md","ref":"Usage/Spider/Run.md","articles":[]},"previous":{"title":"可配置爬虫","level":"1.3.2.1.2","depth":4,"path":"Usage/Spider/ConfigurableSpider.md","ref":"Usage/Spider/ConfigurableSpider.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Spider/Deploy.md","mtime":"2019-06-16T04:35:01.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
598
docs/Usage/Spider/Run.html
Normal file
598
docs/Usage/Spider/Run.html
Normal file
@@ -0,0 +1,598 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>运行爬虫 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Analytics.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="Deploy.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Node/">
|
||||
|
||||
<a href="../Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Node/View.html">
|
||||
|
||||
<a href="../Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Node/Edit.html">
|
||||
|
||||
<a href="../Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="Create.html">
|
||||
|
||||
<a href="Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="CustomizedSpider.html">
|
||||
|
||||
<a href="CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="ConfigurableSpider.html">
|
||||
|
||||
<a href="ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="Deploy.html">
|
||||
|
||||
<a href="Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.2.3" data-path="Run.html">
|
||||
|
||||
<a href="Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="Analytics.html">
|
||||
|
||||
<a href="Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Task/">
|
||||
|
||||
<a href="../Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Task/View.html">
|
||||
|
||||
<a href="../Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Task/Action.html">
|
||||
|
||||
<a href="../Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Task/DownloadResults.html">
|
||||
|
||||
<a href="../Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >运行爬虫</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="运行爬虫">运行爬虫</h2>
|
||||
<p>我们有两种运行爬虫的方式：</p>
|
||||
<ol>
|
||||
<li>手动触发</li>
|
||||
<li>定时任务触发</li>
|
||||
</ol>
|
||||
<h3 id="手动触发">手动触发</h3>
|
||||
<ol>
|
||||
<li>在<code>爬虫列表</code>中<code>操作</code>列点击<code>运行</code>按钮，或者</li>
|
||||
<li>在<code>爬虫详情</code>中<code>概览</code>标签下点击<code>运行</code>按钮，或者</li>
|
||||
<li>对于<code>自定义爬虫</code>，可以在<code>配置</code>标签下点击<code>运行</code>按钮</li>
|
||||
</ol>
|
||||
<p>然后，Crawlab会提示任务已经派发到队列中去了，然后你可以在<code>爬虫详情</code>左侧看到新创建的任务。点击<code>创建时间</code>可以导航至<code>任务详情</code>。</p>
|
||||
<h3 id="定时任务触发">定时任务触发</h3>
|
||||
<p><code>定时任务触发</code>是比较常用的功能，对于<code>增量抓取</code>或对实时性有要求的任务很重要。这在<a href="../Schedule/">定时任务</a>中会详细介绍。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="Deploy.html" class="navigation navigation-prev " aria-label="Previous page: 部署爬虫">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Analytics.html" class="navigation navigation-next " aria-label="Next page: 统计数据">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"运行爬虫","level":"1.3.2.3","depth":3,"next":{"title":"统计数据","level":"1.3.2.4","depth":3,"path":"Usage/Spider/Analytics.md","ref":"Usage/Spider/Analytics.md","articles":[]},"previous":{"title":"部署爬虫","level":"1.3.2.2","depth":3,"path":"Usage/Spider/Deploy.md","ref":"Usage/Spider/Deploy.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Spider/Run.md","mtime":"2019-06-16T04:47:14.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
592
docs/Usage/Spider/index.html
Normal file
592
docs/Usage/Spider/index.html
Normal file
@@ -0,0 +1,592 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>爬虫 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Create.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="../Node/Edit.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Node/">
|
||||
|
||||
<a href="../Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Node/View.html">
|
||||
|
||||
<a href="../Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Node/Edit.html">
|
||||
|
||||
<a href="../Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.2" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="Create.html">
|
||||
|
||||
<a href="Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="CustomizedSpider.html">
|
||||
|
||||
<a href="CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="ConfigurableSpider.html">
|
||||
|
||||
<a href="ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="Deploy.html">
|
||||
|
||||
<a href="Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="Run.html">
|
||||
|
||||
<a href="Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="Analytics.html">
|
||||
|
||||
<a href="Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="../Task/">
|
||||
|
||||
<a href="../Task/">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="../Task/View.html">
|
||||
|
||||
<a href="../Task/View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="../Task/Action.html">
|
||||
|
||||
<a href="../Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="../Task/DownloadResults.html">
|
||||
|
||||
<a href="../Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >爬虫</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="爬虫">爬虫</h2>
|
||||
<p>爬虫就是我们通常说的网络爬虫了，本小节将介绍如下内容：</p>
|
||||
<ol>
|
||||
<li><a href="Create.html">创建爬虫</a></li>
|
||||
<li><a href="Deploy.html">部署爬虫</a></li>
|
||||
<li><a href="Run.html">运行爬虫</a></li>
|
||||
<li><a href="ConfigurableSpider.html">可配置爬虫</a></li>
|
||||
<li><a href="Analytics.html">统计数据</a></li>
|
||||
</ol>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="../Node/Edit.html" class="navigation navigation-prev " aria-label="Previous page: 修改节点信息">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Create.html" class="navigation navigation-next " aria-label="Next page: 创建爬虫">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"爬虫","level":"1.3.2","depth":2,"next":{"title":"创建爬虫","level":"1.3.2.1","depth":3,"path":"Usage/Spider/Create.md","ref":"Usage/Spider/Create.md","articles":[{"title":"自定义爬虫","level":"1.3.2.1.1","depth":4,"path":"Usage/Spider/CustomizedSpider.md","ref":"Usage/Spider/CustomizedSpider.md","articles":[]},{"title":"可配置爬虫","level":"1.3.2.1.2","depth":4,"path":"Usage/Spider/ConfigurableSpider.md","ref":"Usage/Spider/ConfigurableSpider.md","articles":[]}]},"previous":{"title":"修改节点信息","level":"1.3.1.2","depth":3,"path":"Usage/Node/Edit.md","ref":"Usage/Node/Edit.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Spider/README.md","mtime":"2019-06-16T03:56:24.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
589
docs/Usage/Task/Action.html
Normal file
589
docs/Usage/Task/Action.html
Normal file
@@ -0,0 +1,589 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>操作任务 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="DownloadResults.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="View.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Node/">
|
||||
|
||||
<a href="../Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Node/View.html">
|
||||
|
||||
<a href="../Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Node/Edit.html">
|
||||
|
||||
<a href="../Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Spider/">
|
||||
|
||||
<a href="../Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Spider/Create.html">
|
||||
|
||||
<a href="../Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Spider/Deploy.html">
|
||||
|
||||
<a href="../Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Spider/Run.html">
|
||||
|
||||
<a href="../Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Spider/Analytics.html">
|
||||
|
||||
<a href="../Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="View.html">
|
||||
|
||||
<a href="View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.3.2" data-path="Action.html">
|
||||
|
||||
<a href="Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="DownloadResults.html">
|
||||
|
||||
<a href="DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >操作任务</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="操作任务">操作任务</h2>
|
||||
<h3 id="停止任务">停止任务</h3>
|
||||
<p>当任务运行起来之后，我们因为某个原因可能需要终止任务，这时我们需要在Crawlab中停止该任务。</p>
|
||||
<p>导航至需要停止的任务的<code>任务详情</code>，点击<code>停止</code>按钮来终止任务。</p>
|
||||
<h3 id="删除任务">删除任务</h3>
|
||||
<p>在<code>任务列表</code>中，点击<code>操作</code>列中的<code>删除</code>按钮，确认删除该任务。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="View.html" class="navigation navigation-prev " aria-label="Previous page: 查看任务">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="DownloadResults.html" class="navigation navigation-next " aria-label="Next page: 下载结果">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"操作任务","level":"1.3.3.2","depth":3,"next":{"title":"下载结果","level":"1.3.3.3","depth":3,"path":"Usage/Task/DownloadResults.md","ref":"Usage/Task/DownloadResults.md","articles":[]},"previous":{"title":"查看任务","level":"1.3.3.1","depth":3,"path":"Usage/Task/View.md","ref":"Usage/Task/View.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Task/Action.md","mtime":"2019-06-16T13:15:50.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
586
docs/Usage/Task/DownloadResults.html
Normal file
586
docs/Usage/Task/DownloadResults.html
Normal file
@@ -0,0 +1,586 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>下载结果 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="../Schedule/" />
|
||||
|
||||
|
||||
<link rel="prev" href="Action.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Node/">
|
||||
|
||||
<a href="../Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Node/View.html">
|
||||
|
||||
<a href="../Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Node/Edit.html">
|
||||
|
||||
<a href="../Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Spider/">
|
||||
|
||||
<a href="../Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Spider/Create.html">
|
||||
|
||||
<a href="../Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Spider/Deploy.html">
|
||||
|
||||
<a href="../Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Spider/Run.html">
|
||||
|
||||
<a href="../Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Spider/Analytics.html">
|
||||
|
||||
<a href="../Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="View.html">
|
||||
|
||||
<a href="View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="Action.html">
|
||||
|
||||
<a href="Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.3.3" data-path="DownloadResults.html">
|
||||
|
||||
<a href="DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >下载结果</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="下载结果">下载结果</h2>
|
||||
<p>结果储存在数据库中之后，我们有时候需要将其导出，这时可以在界面中进行导出操作。</p>
|
||||
<p>导航至<code>任务详情</code>，点击<code>结果</code>标签，点击<code>下载CSV</code>按钮，等待一会儿，结果就会以<code>CSV</code>的形式下载到本地。</p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="Action.html" class="navigation navigation-prev " aria-label="Previous page: 操作任务">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../Schedule/" class="navigation navigation-next " aria-label="Next page: 定时任务">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"下载结果","level":"1.3.3.3","depth":3,"next":{"title":"定时任务","level":"1.3.4","depth":2,"path":"Usage/Schedule/README.md","ref":"Usage/Schedule/README.md","articles":[]},"previous":{"title":"操作任务","level":"1.3.3.2","depth":3,"path":"Usage/Task/Action.md","ref":"Usage/Task/Action.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Task/DownloadResults.md","mtime":"2019-06-16T13:18:17.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
594
docs/Usage/Task/View.html
Normal file
594
docs/Usage/Task/View.html
Normal file
@@ -0,0 +1,594 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>查看任务 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Action.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="./" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Node/">
|
||||
|
||||
<a href="../Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Node/View.html">
|
||||
|
||||
<a href="../Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Node/Edit.html">
|
||||
|
||||
<a href="../Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Spider/">
|
||||
|
||||
<a href="../Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Spider/Create.html">
|
||||
|
||||
<a href="../Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Spider/Deploy.html">
|
||||
|
||||
<a href="../Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Spider/Run.html">
|
||||
|
||||
<a href="../Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Spider/Analytics.html">
|
||||
|
||||
<a href="../Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter active" data-level="1.3.3.1" data-path="View.html">
|
||||
|
||||
<a href="View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="Action.html">
|
||||
|
||||
<a href="Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="DownloadResults.html">
|
||||
|
||||
<a href="DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >查看任务</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="查看任务">查看任务</h2>
|
||||
<h3 id="任务列表">任务列表</h3>
|
||||
<p>点击<code>侧边栏</code>的<code>任务</code>导航至<code>任务列表</code>。可以看到最近的10个生成的任务。可以根据<code>节点</code>、<code>爬虫</code>来过滤任务。</p>
|
||||
<p><img src="https://crawlab.oss-cn-hangzhou.aliyuncs.com/gitbook/task-list.png" alt=""></p>
|
||||
<p>点击<code>操作</code>列的<code>查看</code>按钮，进入到该任务的<code>任务详情</code>。</p>
|
||||
<h3 id="任务日志">任务日志</h3>
|
||||
<p>点击<code>日志</code>标签，可以查看任务日志。</p>
|
||||
<p><img src="https://crawlab.oss-cn-hangzhou.aliyuncs.com/gitbook/task-detail-log.png" alt=""></p>
|
||||
<h3 id="任务结果">任务结果</h3>
|
||||
<p>点击<code>结果</code>标签，可以查看任务结果。</p>
|
||||
<p><img src="https://crawlab.oss-cn-hangzhou.aliyuncs.com/gitbook/task-detail-results.png" alt=""></p>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="./" class="navigation navigation-prev " aria-label="Previous page: 任务">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Action.html" class="navigation navigation-next " aria-label="Next page: 操作任务">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"查看任务","level":"1.3.3.1","depth":3,"next":{"title":"操作任务","level":"1.3.3.2","depth":3,"path":"Usage/Task/Action.md","ref":"Usage/Task/Action.md","articles":[]},"previous":{"title":"任务","level":"1.3.3","depth":2,"path":"Usage/Task/README.md","ref":"Usage/Task/README.md","articles":[{"title":"查看任务","level":"1.3.3.1","depth":3,"path":"Usage/Task/View.md","ref":"Usage/Task/View.md","articles":[]},{"title":"操作任务","level":"1.3.3.2","depth":3,"path":"Usage/Task/Action.md","ref":"Usage/Task/Action.md","articles":[]},{"title":"下载结果","level":"1.3.3.3","depth":3,"path":"Usage/Task/DownloadResults.md","ref":"Usage/Task/DownloadResults.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Task/View.md","mtime":"2019-06-16T13:13:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
591
docs/Usage/Task/index.html
Normal file
591
docs/Usage/Task/index.html
Normal file
@@ -0,0 +1,591 @@
|
||||
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="" >
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>任务 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/style.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="HandheldFriendly" content="true"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
|
||||
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="View.html" />
|
||||
|
||||
|
||||
<link rel="prev" href="../Spider/Analytics.html" />
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="book">
|
||||
<div class="book-summary">
|
||||
|
||||
|
||||
<div id="book-search-input" role="search">
|
||||
<input type="text" placeholder="Type to search" />
|
||||
</div>
|
||||
|
||||
|
||||
<nav role="navigation">
|
||||
|
||||
|
||||
|
||||
<ul class="summary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.1" data-path="../../">
|
||||
|
||||
<a href="../../">
|
||||
|
||||
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../../Installation/">
|
||||
|
||||
<a href="../../Installation/">
|
||||
|
||||
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../../Installation/Docker.html">
|
||||
|
||||
<a href="../../Installation/Docker.html">
|
||||
|
||||
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../../Installation/Direct.html">
|
||||
|
||||
<a href="../../Installation/Direct.html">
|
||||
|
||||
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../../Installation/Preview.html">
|
||||
|
||||
<a href="../../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="../">
|
||||
|
||||
<a href="../">
|
||||
|
||||
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="../Node/">
|
||||
|
||||
<a href="../Node/">
|
||||
|
||||
|
||||
节点
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="../Node/View.html">
|
||||
|
||||
<a href="../Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="../Node/Edit.html">
|
||||
|
||||
<a href="../Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="../Spider/">
|
||||
|
||||
<a href="../Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="../Spider/Create.html">
|
||||
|
||||
<a href="../Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="../Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="../Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="../Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="../Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="../Spider/Deploy.html">
|
||||
|
||||
<a href="../Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="../Spider/Run.html">
|
||||
|
||||
<a href="../Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="../Spider/Analytics.html">
|
||||
|
||||
<a href="../Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter active" data-level="1.3.3" data-path="./">
|
||||
|
||||
<a href="./">
|
||||
|
||||
|
||||
任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="View.html">
|
||||
|
||||
<a href="View.html">
|
||||
|
||||
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="Action.html">
|
||||
|
||||
<a href="Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="DownloadResults.html">
|
||||
|
||||
<a href="DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="../Schedule/">
|
||||
|
||||
<a href="../Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="../Site/">
|
||||
|
||||
<a href="../Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4" data-path="../../Architecture/">
|
||||
|
||||
<a href="../../Architecture/">
|
||||
|
||||
|
||||
架构
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../../Examples/">
|
||||
|
||||
<a href="../../Examples/">
|
||||
|
||||
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="divider"></li>
|
||||
|
||||
<li>
|
||||
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
||||
Published with GitBook
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</nav>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="book-body">
|
||||
|
||||
<div class="body-inner">
|
||||
|
||||
|
||||
|
||||
<div class="book-header" role="navigation">
|
||||
|
||||
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="../.." >任务</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="page-wrapper" tabindex="-1" role="main">
|
||||
<div class="page-inner">
|
||||
|
||||
<div id="book-search-results">
|
||||
<div class="search-noresults">
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h2 id="任务">任务</h2>
|
||||
<p>任务其实就是指某一次抓取任务或采集任务。任务与爬虫关联，其执行的也是爬虫指定的执行命令或采集规则。抓取或采集的结果与任务关联，因此可以查看到每一次任务的结果集。Crawlab的任务是整个采集流程的核心，抓取的过程都是跟任务关联起来的，因此任务对于Crawlab来说非常重要。任务被<code>app</code>触发，<code>worker</code>通过任务队列接收任务，然后在其所在节点上执行任务。</p>
|
||||
<p>本小节将介绍以下内容：</p>
|
||||
<ol>
|
||||
<li><a href="View.html">查看任务</a></li>
|
||||
<li><a href="Delete.md">操作任务</a></li>
|
||||
<li><a href="DownloadResults.html">下载结果</a></li>
|
||||
</ol>
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
<div class="search-results">
|
||||
<div class="has-results">
|
||||
|
||||
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
||||
<ul class="search-results-list"></ul>
|
||||
|
||||
</div>
|
||||
<div class="no-results">
|
||||
|
||||
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<a href="../Spider/Analytics.html" class="navigation navigation-prev " aria-label="Previous page: 统计数据">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="View.html" class="navigation navigation-next " aria-label="Next page: 查看任务">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"任务","level":"1.3.3","depth":2,"next":{"title":"查看任务","level":"1.3.3.1","depth":3,"path":"Usage/Task/View.md","ref":"Usage/Task/View.md","articles":[]},"previous":{"title":"统计数据","level":"1.3.2.4","depth":3,"path":"Usage/Spider/Analytics.md","ref":"Usage/Spider/Analytics.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/Task/README.md","mtime":"2019-06-16T13:26:36.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"../..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook.js"></script>
|
||||
<script src="../../gitbook/theme.js"></script>
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-search/search.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
||||
|
||||
|
||||
|
||||
<script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
||||
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>概念 · GitBook</title>
|
||||
<title>使用Crawlab · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
@@ -57,10 +57,10 @@
|
||||
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="Node.html" />
|
||||
<link rel="next" href="Node/" />
|
||||
|
||||
|
||||
<link rel="prev" href="../QuickStart/Run.html" />
|
||||
<link rel="prev" href="../Installation/Preview.html" />
|
||||
|
||||
|
||||
</head>
|
||||
@@ -94,7 +94,7 @@
|
||||
<a href="../">
|
||||
|
||||
|
||||
简介
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
@@ -102,12 +102,12 @@
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="../QuickStart/">
|
||||
<li class="chapter " data-level="1.2" data-path="../Installation/">
|
||||
|
||||
<a href="../QuickStart/">
|
||||
<a href="../Installation/">
|
||||
|
||||
|
||||
快速开始
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
@@ -116,12 +116,12 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="../QuickStart/Installation.html">
|
||||
<li class="chapter " data-level="1.2.1" data-path="../Installation/Docker.html">
|
||||
|
||||
<a href="../QuickStart/Installation.html">
|
||||
<a href="../Installation/Docker.html">
|
||||
|
||||
|
||||
安装
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
@@ -129,12 +129,25 @@
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="../QuickStart/Run.html">
|
||||
<li class="chapter " data-level="1.2.2" data-path="../Installation/Direct.html">
|
||||
|
||||
<a href="../QuickStart/Run.html">
|
||||
<a href="../Installation/Direct.html">
|
||||
|
||||
|
||||
运行
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="../Installation/Preview.html">
|
||||
|
||||
<a href="../Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
@@ -152,7 +165,7 @@
|
||||
<a href="./">
|
||||
|
||||
|
||||
概念
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
@@ -161,9 +174,9 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="Node.html">
|
||||
<li class="chapter " data-level="1.3.1" data-path="Node/">
|
||||
|
||||
<a href="Node.html">
|
||||
<a href="Node/">
|
||||
|
||||
|
||||
节点
|
||||
@@ -172,11 +185,43 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="Node/View.html">
|
||||
|
||||
<a href="Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="Spider.html">
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="Node/Edit.html">
|
||||
|
||||
<a href="Spider.html">
|
||||
<a href="Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="Spider/">
|
||||
|
||||
<a href="Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
@@ -185,11 +230,101 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="Spider/Create.html">
|
||||
|
||||
<a href="Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="Task.html">
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="Task.html">
|
||||
<a href="Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="Spider/Deploy.html">
|
||||
|
||||
<a href="Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="Spider/Run.html">
|
||||
|
||||
<a href="Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="Spider/Analytics.html">
|
||||
|
||||
<a href="Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="Task/">
|
||||
|
||||
<a href="Task/">
|
||||
|
||||
|
||||
任务
|
||||
@@ -198,14 +333,72 @@
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="Deploy.html">
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="Task/View.html">
|
||||
|
||||
<a href="Deploy.html">
|
||||
<a href="Task/View.html">
|
||||
|
||||
|
||||
部署
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="Task/Action.html">
|
||||
|
||||
<a href="Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="Task/DownloadResults.html">
|
||||
|
||||
<a href="Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="Schedule/">
|
||||
|
||||
<a href="Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="Site/">
|
||||
|
||||
<a href="Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
@@ -229,38 +422,6 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="../Architecture/Celery.html">
|
||||
|
||||
<a href="../Architecture/Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="../Architecture/App.html">
|
||||
|
||||
<a href="../Architecture/App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
||||
@@ -268,7 +429,7 @@
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
@@ -277,9 +438,9 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
||||
<li class="chapter " data-level="1.5.1" data-path="../Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="../Examples/">
|
||||
<a href="../Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
@@ -288,19 +449,6 @@
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
||||
|
||||
<a href="../Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
@@ -338,7 +486,7 @@
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href=".." >概念</a>
|
||||
<a href=".." >使用Crawlab</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
@@ -353,7 +501,13 @@
|
||||
|
||||
<section class="normal markdown-section">
|
||||
|
||||
<h1 id="概念">概念</h1>
|
||||
<p>本小节将介绍如何使用Crawlab，包括如下内容：</p>
|
||||
<ol>
|
||||
<li><a href="Node/">节点</a></li>
|
||||
<li><a href="Spider/">爬虫</a></li>
|
||||
<li><a href="Task/">任务</a></li>
|
||||
<li><a href="Schedule/">定时任务</a></li>
|
||||
</ol>
|
||||
|
||||
|
||||
</section>
|
||||
@@ -381,12 +535,12 @@
|
||||
|
||||
|
||||
|
||||
<a href="../QuickStart/Run.html" class="navigation navigation-prev " aria-label="Previous page: 运行">
|
||||
<a href="../Installation/Preview.html" class="navigation navigation-prev " aria-label="Previous page: 预览模式">
|
||||
<i class="fa fa-angle-left"></i>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Node.html" class="navigation navigation-next " aria-label="Next page: 节点">
|
||||
<a href="Node/" class="navigation navigation-next " aria-label="Next page: 节点">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
@@ -397,7 +551,7 @@
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"概念","level":"1.3","depth":1,"next":{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},"previous":{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Concept/README.md","mtime":"2019-03-28T11:49:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"使用Crawlab","level":"1.3","depth":1,"next":{"title":"节点","level":"1.3.1","depth":2,"path":"Usage/Node/README.md","ref":"Usage/Node/README.md","articles":[{"title":"查看节点列表","level":"1.3.1.1","depth":3,"path":"Usage/Node/View.md","ref":"Usage/Node/View.md","articles":[]},{"title":"修改节点信息","level":"1.3.1.2","depth":3,"path":"Usage/Node/Edit.md","ref":"Usage/Node/Edit.md","articles":[]}]},"previous":{"title":"预览模式","level":"1.2.3","depth":2,"path":"Installation/Preview.md","ref":"Installation/Preview.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Usage/README.md","mtime":"2019-06-16T03:28:06.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"..","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 47 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 78 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 89 KiB |
462
docs/index.html
462
docs/index.html
@@ -4,7 +4,7 @@
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
||||
<title>简介 · GitBook</title>
|
||||
<title>Crawlab简介 · GitBook</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||||
<meta name="description" content="">
|
||||
<meta name="generator" content="GitBook 3.2.3">
|
||||
@@ -57,7 +57,7 @@
|
||||
<link rel="shortcut icon" href="gitbook/images/favicon.ico" type="image/x-icon">
|
||||
|
||||
|
||||
<link rel="next" href="QuickStart/" />
|
||||
<link rel="next" href="Installation/" />
|
||||
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@
|
||||
<a href="./">
|
||||
|
||||
|
||||
简介
|
||||
Crawlab简介
|
||||
|
||||
</a>
|
||||
|
||||
@@ -100,12 +100,12 @@
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2" data-path="QuickStart/">
|
||||
<li class="chapter " data-level="1.2" data-path="Installation/">
|
||||
|
||||
<a href="QuickStart/">
|
||||
<a href="Installation/">
|
||||
|
||||
|
||||
快速开始
|
||||
安装Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
@@ -114,12 +114,12 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.2.1" data-path="QuickStart/Installation.html">
|
||||
<li class="chapter " data-level="1.2.1" data-path="Installation/Docker.html">
|
||||
|
||||
<a href="QuickStart/Installation.html">
|
||||
<a href="Installation/Docker.html">
|
||||
|
||||
|
||||
安装
|
||||
Docker
|
||||
|
||||
</a>
|
||||
|
||||
@@ -127,12 +127,25 @@
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.2" data-path="QuickStart/Run.html">
|
||||
<li class="chapter " data-level="1.2.2" data-path="Installation/Direct.html">
|
||||
|
||||
<a href="QuickStart/Run.html">
|
||||
<a href="Installation/Direct.html">
|
||||
|
||||
|
||||
运行
|
||||
直接部署
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.2.3" data-path="Installation/Preview.html">
|
||||
|
||||
<a href="Installation/Preview.html">
|
||||
|
||||
|
||||
预览模式
|
||||
|
||||
</a>
|
||||
|
||||
@@ -145,12 +158,12 @@
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3" data-path="Concept/">
|
||||
<li class="chapter " data-level="1.3" data-path="Usage/">
|
||||
|
||||
<a href="Concept/">
|
||||
<a href="Usage/">
|
||||
|
||||
|
||||
概念
|
||||
使用Crawlab
|
||||
|
||||
</a>
|
||||
|
||||
@@ -159,9 +172,9 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1" data-path="Concept/Node.html">
|
||||
<li class="chapter " data-level="1.3.1" data-path="Usage/Node/">
|
||||
|
||||
<a href="Concept/Node.html">
|
||||
<a href="Usage/Node/">
|
||||
|
||||
|
||||
节点
|
||||
@@ -170,11 +183,43 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.1.1" data-path="Usage/Node/View.html">
|
||||
|
||||
<a href="Usage/Node/View.html">
|
||||
|
||||
|
||||
查看节点列表
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="Concept/Spider.html">
|
||||
<li class="chapter " data-level="1.3.1.2" data-path="Usage/Node/Edit.html">
|
||||
|
||||
<a href="Concept/Spider.html">
|
||||
<a href="Usage/Node/Edit.html">
|
||||
|
||||
|
||||
修改节点信息
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2" data-path="Usage/Spider/">
|
||||
|
||||
<a href="Usage/Spider/">
|
||||
|
||||
|
||||
爬虫
|
||||
@@ -183,11 +228,101 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1" data-path="Usage/Spider/Create.html">
|
||||
|
||||
<a href="Usage/Spider/Create.html">
|
||||
|
||||
|
||||
创建爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.2.1.1" data-path="Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
<a href="Usage/Spider/CustomizedSpider.html">
|
||||
|
||||
|
||||
自定义爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="Concept/Task.html">
|
||||
<li class="chapter " data-level="1.3.2.1.2" data-path="Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
<a href="Concept/Task.html">
|
||||
<a href="Usage/Spider/ConfigurableSpider.html">
|
||||
|
||||
|
||||
可配置爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.2" data-path="Usage/Spider/Deploy.html">
|
||||
|
||||
<a href="Usage/Spider/Deploy.html">
|
||||
|
||||
|
||||
部署爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.3" data-path="Usage/Spider/Run.html">
|
||||
|
||||
<a href="Usage/Spider/Run.html">
|
||||
|
||||
|
||||
运行爬虫
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.2.4" data-path="Usage/Spider/Analytics.html">
|
||||
|
||||
<a href="Usage/Spider/Analytics.html">
|
||||
|
||||
|
||||
统计数据
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3" data-path="Usage/Task/">
|
||||
|
||||
<a href="Usage/Task/">
|
||||
|
||||
|
||||
任务
|
||||
@@ -196,14 +331,72 @@
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="Concept/Deploy.html">
|
||||
<li class="chapter " data-level="1.3.3.1" data-path="Usage/Task/View.html">
|
||||
|
||||
<a href="Concept/Deploy.html">
|
||||
<a href="Usage/Task/View.html">
|
||||
|
||||
|
||||
部署
|
||||
查看任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.2" data-path="Usage/Task/Action.html">
|
||||
|
||||
<a href="Usage/Task/Action.html">
|
||||
|
||||
|
||||
操作任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.3.3" data-path="Usage/Task/DownloadResults.html">
|
||||
|
||||
<a href="Usage/Task/DownloadResults.html">
|
||||
|
||||
|
||||
下载结果
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.4" data-path="Usage/Schedule/">
|
||||
|
||||
<a href="Usage/Schedule/">
|
||||
|
||||
|
||||
定时任务
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.3.5" data-path="Usage/Site/">
|
||||
|
||||
<a href="Usage/Site/">
|
||||
|
||||
|
||||
网站
|
||||
|
||||
</a>
|
||||
|
||||
@@ -227,38 +420,6 @@
|
||||
|
||||
|
||||
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.4.1" data-path="Architecture/Celery.html">
|
||||
|
||||
<a href="Architecture/Celery.html">
|
||||
|
||||
|
||||
Celery
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.4.2" data-path="Architecture/App.html">
|
||||
|
||||
<a href="Architecture/App.html">
|
||||
|
||||
|
||||
App
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5" data-path="Examples/">
|
||||
@@ -266,7 +427,7 @@
|
||||
<a href="Examples/">
|
||||
|
||||
|
||||
Examples
|
||||
样例
|
||||
|
||||
</a>
|
||||
|
||||
@@ -275,9 +436,9 @@
|
||||
<ul class="articles">
|
||||
|
||||
|
||||
<li class="chapter " data-level="1.5.1" data-path="Examples/">
|
||||
<li class="chapter " data-level="1.5.1" data-path="Examples/ScrapyIntegration.html">
|
||||
|
||||
<a href="Examples/">
|
||||
<a href="Examples/ScrapyIntegration.html">
|
||||
|
||||
|
||||
与Scrapy集成
|
||||
@@ -286,19 +447,6 @@
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
<li class="chapter " data-level="1.5.2" data-path="Examples/">
|
||||
|
||||
<a href="Examples/">
|
||||
|
||||
|
||||
与Puppeteer集成
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
@@ -336,7 +484,7 @@
|
||||
<!-- Title -->
|
||||
<h1>
|
||||
<i class="fa fa-circle-o-notch fa-spin"></i>
|
||||
<a href="." >简介</a>
|
||||
<a href="." >Crawlab简介</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
@@ -353,162 +501,12 @@
|
||||
|
||||
<h1 id="crawlab">Crawlab</h1>
|
||||
<p>基于Celery的爬虫分布式爬虫管理平台，支持多种编程语言以及多种爬虫框架.</p>
|
||||
<p><a href="http://139.129.230.98:8080" target="_blank">查看演示 Demo</a></p>
|
||||
<p><a href="https://github.com/tikazyq/crawlab/blob/master/README.md" target="_blank">English Documentation</a></p>
|
||||
<h2 id="要求">要求</h2>
|
||||
<ul>
|
||||
<li>Python3</li>
|
||||
<li>MongoDB</li>
|
||||
<li>Redis</li>
|
||||
</ul>
|
||||
<h2 id="安装">安装</h2>
|
||||
<pre><code class="lang-bash"><span class="hljs-comment"># 安装后台类库</span>
|
||||
pip install -r requirements.txt
|
||||
</code></pre>
|
||||
<pre><code class="lang-bash"><span class="hljs-comment"># 安装前台类库</span>
|
||||
<span class="hljs-built_in">cd</span> frontend
|
||||
npm install
|
||||
</code></pre>
|
||||
<h2 id="配置">配置</h2>
|
||||
<p>请更改配置文件<code>config.py</code>，配置API和数据库连接.</p>
|
||||
<h2 id="快速开始">快速开始</h2>
|
||||
<pre><code class="lang-bash"><span class="hljs-comment"># 启动后端API</span>
|
||||
python app.py
|
||||
|
||||
<span class="hljs-comment"># 启动Flower服务</span>
|
||||
python ./bin/run_flower.py
|
||||
|
||||
<span class="hljs-comment"># 启动worker</span>
|
||||
python ./bin/run_worker.py
|
||||
</code></pre>
|
||||
<pre><code class="lang-bash"><span class="hljs-comment"># 运行前端</span>
|
||||
<span class="hljs-built_in">cd</span> frontend
|
||||
npm run serve
|
||||
</code></pre>
|
||||
<h2 id="截图">截图</h2>
|
||||
<h4 id="首页">首页</h4>
|
||||
<p><img src="img/screenshot-home.png" alt="home"></p>
|
||||
<h4 id="爬虫列表">爬虫列表</h4>
|
||||
<p><img src="img/screenshot-spiders.png" alt="spider-list"></p>
|
||||
<h4 id="爬虫详情---概览">爬虫详情 - 概览</h4>
|
||||
<p><img src="img/screenshot-spider-detail-overview.png" alt="spider-list"></p>
|
||||
<h4 id="任务详情---抓取结果">任务详情 - 抓取结果</h4>
|
||||
<p><img src="img/screenshot-task-detail-results.png" alt="spider-list"></p>
|
||||
<h2 id="架构">架构</h2>
|
||||
<p>Crawlab的架构跟Celery非常相似，但是加入了包括前端、爬虫、Flower在内的额外模块，以支持爬虫管理的功能。</p>
|
||||
<p><img src="img/crawlab-architecture.png" alt="crawlab-architecture"></p>
|
||||
<h3 id="节点">节点</h3>
|
||||
<p>节点其实就是Celery中的Worker。一个节点运行时会连接到一个任务队列（例如Redis）来接收和运行任务。所有爬虫需要在运行时被部署到节点上，用户在部署前需要定义节点的IP地址和端口。</p>
|
||||
<h3 id="爬虫">爬虫</h3>
|
||||
<h5 id="自动发现">自动发现</h5>
|
||||
<p>在<code>config.py</code>文件中，修改变量<code>PROJECT_SOURCE_FILE_FOLDER</code>作为爬虫项目所在的目录。Crawlab后台程序会自动发现这些爬虫项目并储存到数据库中。是不是很方便？</p>
|
||||
<h5 id="部署爬虫">部署爬虫</h5>
|
||||
<p>所有爬虫需要在抓取前被部署当相应当节点中。在"爬虫详情"页面点击"Deploy"按钮，爬虫将被部署到所有有效到节点中。</p>
|
||||
<h5 id="运行爬虫">运行爬虫</h5>
|
||||
<p>部署爬虫之后，你可以在"爬虫详情"页面点击"Run"按钮来启动爬虫。一个爬虫任务将被触发，你可以在任务列表页面中看到这个任务。</p>
|
||||
<h3 id="任务">任务</h3>
|
||||
<p>任务被触发并被节点执行。用户可以在任务详情页面中看到任务到状态、日志和抓取结果。</p>
|
||||
<h3 id="后台应用">后台应用</h3>
|
||||
<p>这是一个Flask应用，提供了必要的API来支持常规操作，例如CRUD、爬虫部署以及任务运行。每一个节点需要启动Flask应用来支持爬虫部署。运行<code>python manage.py app</code>或<code>python ./bin/run_app.py</code>来启动应用。</p>
|
||||
<h3 id="中间者">中间者</h3>
|
||||
<p>中间者跟Celery中定义的一样，作为运行异步任务的队列。</p>
|
||||
<h3 id="前端">前端</h3>
|
||||
<p>前端其实就是一个基于<a href="https://github.com/PanJiaChen/vue-element-admin" target="_blank">Vue-Element-Admin</a>的单页应用。其中重用了很多Element-UI的控件来支持相应的展示。</p>
|
||||
<h2 id="与其他框架的集成">与其他框架的集成</h2>
|
||||
<p>任务是利用python的<code>subprocess</code>模块中的<code>Popen</code>来实现的。任务ID将以环境变量<code>CRAWLAB_TASK_ID</code>的形式存在于爬虫任务运行的进程中，并以此来关联抓取数据。</p>
|
||||
<p>在你的爬虫程序中，你需要将<code>CRAWLAB_TASK_ID</code>的值以<code>task_id</code>作为可以存入数据库中。这样Crawlab就直到如何将爬虫任务与抓取数据关联起来了。当前，Crawlab只支持MongoDB。</p>
|
||||
<h3 id="scrapy">Scrapy</h3>
|
||||
<p>以下是Crawlab跟Scrapy集成的例子，利用了Crawlab传过来的task_id和collection_name。</p>
|
||||
<pre><code class="lang-python"><span class="hljs-keyword">import</span> os
|
||||
<span class="hljs-keyword">from</span> pymongo <span class="hljs-keyword">import</span> MongoClient
|
||||
|
||||
MONGO_HOST = <span class="hljs-string">'192.168.99.100'</span>
|
||||
MONGO_PORT = <span class="hljs-number">27017</span>
|
||||
MONGO_DB = <span class="hljs-string">'crawlab_test'</span>
|
||||
|
||||
<span class="hljs-comment"># scrapy example in the pipeline</span>
|
||||
<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">JuejinPipeline</span><span class="hljs-params">(object)</span>:</span>
|
||||
mongo = MongoClient(host=MONGO_HOST, port=MONGO_PORT)
|
||||
db = mongo[MONGO_DB]
|
||||
col_name = os.environ.get(<span class="hljs-string">'CRAWLAB_COLLECTION'</span>)
|
||||
<span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> col_name:
|
||||
col_name = <span class="hljs-string">'test'</span>
|
||||
col = db[col_name]
|
||||
|
||||
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">process_item</span><span class="hljs-params">(self, item, spider)</span>:</span>
|
||||
item[<span class="hljs-string">'task_id'</span>] = os.environ.get(<span class="hljs-string">'CRAWLAB_TASK_ID'</span>)
|
||||
self.col.save(item)
|
||||
<span class="hljs-keyword">return</span> item
|
||||
</code></pre>
|
||||
<h2 id="与其他框架比较">与其他框架比较</h2>
|
||||
<p>限制以及有一些爬虫管理框架了，因此为啥还要用Crawlab？</p>
|
||||
<p>因为很多现有当平台都依赖于Scrapyd，限制了爬虫的编程语言以及框架，爬虫工程师只能用scrapy和python。当然，scrapy是非常优秀的爬虫框架，但是它不能做一切事情。</p>
|
||||
<p>Crawlab使用起来很方便，也很通用，可以适用于几乎任何主流语言和框架。它还有一个精美的前端界面，让用户可以方便的管理和运行爬虫。</p>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="text-align:center">框架</th>
|
||||
<th style="text-align:center">类型</th>
|
||||
<th style="text-align:center">分布式</th>
|
||||
<th style="text-align:center">前端</th>
|
||||
<th style="text-align:center">依赖于Scrapyd</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td style="text-align:center"><a href="https://github.com/tikazyq/crawlab" target="_blank">Crawlab</a></td>
|
||||
<td style="text-align:center">管理平台</td>
|
||||
<td style="text-align:center">Y</td>
|
||||
<td style="text-align:center">Y</td>
|
||||
<td style="text-align:center">N</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="text-align:center"><a href="https://github.com/Gerapy/Gerapy" target="_blank">Gerapy</a></td>
|
||||
<td style="text-align:center">管理平台</td>
|
||||
<td style="text-align:center">Y</td>
|
||||
<td style="text-align:center">Y</td>
|
||||
<td style="text-align:center">Y</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="text-align:center"><a href="https://github.com/DormyMo/SpiderKeeper" target="_blank">SpiderKeeper</a></td>
|
||||
<td style="text-align:center">管理平台</td>
|
||||
<td style="text-align:center">Y</td>
|
||||
<td style="text-align:center">Y</td>
|
||||
<td style="text-align:center">Y</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="text-align:center"><a href="https://github.com/my8100/scrapydweb" target="_blank">ScrapydWeb</a></td>
|
||||
<td style="text-align:center">管理平台</td>
|
||||
<td style="text-align:center">Y</td>
|
||||
<td style="text-align:center">Y</td>
|
||||
<td style="text-align:center">Y</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="text-align:center"><a href="https://github.com/scrapy/scrapyd" target="_blank">Scrapyd</a></td>
|
||||
<td style="text-align:center">网络服务</td>
|
||||
<td style="text-align:center">Y</td>
|
||||
<td style="text-align:center">N</td>
|
||||
<td style="text-align:center">N/A</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<h2 id="todos">TODOs</h2>
|
||||
<h5 id="后端">后端</h5>
|
||||
<ul>
|
||||
<li>[ ] 文件管理</li>
|
||||
<li>[ ] MySQL数据库支持</li>
|
||||
<li>[ ] 重跑任务</li>
|
||||
<li>[ ] 节点监控</li>
|
||||
<li>[ ] 更多爬虫例子</li>
|
||||
</ul>
|
||||
<h5 id="前端">前端</h5>
|
||||
<ul>
|
||||
<li>[ ] 任务数据统计</li>
|
||||
<li>[ ] 表格过滤</li>
|
||||
<li>[x] 多语言支持 (中文)</li>
|
||||
<li>[ ] 登录和用户管理</li>
|
||||
<li>[ ] 全局搜索</li>
|
||||
</ul>
|
||||
<p><a href="http://114.67.75.98:8080" target="_blank">查看演示 Demo</a></p>
|
||||
<p>Crawlab是基于Celery的分布式爬虫管理平台，可以集成任何语言和任何框架。</p>
|
||||
<p>项目自今年三月份上线以来受到爬虫爱好者们和开发者们的好评，不少使用者还表示会用Crawlab搭建公司的爬虫平台。经过近3个月的迭代，我们陆续上线了定时任务、数据分析、网站信息、可配置爬虫、自动提取字段、下载结果、上传爬虫等功能，将Crawlab打造得更加实用，更加全面，能够真正帮助用户解决爬虫管理困难的问题。</p>
|
||||
<p>Crawlab主要解决的是大量爬虫管理困难的问题，例如需要监控上百个网站的参杂<code>scrapy</code>和<code>selenium</code>的项目不容易做到同时管理，而且命令行管理的成本非常高，还容易出错。Crawlab支持任何语言和任何框架，配合任务调度、任务监控，很容易做到对成规模的爬虫项目进行有效监控管理。</p>
|
||||
<p>本使用手册会帮助您解决在安装使用Crawlab遇到的任何问题。</p>
|
||||
<p>首先，我们来看如何安装Crawlab吧，请查看<a href="Installation/">安装</a>。</p>
|
||||
|
||||
|
||||
</section>
|
||||
@@ -537,7 +535,7 @@ MONGO_DB = <span class="hljs-string">'crawlab_test'</span>
|
||||
|
||||
|
||||
|
||||
<a href="QuickStart/" class="navigation navigation-next navigation-unique" aria-label="Next page: 快速开始">
|
||||
<a href="Installation/" class="navigation navigation-next navigation-unique" aria-label="Next page: 安装Crawlab">
|
||||
<i class="fa fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
@@ -548,7 +546,7 @@ MONGO_DB = <span class="hljs-string">'crawlab_test'</span>
|
||||
<script>
|
||||
var gitbook = gitbook || [];
|
||||
gitbook.push(function() {
|
||||
gitbook.page.hasChanged({"page":{"title":"简介","level":"1.1","depth":1,"next":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"README.md","mtime":"2019-03-28T11:44:15.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":".","book":{"language":""}});
|
||||
gitbook.page.hasChanged({"page":{"title":"Crawlab简介","level":"1.1","depth":1,"next":{"title":"安装Crawlab","level":"1.2","depth":1,"path":"Installation/README.md","ref":"Installation/README.md","articles":[{"title":"Docker","level":"1.2.1","depth":2,"path":"Installation/Docker.md","ref":"Installation/Docker.md","articles":[]},{"title":"直接部署","level":"1.2.2","depth":2,"path":"Installation/Direct.md","ref":"Installation/Direct.md","articles":[]},{"title":"预览模式","level":"1.2.3","depth":2,"path":"Installation/Preview.md","ref":"Installation/Preview.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"README.md","mtime":"2019-06-16T14:03:52.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":".","book":{"language":""}});
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user