updated docs

This commit is contained in:
Marvin Zhang
2019-06-16 22:04:16 +08:00
parent 01a1b39ad1
commit 5bca22489e
91 changed files with 15220 additions and 5995 deletions

View File

@@ -4,7 +4,7 @@
<head>
<meta charset="UTF-8">
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
<title>简介 · GitBook</title>
<title>Crawlab简介 · GitBook</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="description" content="">
<meta name="generator" content="GitBook 3.2.3">
@@ -57,7 +57,7 @@
<link rel="shortcut icon" href="gitbook/images/favicon.ico" type="image/x-icon">
<link rel="next" href="QuickStart/" />
<link rel="next" href="Installation/" />
@@ -92,7 +92,7 @@
<a href="./">
简介
Crawlab简介
</a>
@@ -100,12 +100,12 @@
</li>
<li class="chapter " data-level="1.2" data-path="QuickStart/">
<li class="chapter " data-level="1.2" data-path="Installation/">
<a href="QuickStart/">
<a href="Installation/">
快速开始
安装Crawlab
</a>
@@ -114,12 +114,12 @@
<ul class="articles">
<li class="chapter " data-level="1.2.1" data-path="QuickStart/Installation.html">
<li class="chapter " data-level="1.2.1" data-path="Installation/Docker.html">
<a href="QuickStart/Installation.html">
<a href="Installation/Docker.html">
安装
Docker
</a>
@@ -127,12 +127,25 @@
</li>
<li class="chapter " data-level="1.2.2" data-path="QuickStart/Run.html">
<li class="chapter " data-level="1.2.2" data-path="Installation/Direct.html">
<a href="QuickStart/Run.html">
<a href="Installation/Direct.html">
运行
直接部署
</a>
</li>
<li class="chapter " data-level="1.2.3" data-path="Installation/Preview.html">
<a href="Installation/Preview.html">
预览模式
</a>
@@ -145,12 +158,12 @@
</li>
<li class="chapter " data-level="1.3" data-path="Concept/">
<li class="chapter " data-level="1.3" data-path="Usage/">
<a href="Concept/">
<a href="Usage/">
概念
使用Crawlab
</a>
@@ -159,9 +172,9 @@
<ul class="articles">
<li class="chapter " data-level="1.3.1" data-path="Concept/Node.html">
<li class="chapter " data-level="1.3.1" data-path="Usage/Node/">
<a href="Concept/Node.html">
<a href="Usage/Node/">
节点
@@ -170,11 +183,43 @@
<ul class="articles">
<li class="chapter " data-level="1.3.1.1" data-path="Usage/Node/View.html">
<a href="Usage/Node/View.html">
查看节点列表
</a>
</li>
<li class="chapter " data-level="1.3.2" data-path="Concept/Spider.html">
<li class="chapter " data-level="1.3.1.2" data-path="Usage/Node/Edit.html">
<a href="Concept/Spider.html">
<a href="Usage/Node/Edit.html">
修改节点信息
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.3.2" data-path="Usage/Spider/">
<a href="Usage/Spider/">
爬虫
@@ -183,11 +228,101 @@
<ul class="articles">
<li class="chapter " data-level="1.3.2.1" data-path="Usage/Spider/Create.html">
<a href="Usage/Spider/Create.html">
创建爬虫
</a>
<ul class="articles">
<li class="chapter " data-level="1.3.2.1.1" data-path="Usage/Spider/CustomizedSpider.html">
<a href="Usage/Spider/CustomizedSpider.html">
自定义爬虫
</a>
</li>
<li class="chapter " data-level="1.3.3" data-path="Concept/Task.html">
<li class="chapter " data-level="1.3.2.1.2" data-path="Usage/Spider/ConfigurableSpider.html">
<a href="Concept/Task.html">
<a href="Usage/Spider/ConfigurableSpider.html">
可配置爬虫
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.3.2.2" data-path="Usage/Spider/Deploy.html">
<a href="Usage/Spider/Deploy.html">
部署爬虫
</a>
</li>
<li class="chapter " data-level="1.3.2.3" data-path="Usage/Spider/Run.html">
<a href="Usage/Spider/Run.html">
运行爬虫
</a>
</li>
<li class="chapter " data-level="1.3.2.4" data-path="Usage/Spider/Analytics.html">
<a href="Usage/Spider/Analytics.html">
统计数据
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.3.3" data-path="Usage/Task/">
<a href="Usage/Task/">
任务
@@ -196,14 +331,72 @@
</li>
<ul class="articles">
<li class="chapter " data-level="1.3.4" data-path="Concept/Deploy.html">
<li class="chapter " data-level="1.3.3.1" data-path="Usage/Task/View.html">
<a href="Concept/Deploy.html">
<a href="Usage/Task/View.html">
部署
查看任务
</a>
</li>
<li class="chapter " data-level="1.3.3.2" data-path="Usage/Task/Action.html">
<a href="Usage/Task/Action.html">
操作任务
</a>
</li>
<li class="chapter " data-level="1.3.3.3" data-path="Usage/Task/DownloadResults.html">
<a href="Usage/Task/DownloadResults.html">
下载结果
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.3.4" data-path="Usage/Schedule/">
<a href="Usage/Schedule/">
定时任务
</a>
</li>
<li class="chapter " data-level="1.3.5" data-path="Usage/Site/">
<a href="Usage/Site/">
网站
</a>
@@ -227,38 +420,6 @@
<ul class="articles">
<li class="chapter " data-level="1.4.1" data-path="Architecture/Celery.html">
<a href="Architecture/Celery.html">
Celery
</a>
</li>
<li class="chapter " data-level="1.4.2" data-path="Architecture/App.html">
<a href="Architecture/App.html">
App
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.5" data-path="Examples/">
@@ -266,7 +427,7 @@
<a href="Examples/">
Examples
样例
</a>
@@ -275,9 +436,9 @@
<ul class="articles">
<li class="chapter " data-level="1.5.1" data-path="Examples/">
<li class="chapter " data-level="1.5.1" data-path="Examples/ScrapyIntegration.html">
<a href="Examples/">
<a href="Examples/ScrapyIntegration.html">
与Scrapy集成
@@ -286,19 +447,6 @@
</li>
<li class="chapter " data-level="1.5.2" data-path="Examples/">
<a href="Examples/">
与Puppeteer集成
</a>
</li>
@@ -336,7 +484,7 @@
<!-- Title -->
<h1>
<i class="fa fa-circle-o-notch fa-spin"></i>
<a href="." >简介</a>
<a href="." >Crawlab简介</a>
</h1>
</div>
@@ -353,162 +501,12 @@
<h1 id="crawlab">Crawlab</h1>
<p>&#x57FA;&#x4E8E;Celery&#x7684;&#x722C;&#x866B;&#x5206;&#x5E03;&#x5F0F;&#x722C;&#x866B;&#x7BA1;&#x7406;&#x5E73;&#x53F0;&#xFF0C;&#x652F;&#x6301;&#x591A;&#x79CD;&#x7F16;&#x7A0B;&#x8BED;&#x8A00;&#x4EE5;&#x53CA;&#x591A;&#x79CD;&#x722C;&#x866B;&#x6846;&#x67B6;.</p>
<p><a href="http://139.129.230.98:8080" target="_blank">&#x67E5;&#x770B;&#x6F14;&#x793A; Demo</a></p>
<p><a href="https://github.com/tikazyq/crawlab/blob/master/README.md" target="_blank">English Documentation</a></p>
<h2 id="&#x8981;&#x6C42;">&#x8981;&#x6C42;</h2>
<ul>
<li>Python3</li>
<li>MongoDB</li>
<li>Redis</li>
</ul>
<h2 id="&#x5B89;&#x88C5;">&#x5B89;&#x88C5;</h2>
<pre><code class="lang-bash"><span class="hljs-comment"># &#x5B89;&#x88C5;&#x540E;&#x53F0;&#x7C7B;&#x5E93;</span>
pip install -r requirements.txt
</code></pre>
<pre><code class="lang-bash"><span class="hljs-comment"># &#x5B89;&#x88C5;&#x524D;&#x53F0;&#x7C7B;&#x5E93;</span>
<span class="hljs-built_in">cd</span> frontend
npm install
</code></pre>
<h2 id="&#x914D;&#x7F6E;">&#x914D;&#x7F6E;</h2>
<p>&#x8BF7;&#x66F4;&#x6539;&#x914D;&#x7F6E;&#x6587;&#x4EF6;<code>config.py</code>&#xFF0C;&#x914D;&#x7F6E;API&#x548C;&#x6570;&#x636E;&#x5E93;&#x8FDE;&#x63A5;.</p>
<h2 id="&#x5FEB;&#x901F;&#x5F00;&#x59CB;">&#x5FEB;&#x901F;&#x5F00;&#x59CB;</h2>
<pre><code class="lang-bash"><span class="hljs-comment"># &#x542F;&#x52A8;&#x540E;&#x7AEF;API</span>
python app.py
<span class="hljs-comment"># &#x542F;&#x52A8;Flower&#x670D;&#x52A1;</span>
python ./bin/run_flower.py
<span class="hljs-comment"># &#x542F;&#x52A8;worker</span>
python ./bin/run_worker.py
</code></pre>
<pre><code class="lang-bash"><span class="hljs-comment"># &#x8FD0;&#x884C;&#x524D;&#x7AEF;</span>
<span class="hljs-built_in">cd</span> frontend
npm run serve
</code></pre>
<h2 id="&#x622A;&#x56FE;">&#x622A;&#x56FE;</h2>
<h4 id="&#x9996;&#x9875;">&#x9996;&#x9875;</h4>
<p><img src="img/screenshot-home.png" alt="home"></p>
<h4 id="&#x722C;&#x866B;&#x5217;&#x8868;">&#x722C;&#x866B;&#x5217;&#x8868;</h4>
<p><img src="img/screenshot-spiders.png" alt="spider-list"></p>
<h4 id="&#x722C;&#x866B;&#x8BE6;&#x60C5;---&#x6982;&#x89C8;">&#x722C;&#x866B;&#x8BE6;&#x60C5; - &#x6982;&#x89C8;</h4>
<p><img src="img/screenshot-spider-detail-overview.png" alt="spider-list"></p>
<h4 id="&#x4EFB;&#x52A1;&#x8BE6;&#x60C5;---&#x6293;&#x53D6;&#x7ED3;&#x679C;">&#x4EFB;&#x52A1;&#x8BE6;&#x60C5; - &#x6293;&#x53D6;&#x7ED3;&#x679C;</h4>
<p><img src="img/screenshot-task-detail-results.png" alt="spider-list"></p>
<h2 id="&#x67B6;&#x6784;">&#x67B6;&#x6784;</h2>
<p>Crawlab&#x7684;&#x67B6;&#x6784;&#x8DDF;Celery&#x975E;&#x5E38;&#x76F8;&#x4F3C;&#xFF0C;&#x4F46;&#x662F;&#x52A0;&#x5165;&#x4E86;&#x5305;&#x62EC;&#x524D;&#x7AEF;&#x3001;&#x722C;&#x866B;&#x3001;Flower&#x5728;&#x5185;&#x7684;&#x989D;&#x5916;&#x6A21;&#x5757;&#xFF0C;&#x4EE5;&#x652F;&#x6301;&#x722C;&#x866B;&#x7BA1;&#x7406;&#x7684;&#x529F;&#x80FD;&#x3002;</p>
<p><img src="img/crawlab-architecture.png" alt="crawlab-architecture"></p>
<h3 id="&#x8282;&#x70B9;">&#x8282;&#x70B9;</h3>
<p>&#x8282;&#x70B9;&#x5176;&#x5B9E;&#x5C31;&#x662F;Celery&#x4E2D;&#x7684;Worker&#x3002;&#x4E00;&#x4E2A;&#x8282;&#x70B9;&#x8FD0;&#x884C;&#x65F6;&#x4F1A;&#x8FDE;&#x63A5;&#x5230;&#x4E00;&#x4E2A;&#x4EFB;&#x52A1;&#x961F;&#x5217;&#xFF08;&#x4F8B;&#x5982;Redis&#xFF09;&#x6765;&#x63A5;&#x6536;&#x548C;&#x8FD0;&#x884C;&#x4EFB;&#x52A1;&#x3002;&#x6240;&#x6709;&#x722C;&#x866B;&#x9700;&#x8981;&#x5728;&#x8FD0;&#x884C;&#x65F6;&#x88AB;&#x90E8;&#x7F72;&#x5230;&#x8282;&#x70B9;&#x4E0A;&#xFF0C;&#x7528;&#x6237;&#x5728;&#x90E8;&#x7F72;&#x524D;&#x9700;&#x8981;&#x5B9A;&#x4E49;&#x8282;&#x70B9;&#x7684;IP&#x5730;&#x5740;&#x548C;&#x7AEF;&#x53E3;&#x3002;</p>
<h3 id="&#x722C;&#x866B;">&#x722C;&#x866B;</h3>
<h5 id="&#x81EA;&#x52A8;&#x53D1;&#x73B0;">&#x81EA;&#x52A8;&#x53D1;&#x73B0;</h5>
<p>&#x5728;<code>config.py</code>&#x6587;&#x4EF6;&#x4E2D;&#xFF0C;&#x4FEE;&#x6539;&#x53D8;&#x91CF;<code>PROJECT_SOURCE_FILE_FOLDER</code>&#x4F5C;&#x4E3A;&#x722C;&#x866B;&#x9879;&#x76EE;&#x6240;&#x5728;&#x7684;&#x76EE;&#x5F55;&#x3002;Crawlab&#x540E;&#x53F0;&#x7A0B;&#x5E8F;&#x4F1A;&#x81EA;&#x52A8;&#x53D1;&#x73B0;&#x8FD9;&#x4E9B;&#x722C;&#x866B;&#x9879;&#x76EE;&#x5E76;&#x50A8;&#x5B58;&#x5230;&#x6570;&#x636E;&#x5E93;&#x4E2D;&#x3002;&#x662F;&#x4E0D;&#x662F;&#x5F88;&#x65B9;&#x4FBF;&#xFF1F;</p>
<h5 id="&#x90E8;&#x7F72;&#x722C;&#x866B;">&#x90E8;&#x7F72;&#x722C;&#x866B;</h5>
<p>&#x6240;&#x6709;&#x722C;&#x866B;&#x9700;&#x8981;&#x5728;&#x6293;&#x53D6;&#x524D;&#x88AB;&#x90E8;&#x7F72;&#x5F53;&#x76F8;&#x5E94;&#x5F53;&#x8282;&#x70B9;&#x4E2D;&#x3002;&#x5728;&quot;&#x722C;&#x866B;&#x8BE6;&#x60C5;&quot;&#x9875;&#x9762;&#x70B9;&#x51FB;&quot;Deploy&quot;&#x6309;&#x94AE;&#xFF0C;&#x722C;&#x866B;&#x5C06;&#x88AB;&#x90E8;&#x7F72;&#x5230;&#x6240;&#x6709;&#x6709;&#x6548;&#x5230;&#x8282;&#x70B9;&#x4E2D;&#x3002;</p>
<h5 id="&#x8FD0;&#x884C;&#x722C;&#x866B;">&#x8FD0;&#x884C;&#x722C;&#x866B;</h5>
<p>&#x90E8;&#x7F72;&#x722C;&#x866B;&#x4E4B;&#x540E;&#xFF0C;&#x4F60;&#x53EF;&#x4EE5;&#x5728;&quot;&#x722C;&#x866B;&#x8BE6;&#x60C5;&quot;&#x9875;&#x9762;&#x70B9;&#x51FB;&quot;Run&quot;&#x6309;&#x94AE;&#x6765;&#x542F;&#x52A8;&#x722C;&#x866B;&#x3002;&#x4E00;&#x4E2A;&#x722C;&#x866B;&#x4EFB;&#x52A1;&#x5C06;&#x88AB;&#x89E6;&#x53D1;&#xFF0C;&#x4F60;&#x53EF;&#x4EE5;&#x5728;&#x4EFB;&#x52A1;&#x5217;&#x8868;&#x9875;&#x9762;&#x4E2D;&#x770B;&#x5230;&#x8FD9;&#x4E2A;&#x4EFB;&#x52A1;&#x3002;</p>
<h3 id="&#x4EFB;&#x52A1;">&#x4EFB;&#x52A1;</h3>
<p>&#x4EFB;&#x52A1;&#x88AB;&#x89E6;&#x53D1;&#x5E76;&#x88AB;&#x8282;&#x70B9;&#x6267;&#x884C;&#x3002;&#x7528;&#x6237;&#x53EF;&#x4EE5;&#x5728;&#x4EFB;&#x52A1;&#x8BE6;&#x60C5;&#x9875;&#x9762;&#x4E2D;&#x770B;&#x5230;&#x4EFB;&#x52A1;&#x5230;&#x72B6;&#x6001;&#x3001;&#x65E5;&#x5FD7;&#x548C;&#x6293;&#x53D6;&#x7ED3;&#x679C;&#x3002;</p>
<h3 id="&#x540E;&#x53F0;&#x5E94;&#x7528;">&#x540E;&#x53F0;&#x5E94;&#x7528;</h3>
<p>&#x8FD9;&#x662F;&#x4E00;&#x4E2A;Flask&#x5E94;&#x7528;&#xFF0C;&#x63D0;&#x4F9B;&#x4E86;&#x5FC5;&#x8981;&#x7684;API&#x6765;&#x652F;&#x6301;&#x5E38;&#x89C4;&#x64CD;&#x4F5C;&#xFF0C;&#x4F8B;&#x5982;CRUD&#x3001;&#x722C;&#x866B;&#x90E8;&#x7F72;&#x4EE5;&#x53CA;&#x4EFB;&#x52A1;&#x8FD0;&#x884C;&#x3002;&#x6BCF;&#x4E00;&#x4E2A;&#x8282;&#x70B9;&#x9700;&#x8981;&#x542F;&#x52A8;Flask&#x5E94;&#x7528;&#x6765;&#x652F;&#x6301;&#x722C;&#x866B;&#x90E8;&#x7F72;&#x3002;&#x8FD0;&#x884C;<code>python manage.py app</code>&#x6216;<code>python ./bin/run_app.py</code>&#x6765;&#x542F;&#x52A8;&#x5E94;&#x7528;&#x3002;</p>
<h3 id="&#x4E2D;&#x95F4;&#x8005;">&#x4E2D;&#x95F4;&#x8005;</h3>
<p>&#x4E2D;&#x95F4;&#x8005;&#x8DDF;Celery&#x4E2D;&#x5B9A;&#x4E49;&#x7684;&#x4E00;&#x6837;&#xFF0C;&#x4F5C;&#x4E3A;&#x8FD0;&#x884C;&#x5F02;&#x6B65;&#x4EFB;&#x52A1;&#x7684;&#x961F;&#x5217;&#x3002;</p>
<h3 id="&#x524D;&#x7AEF;">&#x524D;&#x7AEF;</h3>
<p>&#x524D;&#x7AEF;&#x5176;&#x5B9E;&#x5C31;&#x662F;&#x4E00;&#x4E2A;&#x57FA;&#x4E8E;<a href="https://github.com/PanJiaChen/vue-element-admin" target="_blank">Vue-Element-Admin</a>&#x7684;&#x5355;&#x9875;&#x5E94;&#x7528;&#x3002;&#x5176;&#x4E2D;&#x91CD;&#x7528;&#x4E86;&#x5F88;&#x591A;Element-UI&#x7684;&#x63A7;&#x4EF6;&#x6765;&#x652F;&#x6301;&#x76F8;&#x5E94;&#x7684;&#x5C55;&#x793A;&#x3002;</p>
<h2 id="&#x4E0E;&#x5176;&#x4ED6;&#x6846;&#x67B6;&#x7684;&#x96C6;&#x6210;">&#x4E0E;&#x5176;&#x4ED6;&#x6846;&#x67B6;&#x7684;&#x96C6;&#x6210;</h2>
<p>&#x4EFB;&#x52A1;&#x662F;&#x5229;&#x7528;python&#x7684;<code>subprocess</code>&#x6A21;&#x5757;&#x4E2D;&#x7684;<code>Popen</code>&#x6765;&#x5B9E;&#x73B0;&#x7684;&#x3002;&#x4EFB;&#x52A1;ID&#x5C06;&#x4EE5;&#x73AF;&#x5883;&#x53D8;&#x91CF;<code>CRAWLAB_TASK_ID</code>&#x7684;&#x5F62;&#x5F0F;&#x5B58;&#x5728;&#x4E8E;&#x722C;&#x866B;&#x4EFB;&#x52A1;&#x8FD0;&#x884C;&#x7684;&#x8FDB;&#x7A0B;&#x4E2D;&#xFF0C;&#x5E76;&#x4EE5;&#x6B64;&#x6765;&#x5173;&#x8054;&#x6293;&#x53D6;&#x6570;&#x636E;&#x3002;</p>
<p>&#x5728;&#x4F60;&#x7684;&#x722C;&#x866B;&#x7A0B;&#x5E8F;&#x4E2D;&#xFF0C;&#x4F60;&#x9700;&#x8981;&#x5C06;<code>CRAWLAB_TASK_ID</code>&#x7684;&#x503C;&#x4EE5;<code>task_id</code>&#x4F5C;&#x4E3A;&#x53EF;&#x4EE5;&#x5B58;&#x5165;&#x6570;&#x636E;&#x5E93;&#x4E2D;&#x3002;&#x8FD9;&#x6837;Crawlab&#x5C31;&#x76F4;&#x5230;&#x5982;&#x4F55;&#x5C06;&#x722C;&#x866B;&#x4EFB;&#x52A1;&#x4E0E;&#x6293;&#x53D6;&#x6570;&#x636E;&#x5173;&#x8054;&#x8D77;&#x6765;&#x4E86;&#x3002;&#x5F53;&#x524D;&#xFF0C;Crawlab&#x53EA;&#x652F;&#x6301;MongoDB&#x3002;</p>
<h3 id="scrapy">Scrapy</h3>
<p>&#x4EE5;&#x4E0B;&#x662F;Crawlab&#x8DDF;Scrapy&#x96C6;&#x6210;&#x7684;&#x4F8B;&#x5B50;&#xFF0C;&#x5229;&#x7528;&#x4E86;Crawlab&#x4F20;&#x8FC7;&#x6765;&#x7684;task_id&#x548C;collection_name&#x3002;</p>
<pre><code class="lang-python"><span class="hljs-keyword">import</span> os
<span class="hljs-keyword">from</span> pymongo <span class="hljs-keyword">import</span> MongoClient
MONGO_HOST = <span class="hljs-string">&apos;192.168.99.100&apos;</span>
MONGO_PORT = <span class="hljs-number">27017</span>
MONGO_DB = <span class="hljs-string">&apos;crawlab_test&apos;</span>
<span class="hljs-comment"># scrapy example in the pipeline</span>
<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">JuejinPipeline</span><span class="hljs-params">(object)</span>:</span>
mongo = MongoClient(host=MONGO_HOST, port=MONGO_PORT)
db = mongo[MONGO_DB]
col_name = os.environ.get(<span class="hljs-string">&apos;CRAWLAB_COLLECTION&apos;</span>)
<span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> col_name:
col_name = <span class="hljs-string">&apos;test&apos;</span>
col = db[col_name]
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">process_item</span><span class="hljs-params">(self, item, spider)</span>:</span>
item[<span class="hljs-string">&apos;task_id&apos;</span>] = os.environ.get(<span class="hljs-string">&apos;CRAWLAB_TASK_ID&apos;</span>)
self.col.save(item)
<span class="hljs-keyword">return</span> item
</code></pre>
<h2 id="&#x4E0E;&#x5176;&#x4ED6;&#x6846;&#x67B6;&#x6BD4;&#x8F83;">&#x4E0E;&#x5176;&#x4ED6;&#x6846;&#x67B6;&#x6BD4;&#x8F83;</h2>
<p>&#x9650;&#x5236;&#x4EE5;&#x53CA;&#x6709;&#x4E00;&#x4E9B;&#x722C;&#x866B;&#x7BA1;&#x7406;&#x6846;&#x67B6;&#x4E86;&#xFF0C;&#x56E0;&#x6B64;&#x4E3A;&#x5565;&#x8FD8;&#x8981;&#x7528;Crawlab&#xFF1F;</p>
<p>&#x56E0;&#x4E3A;&#x5F88;&#x591A;&#x73B0;&#x6709;&#x5F53;&#x5E73;&#x53F0;&#x90FD;&#x4F9D;&#x8D56;&#x4E8E;Scrapyd&#xFF0C;&#x9650;&#x5236;&#x4E86;&#x722C;&#x866B;&#x7684;&#x7F16;&#x7A0B;&#x8BED;&#x8A00;&#x4EE5;&#x53CA;&#x6846;&#x67B6;&#xFF0C;&#x722C;&#x866B;&#x5DE5;&#x7A0B;&#x5E08;&#x53EA;&#x80FD;&#x7528;scrapy&#x548C;python&#x3002;&#x5F53;&#x7136;&#xFF0C;scrapy&#x662F;&#x975E;&#x5E38;&#x4F18;&#x79C0;&#x7684;&#x722C;&#x866B;&#x6846;&#x67B6;&#xFF0C;&#x4F46;&#x662F;&#x5B83;&#x4E0D;&#x80FD;&#x505A;&#x4E00;&#x5207;&#x4E8B;&#x60C5;&#x3002;</p>
<p>Crawlab&#x4F7F;&#x7528;&#x8D77;&#x6765;&#x5F88;&#x65B9;&#x4FBF;&#xFF0C;&#x4E5F;&#x5F88;&#x901A;&#x7528;&#xFF0C;&#x53EF;&#x4EE5;&#x9002;&#x7528;&#x4E8E;&#x51E0;&#x4E4E;&#x4EFB;&#x4F55;&#x4E3B;&#x6D41;&#x8BED;&#x8A00;&#x548C;&#x6846;&#x67B6;&#x3002;&#x5B83;&#x8FD8;&#x6709;&#x4E00;&#x4E2A;&#x7CBE;&#x7F8E;&#x7684;&#x524D;&#x7AEF;&#x754C;&#x9762;&#xFF0C;&#x8BA9;&#x7528;&#x6237;&#x53EF;&#x4EE5;&#x65B9;&#x4FBF;&#x7684;&#x7BA1;&#x7406;&#x548C;&#x8FD0;&#x884C;&#x722C;&#x866B;&#x3002;</p>
<table>
<thead>
<tr>
<th style="text-align:center">&#x6846;&#x67B6;</th>
<th style="text-align:center">&#x7C7B;&#x578B;</th>
<th style="text-align:center">&#x5206;&#x5E03;&#x5F0F;</th>
<th style="text-align:center">&#x524D;&#x7AEF;</th>
<th style="text-align:center">&#x4F9D;&#x8D56;&#x4E8E;Scrapyd</th>
</tr>
</thead>
<tbody>
<tr>
<td style="text-align:center"><a href="https://github.com/tikazyq/crawlab" target="_blank">Crawlab</a></td>
<td style="text-align:center">&#x7BA1;&#x7406;&#x5E73;&#x53F0;</td>
<td style="text-align:center">Y</td>
<td style="text-align:center">Y</td>
<td style="text-align:center">N</td>
</tr>
<tr>
<td style="text-align:center"><a href="https://github.com/Gerapy/Gerapy" target="_blank">Gerapy</a></td>
<td style="text-align:center">&#x7BA1;&#x7406;&#x5E73;&#x53F0;</td>
<td style="text-align:center">Y</td>
<td style="text-align:center">Y</td>
<td style="text-align:center">Y</td>
</tr>
<tr>
<td style="text-align:center"><a href="https://github.com/DormyMo/SpiderKeeper" target="_blank">SpiderKeeper</a></td>
<td style="text-align:center">&#x7BA1;&#x7406;&#x5E73;&#x53F0;</td>
<td style="text-align:center">Y</td>
<td style="text-align:center">Y</td>
<td style="text-align:center">Y</td>
</tr>
<tr>
<td style="text-align:center"><a href="https://github.com/my8100/scrapydweb" target="_blank">ScrapydWeb</a></td>
<td style="text-align:center">&#x7BA1;&#x7406;&#x5E73;&#x53F0;</td>
<td style="text-align:center">Y</td>
<td style="text-align:center">Y</td>
<td style="text-align:center">Y</td>
</tr>
<tr>
<td style="text-align:center"><a href="https://github.com/scrapy/scrapyd" target="_blank">Scrapyd</a></td>
<td style="text-align:center">&#x7F51;&#x7EDC;&#x670D;&#x52A1;</td>
<td style="text-align:center">Y</td>
<td style="text-align:center">N</td>
<td style="text-align:center">N/A</td>
</tr>
</tbody>
</table>
<h2 id="todos">TODOs</h2>
<h5 id="&#x540E;&#x7AEF;">&#x540E;&#x7AEF;</h5>
<ul>
<li>[ ] &#x6587;&#x4EF6;&#x7BA1;&#x7406;</li>
<li>[ ] MySQL&#x6570;&#x636E;&#x5E93;&#x652F;&#x6301;</li>
<li>[ ] &#x91CD;&#x8DD1;&#x4EFB;&#x52A1;</li>
<li>[ ] &#x8282;&#x70B9;&#x76D1;&#x63A7;</li>
<li>[ ] &#x66F4;&#x591A;&#x722C;&#x866B;&#x4F8B;&#x5B50;</li>
</ul>
<h5 id="&#x524D;&#x7AEF;">&#x524D;&#x7AEF;</h5>
<ul>
<li>[ ] &#x4EFB;&#x52A1;&#x6570;&#x636E;&#x7EDF;&#x8BA1;</li>
<li>[ ] &#x8868;&#x683C;&#x8FC7;&#x6EE4;</li>
<li>[x] &#x591A;&#x8BED;&#x8A00;&#x652F;&#x6301; (&#x4E2D;&#x6587;)</li>
<li>[ ] &#x767B;&#x5F55;&#x548C;&#x7528;&#x6237;&#x7BA1;&#x7406;</li>
<li>[ ] &#x5168;&#x5C40;&#x641C;&#x7D22;</li>
</ul>
<p><a href="http://114.67.75.98:8080" target="_blank">&#x67E5;&#x770B;&#x6F14;&#x793A; Demo</a></p>
<p>Crawlab&#x662F;&#x57FA;&#x4E8E;Celery&#x7684;&#x5206;&#x5E03;&#x5F0F;&#x722C;&#x866B;&#x7BA1;&#x7406;&#x5E73;&#x53F0;&#xFF0C;&#x53EF;&#x4EE5;&#x96C6;&#x6210;&#x4EFB;&#x4F55;&#x8BED;&#x8A00;&#x548C;&#x4EFB;&#x4F55;&#x6846;&#x67B6;&#x3002;</p>
<p>&#x9879;&#x76EE;&#x81EA;&#x4ECA;&#x5E74;&#x4E09;&#x6708;&#x4EFD;&#x4E0A;&#x7EBF;&#x4EE5;&#x6765;&#x53D7;&#x5230;&#x722C;&#x866B;&#x7231;&#x597D;&#x8005;&#x4EEC;&#x548C;&#x5F00;&#x53D1;&#x8005;&#x4EEC;&#x7684;&#x597D;&#x8BC4;&#xFF0C;&#x4E0D;&#x5C11;&#x4F7F;&#x7528;&#x8005;&#x8FD8;&#x8868;&#x793A;&#x4F1A;&#x7528;Crawlab&#x642D;&#x5EFA;&#x516C;&#x53F8;&#x7684;&#x722C;&#x866B;&#x5E73;&#x53F0;&#x3002;&#x7ECF;&#x8FC7;&#x8FD1;3&#x4E2A;&#x6708;&#x7684;&#x8FED;&#x4EE3;&#xFF0C;&#x6211;&#x4EEC;&#x9646;&#x7EED;&#x4E0A;&#x7EBF;&#x4E86;&#x5B9A;&#x65F6;&#x4EFB;&#x52A1;&#x3001;&#x6570;&#x636E;&#x5206;&#x6790;&#x3001;&#x7F51;&#x7AD9;&#x4FE1;&#x606F;&#x3001;&#x53EF;&#x914D;&#x7F6E;&#x722C;&#x866B;&#x3001;&#x81EA;&#x52A8;&#x63D0;&#x53D6;&#x5B57;&#x6BB5;&#x3001;&#x4E0B;&#x8F7D;&#x7ED3;&#x679C;&#x3001;&#x4E0A;&#x4F20;&#x722C;&#x866B;&#x7B49;&#x529F;&#x80FD;&#xFF0C;&#x5C06;Crawlab&#x6253;&#x9020;&#x5F97;&#x66F4;&#x52A0;&#x5B9E;&#x7528;&#xFF0C;&#x66F4;&#x52A0;&#x5168;&#x9762;&#xFF0C;&#x80FD;&#x591F;&#x771F;&#x6B63;&#x5E2E;&#x52A9;&#x7528;&#x6237;&#x89E3;&#x51B3;&#x722C;&#x866B;&#x7BA1;&#x7406;&#x56F0;&#x96BE;&#x7684;&#x95EE;&#x9898;&#x3002;</p>
<p>Crawlab&#x4E3B;&#x8981;&#x89E3;&#x51B3;&#x7684;&#x662F;&#x5927;&#x91CF;&#x722C;&#x866B;&#x7BA1;&#x7406;&#x56F0;&#x96BE;&#x7684;&#x95EE;&#x9898;&#xFF0C;&#x4F8B;&#x5982;&#x9700;&#x8981;&#x76D1;&#x63A7;&#x4E0A;&#x767E;&#x4E2A;&#x7F51;&#x7AD9;&#x7684;&#x53C2;&#x6742;<code>scrapy</code>&#x548C;<code>selenium</code>&#x7684;&#x9879;&#x76EE;&#x4E0D;&#x5BB9;&#x6613;&#x505A;&#x5230;&#x540C;&#x65F6;&#x7BA1;&#x7406;&#xFF0C;&#x800C;&#x4E14;&#x547D;&#x4EE4;&#x884C;&#x7BA1;&#x7406;&#x7684;&#x6210;&#x672C;&#x975E;&#x5E38;&#x9AD8;&#xFF0C;&#x8FD8;&#x5BB9;&#x6613;&#x51FA;&#x9519;&#x3002;Crawlab&#x652F;&#x6301;&#x4EFB;&#x4F55;&#x8BED;&#x8A00;&#x548C;&#x4EFB;&#x4F55;&#x6846;&#x67B6;&#xFF0C;&#x914D;&#x5408;&#x4EFB;&#x52A1;&#x8C03;&#x5EA6;&#x3001;&#x4EFB;&#x52A1;&#x76D1;&#x63A7;&#xFF0C;&#x5F88;&#x5BB9;&#x6613;&#x505A;&#x5230;&#x5BF9;&#x6210;&#x89C4;&#x6A21;&#x7684;&#x722C;&#x866B;&#x9879;&#x76EE;&#x8FDB;&#x884C;&#x6709;&#x6548;&#x76D1;&#x63A7;&#x7BA1;&#x7406;&#x3002;</p>
<p>&#x672C;&#x4F7F;&#x7528;&#x624B;&#x518C;&#x4F1A;&#x5E2E;&#x52A9;&#x60A8;&#x89E3;&#x51B3;&#x5728;&#x5B89;&#x88C5;&#x4F7F;&#x7528;Crawlab&#x9047;&#x5230;&#x7684;&#x4EFB;&#x4F55;&#x95EE;&#x9898;&#x3002;</p>
<p>&#x9996;&#x5148;&#xFF0C;&#x6211;&#x4EEC;&#x6765;&#x770B;&#x5982;&#x4F55;&#x5B89;&#x88C5;Crawlab&#x5427;&#xFF0C;&#x8BF7;&#x67E5;&#x770B;<a href="Installation/">&#x5B89;&#x88C5;</a>&#x3002;</p>
</section>
@@ -537,7 +535,7 @@ MONGO_DB = <span class="hljs-string">&apos;crawlab_test&apos;</span>
<a href="QuickStart/" class="navigation navigation-next navigation-unique" aria-label="Next page: 快速开始">
<a href="Installation/" class="navigation navigation-next navigation-unique" aria-label="Next page: 安装Crawlab">
<i class="fa fa-angle-right"></i>
</a>
@@ -548,7 +546,7 @@ MONGO_DB = <span class="hljs-string">&apos;crawlab_test&apos;</span>
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"简介","level":"1.1","depth":1,"next":{"title":"快速开始","level":"1.2","depth":1,"path":"QuickStart/README.md","ref":"QuickStart/README.md","articles":[{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},{"title":"运行","level":"1.2.2","depth":2,"path":"QuickStart/Run.md","ref":"QuickStart/Run.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"README.md","mtime":"2019-03-28T11:44:15.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":".","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"Crawlab简介","level":"1.1","depth":1,"next":{"title":"安装Crawlab","level":"1.2","depth":1,"path":"Installation/README.md","ref":"Installation/README.md","articles":[{"title":"Docker","level":"1.2.1","depth":2,"path":"Installation/Docker.md","ref":"Installation/Docker.md","articles":[]},{"title":"直接部署","level":"1.2.2","depth":2,"path":"Installation/Direct.md","ref":"Installation/Direct.md","articles":[]},{"title":"预览模式","level":"1.2.3","depth":2,"path":"Installation/Preview.md","ref":"Installation/Preview.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"README.md","mtime":"2019-06-16T14:03:52.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":".","book":{"language":""}});
});
</script>
</div>