git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

updated docs

Browse Source
This commit is contained in:
Marvin Zhang
2019-06-16 22:04:16 +08:00
parent 99ccee9d50
commit 0b40fab625
91 changed files with 15220 additions and 5995 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
docs/Examples/PuppeteerIntegration.md Normal file
View File

600
docs/Examples/ScrapyIntegration.html Normal file
View File

@@ -0,0 +1,600 @@
<!DOCTYPE HTML>
<html lang="" >
<head>
<meta charset="UTF-8">
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
<title>与Scrapy集成 · GitBook</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="description" content="">
<meta name="generator" content="GitBook 3.2.3">
<link rel="stylesheet" href="../gitbook/style.css">
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
<meta name="HandheldFriendly" content="true"/>
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="black">
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
<link rel="prev" href="./" />
</head>
<body>
<div class="book">
<div class="book-summary">
<div id="book-search-input" role="search">
<input type="text" placeholder="Type to search" />
</div>
<nav role="navigation">
<ul class="summary">
<li class="chapter " data-level="1.1" data-path="../">
<a href="../">
Crawlab简介
</a>
</li>
<li class="chapter " data-level="1.2" data-path="../Installation/">
<a href="../Installation/">
安装Crawlab
</a>
<ul class="articles">
<li class="chapter " data-level="1.2.1" data-path="../Installation/Docker.html">
<a href="../Installation/Docker.html">
Docker
</a>
</li>
<li class="chapter " data-level="1.2.2" data-path="../Installation/Direct.html">
<a href="../Installation/Direct.html">
直接部署
</a>
</li>
<li class="chapter " data-level="1.2.3" data-path="../Installation/Preview.html">
<a href="../Installation/Preview.html">
预览模式
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.3" data-path="../Usage/">
<a href="../Usage/">
使用Crawlab
</a>
<ul class="articles">
<li class="chapter " data-level="1.3.1" data-path="../Usage/Node/">
<a href="../Usage/Node/">
节点
</a>
<ul class="articles">
<li class="chapter " data-level="1.3.1.1" data-path="../Usage/Node/View.html">
<a href="../Usage/Node/View.html">
查看节点列表
</a>
</li>
<li class="chapter " data-level="1.3.1.2" data-path="../Usage/Node/Edit.html">
<a href="../Usage/Node/Edit.html">
修改节点信息
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.3.2" data-path="../Usage/Spider/">
<a href="../Usage/Spider/">
爬虫
</a>
<ul class="articles">
<li class="chapter " data-level="1.3.2.1" data-path="../Usage/Spider/Create.html">
<a href="../Usage/Spider/Create.html">
创建爬虫
</a>
<ul class="articles">
<li class="chapter " data-level="1.3.2.1.1" data-path="../Usage/Spider/CustomizedSpider.html">
<a href="../Usage/Spider/CustomizedSpider.html">
自定义爬虫
</a>
</li>
<li class="chapter " data-level="1.3.2.1.2" data-path="../Usage/Spider/ConfigurableSpider.html">
<a href="../Usage/Spider/ConfigurableSpider.html">
可配置爬虫
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.3.2.2" data-path="../Usage/Spider/Deploy.html">
<a href="../Usage/Spider/Deploy.html">
部署爬虫
</a>
</li>
<li class="chapter " data-level="1.3.2.3" data-path="../Usage/Spider/Run.html">
<a href="../Usage/Spider/Run.html">
运行爬虫
</a>
</li>
<li class="chapter " data-level="1.3.2.4" data-path="../Usage/Spider/Analytics.html">
<a href="../Usage/Spider/Analytics.html">
统计数据
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.3.3" data-path="../Usage/Task/">
<a href="../Usage/Task/">
任务
</a>
<ul class="articles">
<li class="chapter " data-level="1.3.3.1" data-path="../Usage/Task/View.html">
<a href="../Usage/Task/View.html">
查看任务
</a>
</li>
<li class="chapter " data-level="1.3.3.2" data-path="../Usage/Task/Action.html">
<a href="../Usage/Task/Action.html">
操作任务
</a>
</li>
<li class="chapter " data-level="1.3.3.3" data-path="../Usage/Task/DownloadResults.html">
<a href="../Usage/Task/DownloadResults.html">
下载结果
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.3.4" data-path="../Usage/Schedule/">
<a href="../Usage/Schedule/">
定时任务
</a>
</li>
<li class="chapter " data-level="1.3.5" data-path="../Usage/Site/">
<a href="../Usage/Site/">
网站
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.4" data-path="../Architecture/">
<a href="../Architecture/">
架构
</a>
</li>
<li class="chapter " data-level="1.5" data-path="./">
<a href="./">
样例
</a>
<ul class="articles">
<li class="chapter active" data-level="1.5.1" data-path="ScrapyIntegration.html">
<a href="ScrapyIntegration.html">
与Scrapy集成
</a>
</li>
</ul>
</li>
<li class="divider"></li>
<li>
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
Published with GitBook
</a>
</li>
</ul>
</nav>
</div>
<div class="book-body">
<div class="body-inner">
<div class="book-header" role="navigation">
<!-- Title -->
<h1>
<i class="fa fa-circle-o-notch fa-spin"></i>
<a href=".." >与Scrapy集成</a>
</h1>
</div>
<div class="page-wrapper" tabindex="-1" role="main">
<div class="page-inner">
<div id="book-search-results">
<div class="search-noresults">
<section class="normal markdown-section">
<h3 id="&#x4E0E;scrapy&#x96C6;&#x6210;">&#x4E0E;Scrapy&#x96C6;&#x6210;</h3>
<p>&#x4EE5;&#x4E0B;&#x662F;Crawlab&#x8DDF;<code>Scrapy</code>&#x96C6;&#x6210;&#x7684;&#x4F8B;&#x5B50;&#xFF0C;&#x5229;&#x7528;&#x4E86;Crawlab&#x4F20;&#x8FC7;&#x6765;&#x7684;<code>task_id</code>&#x548C;<code>collection_name</code>&#x3002;</p>
<pre><code class="lang-python"><span class="hljs-keyword">import</span> os
<span class="hljs-keyword">from</span> pymongo <span class="hljs-keyword">import</span> MongoClient
MONGO_HOST = <span class="hljs-string">&apos;192.168.99.100&apos;</span>
MONGO_PORT = <span class="hljs-number">27017</span>
MONGO_DB = <span class="hljs-string">&apos;crawlab_test&apos;</span>
<span class="hljs-comment"># scrapy example in the pipeline</span>
<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">JuejinPipeline</span><span class="hljs-params">(object)</span>:</span>
mongo = MongoClient(host=MONGO_HOST, port=MONGO_PORT)
db = mongo[MONGO_DB]
col_name = os.environ.get(<span class="hljs-string">&apos;CRAWLAB_COLLECTION&apos;</span>)
<span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> col_name:
col_name = <span class="hljs-string">&apos;test&apos;</span>
col = db[col_name]
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">process_item</span><span class="hljs-params">(self, item, spider)</span>:</span>
item[<span class="hljs-string">&apos;task_id&apos;</span>] = os.environ.get(<span class="hljs-string">&apos;CRAWLAB_TASK_ID&apos;</span>)
self.col.save(item)
<span class="hljs-keyword">return</span> item
</code></pre>
</section>
</div>
<div class="search-results">
<div class="has-results">
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
<ul class="search-results-list"></ul>
</div>
<div class="no-results">
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
</div>
</div>
</div>
</div>
</div>
</div>
<a href="./" class="navigation navigation-prev navigation-unique" aria-label="Previous page: 样例">
<i class="fa fa-angle-left"></i>
</a>
</div>
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"与Scrapy集成","level":"1.5.1","depth":2,"previous":{"title":"样例","level":"1.5","depth":1,"path":"Examples/README.md","ref":"Examples/README.md","articles":[{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/ScrapyIntegration.md","ref":"Examples/ScrapyIntegration.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Examples/ScrapyIntegration.md","mtime":"2019-06-16T13:46:53.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>
<script src="../gitbook/gitbook.js"></script>
<script src="../gitbook/theme.js"></script>
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
</body>
</html>

309
docs/Examples/index.html
View File

@@ -4,7 +4,7 @@
<head>
<meta charset="UTF-8">
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
<title>Examples · GitBook</title>
<title>样例 · GitBook</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="description" content="">
<meta name="generator" content="GitBook 3.2.3">
@@ -57,10 +57,10 @@
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
<link rel="next" href="./" />
<link rel="next" href="ScrapyIntegration.html" />
<link rel="prev" href="../Architecture/App.html" />
<link rel="prev" href="../Architecture/" />
</head>
@@ -94,7 +94,7 @@
<a href="../">
简介
Crawlab简介
</a>
@@ -102,12 +102,12 @@
</li>
<li class="chapter " data-level="1.2" data-path="../QuickStart/">
<li class="chapter " data-level="1.2" data-path="../Installation/">
<a href="../QuickStart/">
<a href="../Installation/">
快速开始
安装Crawlab
</a>
@@ -116,12 +116,12 @@
<ul class="articles">
<li class="chapter " data-level="1.2.1" data-path="../QuickStart/Installation.html">
<li class="chapter " data-level="1.2.1" data-path="../Installation/Docker.html">
<a href="../QuickStart/Installation.html">
<a href="../Installation/Docker.html">
安装
Docker
</a>
@@ -129,12 +129,25 @@
</li>
<li class="chapter " data-level="1.2.2" data-path="../QuickStart/Run.html">
<li class="chapter " data-level="1.2.2" data-path="../Installation/Direct.html">
<a href="../QuickStart/Run.html">
<a href="../Installation/Direct.html">
运行
直接部署
</a>
</li>
<li class="chapter " data-level="1.2.3" data-path="../Installation/Preview.html">
<a href="../Installation/Preview.html">
预览模式
</a>
@@ -147,12 +160,12 @@
</li>
<li class="chapter " data-level="1.3" data-path="../Concept/">
<li class="chapter " data-level="1.3" data-path="../Usage/">
<a href="../Concept/">
<a href="../Usage/">
概念
使用Crawlab
</a>
@@ -161,9 +174,9 @@
<ul class="articles">
<li class="chapter " data-level="1.3.1" data-path="../Concept/Node.html">
<li class="chapter " data-level="1.3.1" data-path="../Usage/Node/">
<a href="../Concept/Node.html">
<a href="../Usage/Node/">
节点
@@ -172,11 +185,43 @@
<ul class="articles">
<li class="chapter " data-level="1.3.1.1" data-path="../Usage/Node/View.html">
<a href="../Usage/Node/View.html">
查看节点列表
</a>
</li>
<li class="chapter " data-level="1.3.2" data-path="../Concept/Spider.html">
<li class="chapter " data-level="1.3.1.2" data-path="../Usage/Node/Edit.html">
<a href="../Concept/Spider.html">
<a href="../Usage/Node/Edit.html">
修改节点信息
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.3.2" data-path="../Usage/Spider/">
<a href="../Usage/Spider/">
爬虫
@@ -185,11 +230,101 @@
<ul class="articles">
<li class="chapter " data-level="1.3.2.1" data-path="../Usage/Spider/Create.html">
<a href="../Usage/Spider/Create.html">
创建爬虫
</a>
<ul class="articles">
<li class="chapter " data-level="1.3.2.1.1" data-path="../Usage/Spider/CustomizedSpider.html">
<a href="../Usage/Spider/CustomizedSpider.html">
自定义爬虫
</a>
</li>
<li class="chapter " data-level="1.3.3" data-path="../Concept/Task.html">
<li class="chapter " data-level="1.3.2.1.2" data-path="../Usage/Spider/ConfigurableSpider.html">
<a href="../Concept/Task.html">
<a href="../Usage/Spider/ConfigurableSpider.html">
可配置爬虫
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.3.2.2" data-path="../Usage/Spider/Deploy.html">
<a href="../Usage/Spider/Deploy.html">
部署爬虫
</a>
</li>
<li class="chapter " data-level="1.3.2.3" data-path="../Usage/Spider/Run.html">
<a href="../Usage/Spider/Run.html">
运行爬虫
</a>
</li>
<li class="chapter " data-level="1.3.2.4" data-path="../Usage/Spider/Analytics.html">
<a href="../Usage/Spider/Analytics.html">
统计数据
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.3.3" data-path="../Usage/Task/">
<a href="../Usage/Task/">
任务
@@ -198,14 +333,72 @@
</li>
<ul class="articles">
<li class="chapter " data-level="1.3.4" data-path="../Concept/Deploy.html">
<li class="chapter " data-level="1.3.3.1" data-path="../Usage/Task/View.html">
<a href="../Concept/Deploy.html">
<a href="../Usage/Task/View.html">
部署
查看任务
</a>
</li>
<li class="chapter " data-level="1.3.3.2" data-path="../Usage/Task/Action.html">
<a href="../Usage/Task/Action.html">
操作任务
</a>
</li>
<li class="chapter " data-level="1.3.3.3" data-path="../Usage/Task/DownloadResults.html">
<a href="../Usage/Task/DownloadResults.html">
下载结果
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.3.4" data-path="../Usage/Schedule/">
<a href="../Usage/Schedule/">
定时任务
</a>
</li>
<li class="chapter " data-level="1.3.5" data-path="../Usage/Site/">
<a href="../Usage/Site/">
网站
</a>
@@ -229,38 +422,6 @@
<ul class="articles">
<li class="chapter " data-level="1.4.1" data-path="../Architecture/Celery.html">
<a href="../Architecture/Celery.html">
Celery
</a>
</li>
<li class="chapter " data-level="1.4.2" data-path="../Architecture/App.html">
<a href="../Architecture/App.html">
App
</a>
</li>
</ul>
</li>
<li class="chapter active" data-level="1.5" data-path="./">
@@ -268,7 +429,7 @@
<a href="./">
Examples
样例
</a>
@@ -277,9 +438,9 @@
<ul class="articles">
<li class="chapter active" data-level="1.5.1" data-path="./">
<li class="chapter " data-level="1.5.1" data-path="ScrapyIntegration.html">
<a href="./">
<a href="ScrapyIntegration.html">
与Scrapy集成
@@ -288,19 +449,6 @@
</li>
<li class="chapter active" data-level="1.5.2" data-path="./">
<a href="./">
与Puppeteer集成
</a>
</li>
@@ -338,7 +486,7 @@
<!-- Title -->
<h1>
<i class="fa fa-circle-o-notch fa-spin"></i>
<a href=".." >Examples</a>
<a href=".." >样例</a>
</h1>
</div>
@@ -353,7 +501,10 @@
<section class="normal markdown-section">
<h1 id="examples">Examples</h1>
<h2 id="&#x6837;&#x4F8B;">&#x6837;&#x4F8B;</h2>
<ol>
<li><a href="ScrapyIntegration.html">&#x4E0E;Scrapy&#x96C6;&#x6210;</a></li>
</ol>
</section>
@@ -381,12 +532,12 @@
<a href="../Architecture/App.html" class="navigation navigation-prev " aria-label="Previous page: App">
<a href="../Architecture/" class="navigation navigation-prev " aria-label="Previous page: 架构">
<i class="fa fa-angle-left"></i>
</a>
<a href="./" class="navigation navigation-next " aria-label="Next page: 与Scrapy集成">
<a href="ScrapyIntegration.html" class="navigation navigation-next " aria-label="Next page: 与Scrapy集成">
<i class="fa fa-angle-right"></i>
</a>
@@ -397,7 +548,7 @@
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"Examples","level":"1.5","depth":1,"next":{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/README.md","ref":"Examples/README.md","articles":[]},"previous":{"title":"App","level":"1.4.2","depth":2,"path":"Architecture/App.md","ref":"Architecture/App.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Examples/README.md","mtime":"2019-03-28T11:41:28.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-04T15:47:26.876Z"},"basePath":"..","book":{"language":""}});
gitbook.page.hasChanged({"page":{"title":"样例","level":"1.5","depth":1,"next":{"title":"与Scrapy集成","level":"1.5.1","depth":2,"path":"Examples/ScrapyIntegration.md","ref":"Examples/ScrapyIntegration.md","articles":[]},"previous":{"title":"架构","level":"1.4","depth":1,"path":"Architecture/README.md","ref":"Architecture/README.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Examples/README.md","mtime":"2019-06-16T13:48:53.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>