mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-22 17:31:03 +01:00
480 lines
15 KiB
HTML
480 lines
15 KiB
HTML
|
|
<!DOCTYPE HTML>
|
|
<html lang="" >
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
|
<title>运行 · GitBook</title>
|
|
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
|
<meta name="description" content="">
|
|
<meta name="generator" content="GitBook 3.2.3">
|
|
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../gitbook/style.css">
|
|
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<meta name="HandheldFriendly" content="true"/>
|
|
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
|
<meta name="apple-mobile-web-app-capable" content="yes">
|
|
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
|
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
|
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
|
|
|
|
|
<link rel="next" href="../Concept/" />
|
|
|
|
|
|
<link rel="prev" href="Installation.html" />
|
|
|
|
|
|
</head>
|
|
<body>
|
|
|
|
<div class="book">
|
|
<div class="book-summary">
|
|
|
|
|
|
<div id="book-search-input" role="search">
|
|
<input type="text" placeholder="Type to search" />
|
|
</div>
|
|
|
|
|
|
<nav role="navigation">
|
|
|
|
|
|
|
|
<ul class="summary">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="chapter " data-level="1.1" data-path="../">
|
|
|
|
<a href="../">
|
|
|
|
|
|
简介
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.2" data-path="./">
|
|
|
|
<a href="./">
|
|
|
|
|
|
快速开始
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<ul class="articles">
|
|
|
|
|
|
<li class="chapter " data-level="1.2.1" data-path="Installation.html">
|
|
|
|
<a href="Installation.html">
|
|
|
|
|
|
安装
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter active" data-level="1.2.2" data-path="Run.html">
|
|
|
|
<a href="Run.html">
|
|
|
|
|
|
运行
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3" data-path="../Concept/">
|
|
|
|
<a href="../Concept/">
|
|
|
|
|
|
概念
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<ul class="articles">
|
|
|
|
|
|
<li class="chapter " data-level="1.3.1" data-path="../Concept/Node.html">
|
|
|
|
<a href="../Concept/Node.html">
|
|
|
|
|
|
节点
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3.2" data-path="../Concept/Spider.html">
|
|
|
|
<a href="../Concept/Spider.html">
|
|
|
|
|
|
爬虫
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3.3" data-path="../Concept/Task.html">
|
|
|
|
<a href="../Concept/Task.html">
|
|
|
|
|
|
任务
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3.4" data-path="../Concept/Deploy.html">
|
|
|
|
<a href="../Concept/Deploy.html">
|
|
|
|
|
|
部署
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
|
|
|
<a href="../Architecture/">
|
|
|
|
|
|
架构
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<ul class="articles">
|
|
|
|
|
|
<li class="chapter " data-level="1.4.1" data-path="../Architecture/Celery.html">
|
|
|
|
<a href="../Architecture/Celery.html">
|
|
|
|
|
|
Celery
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.4.2" data-path="../Architecture/App.html">
|
|
|
|
<a href="../Architecture/App.html">
|
|
|
|
|
|
App
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
|
|
|
<a href="../Examples/">
|
|
|
|
|
|
Examples
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<ul class="articles">
|
|
|
|
|
|
<li class="chapter " data-level="1.5.1" data-path="../Examples/">
|
|
|
|
<a href="../Examples/">
|
|
|
|
|
|
与Scrapy集成
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.5.2" data-path="../Examples/">
|
|
|
|
<a href="../Examples/">
|
|
|
|
|
|
与Puppeteer集成
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
<li class="divider"></li>
|
|
|
|
<li>
|
|
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
|
Published with GitBook
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
|
|
|
|
</nav>
|
|
|
|
|
|
</div>
|
|
|
|
<div class="book-body">
|
|
|
|
<div class="body-inner">
|
|
|
|
|
|
|
|
<div class="book-header" role="navigation">
|
|
|
|
|
|
<!-- Title -->
|
|
<h1>
|
|
<i class="fa fa-circle-o-notch fa-spin"></i>
|
|
<a href=".." >运行</a>
|
|
</h1>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<div class="page-wrapper" tabindex="-1" role="main">
|
|
<div class="page-inner">
|
|
|
|
<div id="book-search-results">
|
|
<div class="search-noresults">
|
|
|
|
<section class="normal markdown-section">
|
|
|
|
<h1 id="运行">运行</h1>
|
|
<p>在运行之前需要对Crawlab进行一些配置，配置文件为<code>config.py</code>。</p>
|
|
<pre><code class="lang-python"><span class="hljs-comment"># project variables</span>
|
|
PROJECT_SOURCE_FILE_FOLDER = <span class="hljs-string">'/Users/yeqing/projects/crawlab/spiders'</span> <span class="hljs-comment"># 爬虫源码根目录</span>
|
|
PROJECT_DEPLOY_FILE_FOLDER = <span class="hljs-string">'/var/crawlab'</span> <span class="hljs-comment"># 爬虫部署根目录</span>
|
|
PROJECT_LOGS_FOLDER = <span class="hljs-string">'/var/logs/crawlab'</span> <span class="hljs-comment"># 日志目录</span>
|
|
PROJECT_TMP_FOLDER = <span class="hljs-string">'/tmp'</span> <span class="hljs-comment"># 临时文件目录</span>
|
|
|
|
<span class="hljs-comment"># celery variables</span>
|
|
BROKER_URL = <span class="hljs-string">'redis://192.168.99.100:6379/0'</span> <span class="hljs-comment"># 中间者URL，连接redis</span>
|
|
CELERY_RESULT_BACKEND = <span class="hljs-string">'mongodb://192.168.99.100:27017/'</span> <span class="hljs-comment"># CELERY后台URL</span>
|
|
CELERY_MONGODB_BACKEND_SETTINGS = {
|
|
<span class="hljs-string">'database'</span>: <span class="hljs-string">'crawlab_test'</span>,
|
|
<span class="hljs-string">'taskmeta_collection'</span>: <span class="hljs-string">'tasks_celery'</span>,
|
|
}
|
|
CELERY_TIMEZONE = <span class="hljs-string">'Asia/Shanghai'</span>
|
|
CELERY_ENABLE_UTC = <span class="hljs-keyword">True</span>
|
|
|
|
<span class="hljs-comment"># flower variables</span>
|
|
FLOWER_API_ENDPOINT = <span class="hljs-string">'http://localhost:5555/api'</span> <span class="hljs-comment"># Flower服务地址</span>
|
|
|
|
<span class="hljs-comment"># database variables</span>
|
|
MONGO_HOST = <span class="hljs-string">'192.168.99.100'</span>
|
|
MONGO_PORT = <span class="hljs-number">27017</span>
|
|
MONGO_DB = <span class="hljs-string">'crawlab_test'</span>
|
|
|
|
<span class="hljs-comment"># flask variables</span>
|
|
DEBUG = <span class="hljs-keyword">True</span>
|
|
FLASK_HOST = <span class="hljs-string">'127.0.0.1'</span>
|
|
FLASK_PORT = <span class="hljs-number">8000</span>
|
|
</code></pre>
|
|
<p>启动后端API，也就是一个Flask App，可以直接启动，或者用gunicorn代替。</p>
|
|
<pre><code class="lang-bash">python app.py
|
|
</code></pre>
|
|
<p>启动Flower服务（抱歉目前集成Flower到App服务中，必须单独启动来获取节点信息，后面的版本不需要这个操作）。</p>
|
|
<pre><code class="lang-bash">python ./bin/run_flower.py
|
|
</code></pre>
|
|
<p>启动本地Worker。在其他节点中如果想只是想执行任务的话，只需要启动这一个服务就可以了。</p>
|
|
<pre><code class="lang-bash">python ./bin/run_worker.py
|
|
</code></pre>
|
|
<p>启动前端服务器。</p>
|
|
<pre><code class="lang-bash"><span class="hljs-built_in">cd</span> ../frontend
|
|
npm run serve
|
|
</code></pre>
|
|
|
|
|
|
</section>
|
|
|
|
</div>
|
|
<div class="search-results">
|
|
<div class="has-results">
|
|
|
|
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
|
<ul class="search-results-list"></ul>
|
|
|
|
</div>
|
|
<div class="no-results">
|
|
|
|
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
|
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
<a href="Installation.html" class="navigation navigation-prev " aria-label="Previous page: 安装">
|
|
<i class="fa fa-angle-left"></i>
|
|
</a>
|
|
|
|
|
|
<a href="../Concept/" class="navigation navigation-next " aria-label="Next page: 概念">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
|
|
|
|
|
|
</div>
|
|
|
|
<script>
|
|
var gitbook = gitbook || [];
|
|
gitbook.push(function() {
|
|
gitbook.page.hasChanged({"page":{"title":"运行","level":"1.2.2","depth":2,"next":{"title":"概念","level":"1.3","depth":1,"path":"Concept/README.md","ref":"Concept/README.md","articles":[{"title":"节点","level":"1.3.1","depth":2,"path":"Concept/Node.md","ref":"Concept/Node.md","articles":[]},{"title":"爬虫","level":"1.3.2","depth":2,"path":"Concept/Spider.md","ref":"Concept/Spider.md","articles":[]},{"title":"任务","level":"1.3.3","depth":2,"path":"Concept/Task.md","ref":"Concept/Task.md","articles":[]},{"title":"部署","level":"1.3.4","depth":2,"path":"Concept/Deploy.md","ref":"Concept/Deploy.md","articles":[]}]},"previous":{"title":"安装","level":"1.2.1","depth":2,"path":"QuickStart/Installation.md","ref":"QuickStart/Installation.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"QuickStart/Run.md","mtime":"2019-03-28T12:00:11.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-30T13:15:04.840Z"},"basePath":"..","book":{"language":""}});
|
|
});
|
|
</script>
|
|
</div>
|
|
|
|
|
|
<script src="../gitbook/gitbook.js"></script>
|
|
<script src="../gitbook/theme.js"></script>
|
|
|
|
|
|
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
|
|
|
|
|
|
|
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
|
|
|
|
|
|
|
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
|
|
|
|
|
|
|
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
|
|
|
|
|
|
|
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
|
|
|
|
|
|
|
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
|
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|