mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-22 17:31:03 +01:00
633 lines
19 KiB
HTML
633 lines
19 KiB
HTML
|
|
<!DOCTYPE HTML>
|
|
<html lang="" >
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
|
<title>直接部署 · GitBook</title>
|
|
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
|
<meta name="description" content="">
|
|
<meta name="generator" content="GitBook 3.2.3">
|
|
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../gitbook/style.css">
|
|
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<meta name="HandheldFriendly" content="true"/>
|
|
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
|
<meta name="apple-mobile-web-app-capable" content="yes">
|
|
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
|
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
|
|
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
|
|
|
|
|
|
<link rel="next" href="Preview.html" />
|
|
|
|
|
|
<link rel="prev" href="Docker.html" />
|
|
|
|
|
|
</head>
|
|
<body>
|
|
|
|
<div class="book">
|
|
<div class="book-summary">
|
|
|
|
|
|
<div id="book-search-input" role="search">
|
|
<input type="text" placeholder="Type to search" />
|
|
</div>
|
|
|
|
|
|
<nav role="navigation">
|
|
|
|
|
|
|
|
<ul class="summary">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="chapter " data-level="1.1" data-path="../">
|
|
|
|
<a href="../">
|
|
|
|
|
|
Crawlab简介
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.2" data-path="./">
|
|
|
|
<a href="./">
|
|
|
|
|
|
安装Crawlab
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<ul class="articles">
|
|
|
|
|
|
<li class="chapter " data-level="1.2.1" data-path="Docker.html">
|
|
|
|
<a href="Docker.html">
|
|
|
|
|
|
Docker
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter active" data-level="1.2.2" data-path="Direct.html">
|
|
|
|
<a href="Direct.html">
|
|
|
|
|
|
直接部署
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.2.3" data-path="Preview.html">
|
|
|
|
<a href="Preview.html">
|
|
|
|
|
|
预览模式
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3" data-path="../Usage/">
|
|
|
|
<a href="../Usage/">
|
|
|
|
|
|
使用Crawlab
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<ul class="articles">
|
|
|
|
|
|
<li class="chapter " data-level="1.3.1" data-path="../Usage/Node/">
|
|
|
|
<a href="../Usage/Node/">
|
|
|
|
|
|
节点
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<ul class="articles">
|
|
|
|
|
|
<li class="chapter " data-level="1.3.1.1" data-path="../Usage/Node/View.html">
|
|
|
|
<a href="../Usage/Node/View.html">
|
|
|
|
|
|
查看节点列表
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3.1.2" data-path="../Usage/Node/Edit.html">
|
|
|
|
<a href="../Usage/Node/Edit.html">
|
|
|
|
|
|
修改节点信息
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3.2" data-path="../Usage/Spider/">
|
|
|
|
<a href="../Usage/Spider/">
|
|
|
|
|
|
爬虫
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<ul class="articles">
|
|
|
|
|
|
<li class="chapter " data-level="1.3.2.1" data-path="../Usage/Spider/Create.html">
|
|
|
|
<a href="../Usage/Spider/Create.html">
|
|
|
|
|
|
创建爬虫
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<ul class="articles">
|
|
|
|
|
|
<li class="chapter " data-level="1.3.2.1.1" data-path="../Usage/Spider/CustomizedSpider.html">
|
|
|
|
<a href="../Usage/Spider/CustomizedSpider.html">
|
|
|
|
|
|
自定义爬虫
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3.2.1.2" data-path="../Usage/Spider/ConfigurableSpider.html">
|
|
|
|
<a href="../Usage/Spider/ConfigurableSpider.html">
|
|
|
|
|
|
可配置爬虫
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3.2.2" data-path="../Usage/Spider/Deploy.html">
|
|
|
|
<a href="../Usage/Spider/Deploy.html">
|
|
|
|
|
|
部署爬虫
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3.2.3" data-path="../Usage/Spider/Run.html">
|
|
|
|
<a href="../Usage/Spider/Run.html">
|
|
|
|
|
|
运行爬虫
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3.2.4" data-path="../Usage/Spider/Analytics.html">
|
|
|
|
<a href="../Usage/Spider/Analytics.html">
|
|
|
|
|
|
统计数据
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3.3" data-path="../Usage/Task/">
|
|
|
|
<a href="../Usage/Task/">
|
|
|
|
|
|
任务
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<ul class="articles">
|
|
|
|
|
|
<li class="chapter " data-level="1.3.3.1" data-path="../Usage/Task/View.html">
|
|
|
|
<a href="../Usage/Task/View.html">
|
|
|
|
|
|
查看任务
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3.3.2" data-path="../Usage/Task/Action.html">
|
|
|
|
<a href="../Usage/Task/Action.html">
|
|
|
|
|
|
操作任务
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3.3.3" data-path="../Usage/Task/DownloadResults.html">
|
|
|
|
<a href="../Usage/Task/DownloadResults.html">
|
|
|
|
|
|
下载结果
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3.4" data-path="../Usage/Schedule/">
|
|
|
|
<a href="../Usage/Schedule/">
|
|
|
|
|
|
定时任务
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.3.5" data-path="../Usage/Site/">
|
|
|
|
<a href="../Usage/Site/">
|
|
|
|
|
|
网站
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.4" data-path="../Architecture/">
|
|
|
|
<a href="../Architecture/">
|
|
|
|
|
|
架构
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
<li class="chapter " data-level="1.5" data-path="../Examples/">
|
|
|
|
<a href="../Examples/">
|
|
|
|
|
|
样例
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<ul class="articles">
|
|
|
|
|
|
<li class="chapter " data-level="1.5.1" data-path="../Examples/ScrapyIntegration.html">
|
|
|
|
<a href="../Examples/ScrapyIntegration.html">
|
|
|
|
|
|
与Scrapy集成
|
|
|
|
</a>
|
|
|
|
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
<li class="divider"></li>
|
|
|
|
<li>
|
|
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
|
|
Published with GitBook
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
|
|
|
|
</nav>
|
|
|
|
|
|
</div>
|
|
|
|
<div class="book-body">
|
|
|
|
<div class="body-inner">
|
|
|
|
|
|
|
|
<div class="book-header" role="navigation">
|
|
|
|
|
|
<!-- Title -->
|
|
<h1>
|
|
<i class="fa fa-circle-o-notch fa-spin"></i>
|
|
<a href=".." >直接部署</a>
|
|
</h1>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<div class="page-wrapper" tabindex="-1" role="main">
|
|
<div class="page-inner">
|
|
|
|
<div id="book-search-results">
|
|
<div class="search-noresults">
|
|
|
|
<section class="normal markdown-section">
|
|
|
|
<h2 id="直接部署">直接部署</h2>
|
|
<p>直接部署是之前没有Docker时的部署方式，相对于Docker部署来说有些繁琐。但了解如何直接部署可以帮助更深入地理解Docker是如何构建Crawlab镜像的。这里简单介绍一下。</p>
|
|
<h3 id="拉取代码">拉取代码</h3>
|
|
<p>首先是将github上的代码拉取到本地。</p>
|
|
<pre><code class="lang-bash">git <span class="hljs-built_in">clone</span> https://github.com/tikazyq/crawlab
|
|
</code></pre>
|
|
<h3 id="安装">安装</h3>
|
|
<p>安装前端所需库。</p>
|
|
<pre><code class="lang-bash">npm install -g yarn pm2
|
|
<span class="hljs-built_in">cd</span> frontend
|
|
yarn install
|
|
</code></pre>
|
|
<p>安装后端所需库。</p>
|
|
<pre><code class="lang-bash"><span class="hljs-built_in">cd</span> ../crawlab
|
|
pip install -r requirements
|
|
</code></pre>
|
|
<h3 id="配置">配置</h3>
|
|
<p>分别配置前端配置文件<code>./frontend/.env.production</code>和后端配置文件<code>./crawlab/config/config.py</code>。分别需要对部署后API地址以及数据库地址进行配置。</p>
|
|
<h3 id="构建">构建</h3>
|
|
<p>这里的构建是指前端构建，需要执行以下命令。</p>
|
|
<pre><code class="lang-bash"><span class="hljs-built_in">cd</span> ../frontend
|
|
npm run build:prod
|
|
</code></pre>
|
|
<p>构建完成后，会在<code>./frontend</code>目录下创建一个<code>dist</code>文件夹，里面是打包好后的静态文件。</p>
|
|
<h3 id="nginx">Nginx</h3>
|
|
<p>安装<code>nginx</code>，在<code>ubuntu 16.04</code>是以下命令。</p>
|
|
<pre><code class="lang-bash">sudo apt-get install nginx
|
|
</code></pre>
|
|
<p>添加<code>/etc/nginx/conf.d/crawlab.conf</code>文件，输入以下内容。</p>
|
|
<pre><code>server {
|
|
listen 8080;
|
|
server_name dev.crawlab.com;
|
|
root /home/yeqing/jenkins_home/workspace/crawlab_develop/frontend/dist;
|
|
index index.html;
|
|
}
|
|
</code></pre><p>其中，<code>root</code>是静态文件的根目录，这里是<code>npm</code>打包好后的静态文件。</p>
|
|
<p>现在，只需要启动<code>nginx</code>服务就完成了启动前端服务。</p>
|
|
<pre><code class="lang-bash">nginx reload
|
|
</code></pre>
|
|
<h3 id="启动服务">启动服务</h3>
|
|
<p>这里是指启动后端服务。我们用<code>pm2</code>来管理进程。执行以下命令。</p>
|
|
<pre><code class="lang-bash">pm2 start app.py <span class="hljs-comment"># API服务</span>
|
|
pm2 start worker.py <span class="hljs-comment"># Worker</span>
|
|
pm2 start flower.py <span class="hljs-comment"># Flower</span>
|
|
</code></pre>
|
|
<p>这样，<code>pm2</code>会启动3个守护进程来管理这3个服务。我们如果想看后端服务的日志的话，可以执行以下命令。</p>
|
|
<pre><code class="lang-bash">pm2 logs [app]
|
|
</code></pre>
|
|
<p>然后在浏览器中输入<code>http://localhost:8080</code>就可以看到界面了。</p>
|
|
|
|
|
|
</section>
|
|
|
|
</div>
|
|
<div class="search-results">
|
|
<div class="has-results">
|
|
|
|
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
|
|
<ul class="search-results-list"></ul>
|
|
|
|
</div>
|
|
<div class="no-results">
|
|
|
|
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
|
|
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
<a href="Docker.html" class="navigation navigation-prev " aria-label="Previous page: Docker">
|
|
<i class="fa fa-angle-left"></i>
|
|
</a>
|
|
|
|
|
|
<a href="Preview.html" class="navigation navigation-next " aria-label="Next page: 预览模式">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
|
|
|
|
|
|
</div>
|
|
|
|
<script>
|
|
var gitbook = gitbook || [];
|
|
gitbook.push(function() {
|
|
gitbook.page.hasChanged({"page":{"title":"直接部署","level":"1.2.2","depth":2,"next":{"title":"预览模式","level":"1.2.3","depth":2,"path":"Installation/Preview.md","ref":"Installation/Preview.md","articles":[]},"previous":{"title":"Docker","level":"1.2.1","depth":2,"path":"Installation/Docker.md","ref":"Installation/Docker.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"Installation/Direct.md","mtime":"2019-06-16T03:18:40.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-06-16T14:03:57.361Z"},"basePath":"..","book":{"language":""}});
|
|
});
|
|
</script>
|
|
</div>
|
|
|
|
|
|
<script src="../gitbook/gitbook.js"></script>
|
|
<script src="../gitbook/theme.js"></script>
|
|
|
|
|
|
<script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
|
|
|
|
|
|
|
|
<script src="../gitbook/gitbook-plugin-search/search.js"></script>
|
|
|
|
|
|
|
|
<script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
|
|
|
|
|
|
|
|
<script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
|
|
|
|
|
|
|
|
<script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
|
|
|
|
|
|
|
|
<script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
|
|
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|