From 0b40fab625e727c1fd7fccc3cbda6156d40df897 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 16 Jun 2019 22:04:16 +0800 Subject: [PATCH] updated docs --- README-zh.md | 86 +-- docs/Architecture/App.html | 436 ----------- docs/Architecture/Celery.html | 436 ----------- docs/Architecture/index.html | 318 ++++++-- docs/Concept/Deploy.html | 438 ----------- docs/Concept/Node.html | 437 ----------- docs/Concept/Spider.html | 442 ----------- docs/Concept/Task.html | 437 ----------- docs/Examples/PuppeteerIntegration.md | 0 .../Examples/ScrapyIntegration.html | 111 +-- docs/Examples/index.html | 309 ++++++-- docs/Functions/FunctionList.md | 61 -- docs/Installation/Direct.html | 632 ++++++++++++++++ docs/Installation/Docker.html | 707 ++++++++++++++++++ docs/Installation/Preview.html | 588 +++++++++++++++ .../Installation/index.html | 104 +-- docs/QuickStart/Installation.html | 447 ----------- docs/QuickStart/Run.html | 476 ------------ docs/QuickStart/index.html | 440 ----------- docs/Usage/Node/Edit.html | 588 +++++++++++++++ docs/Usage/Node/View.html | 586 +++++++++++++++ docs/Usage/Node/index.html | 589 +++++++++++++++ docs/Usage/Schedule/index.html | 596 +++++++++++++++ docs/Usage/Site/index.html | 586 +++++++++++++++ docs/Usage/Spider/Analytics.html | 587 +++++++++++++++ docs/Usage/Spider/ConfigurableSpider.html | 619 +++++++++++++++ docs/Usage/Spider/Create.html | 590 +++++++++++++++ docs/Usage/Spider/CustomizedSpider.html | 602 +++++++++++++++ docs/Usage/Spider/Deploy.html | 592 +++++++++++++++ docs/Usage/Spider/Run.html | 598 +++++++++++++++ docs/Usage/Spider/index.html | 592 +++++++++++++++ docs/Usage/Task/Action.html | 589 +++++++++++++++ docs/Usage/Task/DownloadResults.html | 586 +++++++++++++++ docs/Usage/Task/View.html | 594 +++++++++++++++ docs/Usage/Task/index.html | 591 +++++++++++++++ docs/{Concept => Usage}/index.html | 308 ++++++-- docs/img/crawlab-architecture.png | Bin 47842 -> 0 bytes docs/img/payment.jpg | Bin 80007 -> 0 bytes docs/img/用户使用流程图.png | Bin 91335 -> 0 bytes docs/index.html | 462 ++++++------ docs/search_index.json | 2 +- gitbook/Architecture/App.md | 2 - gitbook/Architecture/Celery.md | 2 - gitbook/Architecture/README.md | 33 +- gitbook/Concept/Deploy.md | 6 - gitbook/Concept/Node.md | 3 - gitbook/Concept/README.md | 2 - gitbook/Concept/Spider.md | 15 - gitbook/Concept/Task.md | 3 - gitbook/Examples/PuppeteerIntegration.md | 0 gitbook/Examples/README.md | 4 +- gitbook/Examples/ScrapyIntegration.md | 26 + gitbook/Functions/FunctionList.md | 61 -- gitbook/README.md | 2 +- gitbook/SUMMARY.md | 8 +- gitbook/Usage/Schedule/README.md | 25 + gitbook/Usage/Site/README.md | 5 + gitbook/Usage/Task/Action.md | 11 + gitbook/Usage/Task/DownloadResults.md | 5 + gitbook/Usage/Task/README.md | 8 + gitbook/Usage/Task/View.md | 21 + gitbook/_book/Architecture/index.html | 87 +-- .../_book/Examples/PuppeteerIntegration.md | 0 gitbook/_book/Examples/ScrapyIntegration.html | 600 +++++++++++++++ gitbook/_book/Examples/index.html | 78 +- gitbook/_book/Installation/Direct.html | 65 +- gitbook/_book/Installation/Docker.html | 65 +- gitbook/_book/Installation/Preview.html | 65 +- gitbook/_book/Installation/index.html | 65 +- gitbook/_book/Usage/Node/Edit.html | 65 +- gitbook/_book/Usage/Node/View.html | 65 +- gitbook/_book/Usage/Node/index.html | 65 +- gitbook/_book/Usage/Schedule/index.html | 84 +-- gitbook/_book/Usage/Site/index.html | 70 +- gitbook/_book/Usage/Spider/Analytics.html | 65 +- .../Usage/Spider/ConfigurableSpider.html | 65 +- gitbook/_book/Usage/Spider/Create.html | 65 +- .../_book/Usage/Spider/CustomizedSpider.html | 65 +- gitbook/_book/Usage/Spider/Deploy.html | 65 +- gitbook/_book/Usage/Spider/Run.html | 65 +- gitbook/_book/Usage/Spider/index.html | 65 +- gitbook/_book/Usage/Task/Action.html | 589 +++++++++++++++ gitbook/_book/Usage/Task/DownloadResults.html | 586 +++++++++++++++ gitbook/_book/Usage/Task/View.html | 594 +++++++++++++++ gitbook/_book/Usage/Task/index.html | 79 +- gitbook/_book/Usage/index.html | 65 +- gitbook/_book/index.html | 67 +- gitbook/_book/search_index.json | 2 +- gitbook/img/crawlab-architecture.png | Bin 47842 -> 0 bytes gitbook/img/payment.jpg | Bin 80007 -> 0 bytes gitbook/img/用户使用流程图.png | Bin 91335 -> 0 bytes 91 files changed, 15220 insertions(+), 5995 deletions(-) delete mode 100644 docs/Architecture/App.html delete mode 100644 docs/Architecture/Celery.html delete mode 100644 docs/Concept/Deploy.html delete mode 100644 docs/Concept/Node.html delete mode 100644 docs/Concept/Spider.html delete mode 100644 docs/Concept/Task.html create mode 100644 docs/Examples/PuppeteerIntegration.md rename gitbook/_book/Architecture/App.html => docs/Examples/ScrapyIntegration.html (73%) delete mode 100644 docs/Functions/FunctionList.md create mode 100644 docs/Installation/Direct.html create mode 100644 docs/Installation/Docker.html create mode 100644 docs/Installation/Preview.html rename gitbook/_book/Architecture/Celery.html => docs/Installation/index.html (74%) delete mode 100644 docs/QuickStart/Installation.html delete mode 100644 docs/QuickStart/Run.html delete mode 100644 docs/QuickStart/index.html create mode 100644 docs/Usage/Node/Edit.html create mode 100644 docs/Usage/Node/View.html create mode 100644 docs/Usage/Node/index.html create mode 100644 docs/Usage/Schedule/index.html create mode 100644 docs/Usage/Site/index.html create mode 100644 docs/Usage/Spider/Analytics.html create mode 100644 docs/Usage/Spider/ConfigurableSpider.html create mode 100644 docs/Usage/Spider/Create.html create mode 100644 docs/Usage/Spider/CustomizedSpider.html create mode 100644 docs/Usage/Spider/Deploy.html create mode 100644 docs/Usage/Spider/Run.html create mode 100644 docs/Usage/Spider/index.html create mode 100644 docs/Usage/Task/Action.html create mode 100644 docs/Usage/Task/DownloadResults.html create mode 100644 docs/Usage/Task/View.html create mode 100644 docs/Usage/Task/index.html rename docs/{Concept => Usage}/index.html (50%) delete mode 100644 docs/img/crawlab-architecture.png delete mode 100644 docs/img/payment.jpg delete mode 100644 docs/img/用户使用流程图.png delete mode 100644 gitbook/Architecture/App.md delete mode 100644 gitbook/Architecture/Celery.md delete mode 100644 gitbook/Concept/Deploy.md delete mode 100644 gitbook/Concept/Node.md delete mode 100644 gitbook/Concept/README.md delete mode 100644 gitbook/Concept/Spider.md delete mode 100644 gitbook/Concept/Task.md create mode 100644 gitbook/Examples/PuppeteerIntegration.md create mode 100644 gitbook/Examples/ScrapyIntegration.md delete mode 100644 gitbook/Functions/FunctionList.md create mode 100644 gitbook/Usage/Task/Action.md create mode 100644 gitbook/Usage/Task/DownloadResults.md create mode 100644 gitbook/Usage/Task/View.md create mode 100644 gitbook/_book/Examples/PuppeteerIntegration.md create mode 100644 gitbook/_book/Examples/ScrapyIntegration.html create mode 100644 gitbook/_book/Usage/Task/Action.html create mode 100644 gitbook/_book/Usage/Task/DownloadResults.html create mode 100644 gitbook/_book/Usage/Task/View.html delete mode 100644 gitbook/img/crawlab-architecture.png delete mode 100644 gitbook/img/payment.jpg delete mode 100644 gitbook/img/用户使用流程图.png diff --git a/README-zh.md b/README-zh.md index 2f407f00..8fdcb0c9 100644 --- a/README-zh.md +++ b/README-zh.md @@ -20,25 +20,10 @@ ## 安装 -```bash -# 安装后台类库 -pip install -r requirements.txt -``` - -```bash -# 安装前台类库 -cd frontend -npm install -``` - -## 配置 - -请更改配置文件`config.py`，配置API和数据库连接. - -## 快速开始 -```bash -python manage.py serve -``` +三种方式: +1. [Docker](https://tikazyq.github.io/crawlab/Installation/Docker.md)（推荐） +2. [直接部署](https://tikazyq.github.io/crawlab/Installation/Direct.md) +3. [预览模式](https://tikazyq.github.io/crawlab/Installation/Direct.md)（快速体验） ## 截图 @@ -48,7 +33,7 @@ python manage.py serve #### 爬虫列表 -![](https://user-gold-cdn.xitu.io/2019/3/6/169524daf9c8ccef?imageView2/0/w/1280/h/960/format/webp/ignore-error/1) +![](https://crawlab.oss-cn-hangzhou.aliyuncs.com/gitbook/spider-list.png) #### 爬虫详情 - 概览 @@ -56,52 +41,42 @@ python manage.py serve #### 任务详情 - 抓取结果 -![](https://user-gold-cdn.xitu.io/2019/3/6/169524e4064c7f0a?imageView2/0/w/1280/h/960/format/webp/ignore-error/1) - -## 使用流程 - -![user-process](./docs/img/用户使用流程图.png) +![](https://crawlab.oss-cn-hangzhou.aliyuncs.com/gitbook/task-detail-results.png) ## 架构 -Crawlab的架构跟Celery非常相似，但是加入了包括前端、爬虫、Flower在内的额外模块，以支持爬虫管理的功能。 +Crawlab的架构跟Celery非常相似，但是加入了包括前端、爬虫、Flower在内的额外模块，以支持爬虫管理的功能。架构图如下。 -![crawlab-architecture](./docs/img/crawlab-architecture.png) +![](https://crawlab.oss-cn-hangzhou.aliyuncs.com/gitbook/architecture.png) -### 节点 +### 节点 Node -节点其实就是Celery中的Worker。一个节点运行时会连接到一个任务队列（例如Redis）来接收和运行任务。所有爬虫需要在运行时被部署到节点上，用户在部署前需要定义节点的IP地址和端口。 +节点其实就是Celery中的`worker`。一个节点运行时会连接到一个任务队列（例如`Redis`）来接收和运行任务。所有爬虫需要在运行时被部署到节点上，用户在部署前需要定义节点的IP地址和端口。 -### 爬虫 +### 后台应用 Backend App -##### 自动发现 +这是一个Flask应用，提供了必要的API来支持常规操作，例如CRUD、爬虫部署以及任务运行。每一个节点需要启动Flask应用来支持爬虫部署。运行`python app.py`来启动应用。 -在`config.py`文件中，修改变量`PROJECT_SOURCE_FILE_FOLDER`作为爬虫项目所在的目录。Crawlab后台程序会自动发现这些爬虫项目并储存到数据库中。是不是很方便？ +### 爬虫 Spider -##### 部署爬虫 +爬虫源代码或配置规则储存在`App`上，需要被部署到各个`worker`节点中。 -所有爬虫需要在抓取前被部署当相应当节点中。在"爬虫详情"页面点击"Deploy"按钮，爬虫将被部署到所有有效到节点中。 - -##### 运行爬虫 - -部署爬虫之后，你可以在"爬虫详情"页面点击"Run"按钮来启动爬虫。一个爬虫任务将被触发，你可以在任务列表页面中看到这个任务。 - -### 任务 +### 任务 Task 任务被触发并被节点执行。用户可以在任务详情页面中看到任务到状态、日志和抓取结果。 -### 后台应用 - -这是一个Flask应用，提供了必要的API来支持常规操作，例如CRUD、爬虫部署以及任务运行。每一个节点需要启动Flask应用来支持爬虫部署。运行`python manage.py app`或`python ./bin/run_app.py`来启动应用。 - -### 中间者 +### 中间者 Broker 中间者跟Celery中定义的一样，作为运行异步任务的队列。 -### 前端 +### 前端 Frontend 前端其实就是一个基于[Vue-Element-Admin](https://github.com/PanJiaChen/vue-element-admin)的单页应用。其中重用了很多Element-UI的控件来支持相应的展示。 +### Flower + +一个Celery的插件，用于监控Celery节点。 + ## 与其他框架的集成 任务是利用python的`subprocess`模块中的`Popen`来实现的。任务ID将以环境变量`CRAWLAB_TASK_ID`的形式存在于爬虫任务运行的进程中，并以此来关联抓取数据。 @@ -151,26 +126,11 @@ Crawlab使用起来很方便，也很通用，可以适用于几乎任何主流 | [ScrapydWeb](https://github.com/my8100/scrapydweb) | 管理平台 | Y | Y | Y | [Scrapyd](https://github.com/scrapy/scrapyd) | 网络服务 | Y | N | N/A -## TODOs -##### 后端 -- [ ] 文件管理 -- [ ] MySQL数据库支持 -- [ ] 重跑任务 -- [ ] 节点监控 -- [ ] 更多爬虫例子 - -##### 前端 -- [x] 任务数据统计 -- [x] 表格过滤 -- [x] 多语言支持 (中文) -- [ ] 登录和用户管理 -- [ ] 全局搜索 - ## 社区 & 赞助 如果您觉得Crawlab对您的日常开发或公司有帮助，请加作者微信 tikazyq1 并注明"Crawlab"，作者会将你拉入群。或者，您可以扫下方支付宝二维码给作者打赏去升级团队协作软件或买一杯咖啡。

- - + +

diff --git a/docs/Architecture/App.html b/docs/Architecture/App.html deleted file mode 100644 index 0cb7c156..00000000 --- a/docs/Architecture/App.html +++ /dev/null @@ -1,436 +0,0 @@ - - - - - - - App · GitBook - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - - - - - -
- -
- -
- - - - - - - - -
-
- -
-
- -
- -

App

- - -
- -
-
-
- -

results matching ""

-
    - -
    -
    - -

    No results matching ""

    - -
    -
    -
    - -
    -
    - -
    - - - - - - - - - - - - - - -
    - - -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/Architecture/Celery.html b/docs/Architecture/Celery.html deleted file mode 100644 index 252ebe69..00000000 --- a/docs/Architecture/Celery.html +++ /dev/null @@ -1,436 +0,0 @@ - - - - - - - Celery · GitBook - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -
    - - - - - - - - -
    - -
    - -
    - - - - - - - - -
    -
    - -
    -
    - -
    - -

    Celery

    - - -
    - -
    -
    -
    - -

    results matching ""

    -
      - -
      -
      - -

      No results matching ""

      - -
      -
      -
      - -
      -
      - -
      - - - - - - - - - - - - - - -
      - - -
      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/Architecture/index.html b/docs/Architecture/index.html index ff7b9d52..e2f85e07 100644 --- a/docs/Architecture/index.html +++ b/docs/Architecture/index.html @@ -57,10 +57,10 @@ - + - + @@ -94,7 +94,7 @@ - 简介 + Crawlab简介 @@ -102,12 +102,12 @@ -
    • +
    • - + - 快速开始 + 安装Crawlab @@ -116,12 +116,12 @@