diff --git a/README-zh.md b/README-zh.md index 701b314a..a4f19b13 100644 --- a/README-zh.md +++ b/README-zh.md @@ -62,6 +62,10 @@ npm run serve ![spider-list](./docs/img/screenshot-task-detail-results.png) +## 使用流程 + +![user-process](./docs/img/用户使用流程图.png) + ## 架构 Crawlab的架构跟Celery非常相似,但是加入了包括前端、爬虫、Flower在内的额外模块,以支持爬虫管理的功能。 diff --git a/crawlab/config.py b/crawlab/config.py index 7467582e..4ede83b7 100644 --- a/crawlab/config.py +++ b/crawlab/config.py @@ -2,7 +2,7 @@ # 爬虫源码路径 PROJECT_SOURCE_FILE_FOLDER = '../spiders' # 配置python虚拟环境的路径 -PYTHON_ENV_PATH="/Users/chennan/Desktop/2019/env/bin/python" +PYTHON_ENV_PATH = '/Users/chennan/Desktop/2019/env/bin/python' # 爬虫部署路径 PROJECT_DEPLOY_FILE_FOLDER = '../deployfile' @@ -31,4 +31,3 @@ MONGO_DB = 'crawlab_test' DEBUG = True FLASK_HOST = '127.0.0.1' FLASK_PORT = 8000 - diff --git a/crawlab/requirements.txt b/crawlab/requirements.txt new file mode 100644 index 00000000..599ffcd2 --- /dev/null +++ b/crawlab/requirements.txt @@ -0,0 +1,31 @@ +amqp==2.4.2 +aniso8601==6.0.0 +APScheduler==3.6.0 +Babel==2.6.0 +billiard==3.6.0.0 +celery==4.3.0 +certifi==2019.3.9 +chardet==3.0.4 +Click==7.0 +coloredlogs==10.0 +Flask==1.0.2 +Flask-Cors==3.0.7 +Flask-RESTful==0.3.7 +flower==0.9.3 +humanfriendly==4.18 +idna==2.8 +itsdangerous==1.1.0 +Jinja2==2.10 +kombu==4.5.0 +MarkupSafe==1.1.1 +mongoengine==0.17.0 +pymongo==3.7.2 +pytz==2018.9 +redis==3.2.1 +requests==2.21.0 +six==1.12.0 +tornado==5.1.1 +tzlocal==1.5.1 +urllib3==1.24.1 +vine==1.3.0 +Werkzeug==0.15.2 diff --git a/crawlab/routes/tasks.py b/crawlab/routes/tasks.py index 7d0a6376..769b0558 100644 --- a/crawlab/routes/tasks.py +++ b/crawlab/routes/tasks.py @@ -10,6 +10,7 @@ from utils import jsonify from utils.spider import get_spider_col_fields from utils.log import other + class TaskApi(BaseApi): col_name = 'tasks' diff --git a/docs/img/Crawlab用户使用流程图.png b/docs/img/Crawlab用户使用流程图.png deleted file mode 100644 index ea1ae46f..00000000 Binary files a/docs/img/Crawlab用户使用流程图.png and /dev/null differ diff --git a/frontend/src/components/BackToTop/index 2.vue b/frontend/src/components/BackToTop/index 2.vue deleted file mode 100644 index 39977178..00000000 --- a/frontend/src/components/BackToTop/index 2.vue +++ /dev/null @@ -1,116 +0,0 @@ - - - - - diff --git a/frontend/src/components/Breadcrumb/index 2.vue b/frontend/src/components/Breadcrumb/index 2.vue deleted file mode 100644 index 87a389b8..00000000 --- a/frontend/src/components/Breadcrumb/index 2.vue +++ /dev/null @@ -1,86 +0,0 @@ - - - - - diff --git a/frontend/src/components/Charts/keyboard 2.vue b/frontend/src/components/Charts/keyboard 2.vue deleted file mode 100644 index 857b26ae..00000000 --- a/frontend/src/components/Charts/keyboard 2.vue +++ /dev/null @@ -1,156 +0,0 @@ - - - diff --git a/frontend/src/components/Charts/mixins/resize 2.js b/frontend/src/components/Charts/mixins/resize 2.js deleted file mode 100644 index c4c432f4..00000000 --- a/frontend/src/components/Charts/mixins/resize 2.js +++ /dev/null @@ -1,32 +0,0 @@ -import { debounce } from '@/utils' - -export default { - data() { - return { - sidebarElm: null - } - }, - mounted() { - this.__resizeHandler = debounce(() => { - if (this.chart) { - this.chart.resize() - } - }, 100) - window.addEventListener('resize', this.__resizeHandler) - - this.sidebarElm = document.getElementsByClassName('sidebar-container')[0] - this.sidebarElm && this.sidebarElm.addEventListener('transitionend', this.sidebarResizeHandler) - }, - beforeDestroy() { - window.removeEventListener('resize', this.__resizeHandler) - - this.sidebarElm && this.sidebarElm.removeEventListener('transitionend', this.sidebarResizeHandler) - }, - methods: { - sidebarResizeHandler(e) { - if (e.propertyName === 'width') { - this.__resizeHandler() - } - } - } -} diff --git a/frontend/src/components/Common/DialogView 2.vue b/frontend/src/components/Common/DialogView 2.vue deleted file mode 100644 index 7976171e..00000000 --- a/frontend/src/components/Common/DialogView 2.vue +++ /dev/null @@ -1,162 +0,0 @@ - - - - - diff --git a/frontend/src/components/DndList/index 2.vue b/frontend/src/components/DndList/index 2.vue deleted file mode 100644 index 7587daa7..00000000 --- a/frontend/src/components/DndList/index 2.vue +++ /dev/null @@ -1,157 +0,0 @@ - - - - - diff --git a/frontend/src/components/DragSelect/index 2.vue b/frontend/src/components/DragSelect/index 2.vue deleted file mode 100644 index 513be006..00000000 --- a/frontend/src/components/DragSelect/index 2.vue +++ /dev/null @@ -1,61 +0,0 @@ - - - - - diff --git a/frontend/src/components/Dropzone/index 2.vue b/frontend/src/components/Dropzone/index 2.vue deleted file mode 100644 index 15d811d9..00000000 --- a/frontend/src/components/Dropzone/index 2.vue +++ /dev/null @@ -1,297 +0,0 @@ - - - - - diff --git a/frontend/src/components/ErrorLog/index 2.vue b/frontend/src/components/ErrorLog/index 2.vue deleted file mode 100644 index c46cf110..00000000 --- a/frontend/src/components/ErrorLog/index 2.vue +++ /dev/null @@ -1,63 +0,0 @@ - - - - - diff --git a/frontend/src/components/FileList/FileList 2.vue b/frontend/src/components/FileList/FileList 2.vue deleted file mode 100644 index 20a43bec..00000000 --- a/frontend/src/components/FileList/FileList 2.vue +++ /dev/null @@ -1,200 +0,0 @@ - - - - - - - diff --git a/frontend/src/components/GithubCorner/index 2.vue b/frontend/src/components/GithubCorner/index 2.vue deleted file mode 100644 index b5da8874..00000000 --- a/frontend/src/components/GithubCorner/index 2.vue +++ /dev/null @@ -1,51 +0,0 @@ - - - diff --git a/frontend/src/components/Hamburger/index 2.vue b/frontend/src/components/Hamburger/index 2.vue deleted file mode 100644 index 220d67ec..00000000 --- a/frontend/src/components/Hamburger/index 2.vue +++ /dev/null @@ -1,42 +0,0 @@ - - - - - diff --git a/frontend/src/components/HeaderSearch/index 2.vue b/frontend/src/components/HeaderSearch/index 2.vue deleted file mode 100644 index ab0d556a..00000000 --- a/frontend/src/components/HeaderSearch/index 2.vue +++ /dev/null @@ -1,187 +0,0 @@ - - - - - diff --git a/frontend/src/components/ImageCropper/index 2.vue b/frontend/src/components/ImageCropper/index 2.vue deleted file mode 100644 index 04b1ede9..00000000 --- a/frontend/src/components/ImageCropper/index 2.vue +++ /dev/null @@ -1,1420 +0,0 @@ - - - - - - - diff --git a/frontend/src/components/InfoView/NodeInfoView 2.vue b/frontend/src/components/InfoView/NodeInfoView 2.vue deleted file mode 100644 index 69a3601a..00000000 --- a/frontend/src/components/InfoView/NodeInfoView 2.vue +++ /dev/null @@ -1,73 +0,0 @@ - - - - - diff --git a/frontend/src/components/JsonEditor/index 2.vue b/frontend/src/components/JsonEditor/index 2.vue deleted file mode 100644 index d68b8c05..00000000 --- a/frontend/src/components/JsonEditor/index 2.vue +++ /dev/null @@ -1,72 +0,0 @@ -