From 87c81deefb92b91eb50fab6811cc638798644ca7 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 29 May 2019 13:10:31 +0800 Subject: [PATCH] code cleanup --- crawlab/manage.py | 125 ------------------ crawlab/model/__init__.py | 0 crawlab/model/base.py | 6 - crawlab/model/deploy.py | 10 -- crawlab/model/node.py | 12 -- crawlab/model/spider.py | 12 -- crawlab/model/task.py | 9 -- crawlab/setup.py | 24 ---- frontend/src/components/Config/ConfigList.vue | 4 + frontend/src/i18n/zh.js | 15 ++- 10 files changed, 12 insertions(+), 205 deletions(-) delete mode 100644 crawlab/manage.py delete mode 100644 crawlab/model/__init__.py delete mode 100644 crawlab/model/base.py delete mode 100644 crawlab/model/deploy.py delete mode 100644 crawlab/model/node.py delete mode 100644 crawlab/model/spider.py delete mode 100644 crawlab/model/task.py delete mode 100644 crawlab/setup.py diff --git a/crawlab/manage.py b/crawlab/manage.py deleted file mode 100644 index 5b9ae3d3..00000000 --- a/crawlab/manage.py +++ /dev/null @@ -1,125 +0,0 @@ -import os -import subprocess -import sys -from multiprocessing import Process - -import click -from flask import Flask -from flask_cors import CORS -from flask_restful import Api - -from routes.schedules import ScheduleApi -from tasks.scheduler import scheduler - -file_dir = os.path.dirname(os.path.realpath(__file__)) -root_path = os.path.abspath(os.path.join(file_dir, '.')) -sys.path.append(root_path) - -from config import FLASK_HOST, FLASK_PORT, PROJECT_LOGS_FOLDER, BROKER_URL -from constants.manage import ActionType -from routes.deploys import DeployApi -from routes.files import FileApi -from routes.nodes import NodeApi -from routes.spiders import SpiderApi, SpiderImportApi, SpiderManageApi -from routes.stats import StatsApi -from routes.tasks import TaskApi -from tasks.celery import celery_app -from utils.log import other -# flask app instance -app = Flask(__name__) -app.config.from_object('config') - -# init flask api instance -api = Api(app) - -# cors support -CORS(app, supports_credentials=True) - -# reference api routes -api.add_resource(NodeApi, - '/api/nodes', - '/api/nodes/', - '/api/nodes//') -api.add_resource(SpiderImportApi, - '/api/spiders/import/') -api.add_resource(SpiderManageApi, - '/api/spiders/manage/') -api.add_resource(SpiderApi, - '/api/spiders', - '/api/spiders/', - '/api/spiders//') -api.add_resource(DeployApi, - '/api/deploys', - '/api/deploys/', - '/api/deploys//') -api.add_resource(TaskApi, - '/api/tasks', - '/api/tasks/', - '/api/tasks//' - ) -api.add_resource(FileApi, - '/api/files', - '/api/files/') -api.add_resource(StatsApi, - '/api/stats', - '/api/stats/') -api.add_resource(ScheduleApi, - '/api/schedules', - '/api/schedules/') - - -def run_app(): - # create folder if it does not exist - if not os.path.exists(PROJECT_LOGS_FOLDER): - os.makedirs(PROJECT_LOGS_FOLDER) - - # run app instance - app.run(host=FLASK_HOST, port=FLASK_PORT) - - -def run_flower(): - p = subprocess.Popen(['celery', 'flower', '-b', BROKER_URL], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - for line in iter(p.stdout.readline, 'b'): - if line.decode('utf-8') != '': - other.info(line.decode('utf-8')) - - -def run_worker(): - if sys.platform == 'windows': - celery_app.start(argv=['tasks', 'worker', '-P', 'eventlet', '-E', '-l', 'INFO']) - else: - celery_app.start(argv=['tasks', 'worker', '-E', '-l', 'INFO']) - - -def run_scheduler(): - scheduler.run() - - -@click.command() -@click.argument('action', type=click.Choice([ActionType.APP, - ActionType.FLOWER, - ActionType.WORKER, - ActionType.SCHEDULER, - ActionType.RUN_ALL])) -def main(action): - if action == ActionType.APP: - run_app() - elif action == ActionType.FLOWER: - run_flower() - elif action == ActionType.WORKER: - run_worker() - elif action == ActionType.SCHEDULER: - run_scheduler() - elif action == ActionType.RUN_ALL: - p_flower = Process(target=run_flower) - p_flower.start() - p_app = Process(target=run_app) - p_app.start() - p_worker = Process(target=run_worker) - p_worker.start() - p_scheduler = Process(target=run_scheduler) - p_scheduler.start() - - -if __name__ == '__main__': - main() diff --git a/crawlab/model/__init__.py b/crawlab/model/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/crawlab/model/base.py b/crawlab/model/base.py deleted file mode 100644 index 253cc063..00000000 --- a/crawlab/model/base.py +++ /dev/null @@ -1,6 +0,0 @@ -from mongoengine import * -import datetime - - -class BaseModel(Document): - create_ts = DateTimeField(default=datetime.datetime.utcnow) diff --git a/crawlab/model/deploy.py b/crawlab/model/deploy.py deleted file mode 100644 index 2d21c6fc..00000000 --- a/crawlab/model/deploy.py +++ /dev/null @@ -1,10 +0,0 @@ -from mongoengine import * - -from model.base import BaseModel - - -class Deploy(BaseModel): - _id = ObjectIdField() - spider_id = ObjectIdField() - version = IntField() - node_id = ObjectIdField() diff --git a/crawlab/model/node.py b/crawlab/model/node.py deleted file mode 100644 index 4b9b4910..00000000 --- a/crawlab/model/node.py +++ /dev/null @@ -1,12 +0,0 @@ -from mongoengine import * - -from model.base import BaseModel - - -class Node(BaseModel): - _id = ObjectIdField() - ip = StringField() - port = IntField() - name = StringField() - description = StringField() - status = IntField() diff --git a/crawlab/model/spider.py b/crawlab/model/spider.py deleted file mode 100644 index eee86711..00000000 --- a/crawlab/model/spider.py +++ /dev/null @@ -1,12 +0,0 @@ -from mongoengine import * - -from model.base import BaseModel - - -class Spider(BaseModel): - _id = ObjectIdField() - name = StringField() - cmd = StringField() - src = StringField() - type = IntField() - lang = IntField() diff --git a/crawlab/model/task.py b/crawlab/model/task.py deleted file mode 100644 index 9f934eb1..00000000 --- a/crawlab/model/task.py +++ /dev/null @@ -1,9 +0,0 @@ -from mongoengine import * - -from model.base import BaseModel - - -class Task(BaseModel): - _id = ObjectIdField() - deploy_id = ObjectIdField() - file_path = StringField() diff --git a/crawlab/setup.py b/crawlab/setup.py deleted file mode 100644 index 22cda1de..00000000 --- a/crawlab/setup.py +++ /dev/null @@ -1,24 +0,0 @@ -from setuptools import setup, find_packages - -with open("README.md", "r") as fh: - long_description = fh.read() - -with open('requirements.txt') as f: - requirements = [l for l in f.read().splitlines() if l] - -setup( - name='crawlab-server', - version='0.0.1', - url='https://github.com/tikazyq/crawlab', - install_requires=requirements, - license='BSD', - author='Marvin Zhang', - author_email='tikazyq@163.com', - description='Celery-based web crawler admin platform for managing distributed web spiders regardless of languages and frameworks.', - long_description=long_description, - long_description_content_type="text/markdown", - download_url="https://github.com/tikazyq/crawlab/archive/master.zip", - packages=find_packages(), - keywords=['celery', 'python', 'webcrawler', 'crawl', 'scrapy', 'admin'], - zip_safe=True, -) diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue index 6c47570a..908f2c9a 100644 --- a/frontend/src/components/Config/ConfigList.vue +++ b/frontend/src/components/Config/ConfigList.vue @@ -79,6 +79,7 @@
{{$t('Run')}} + {{$t('Extract Fields')}} {{$t('Preview')}} {{$t('Save')}}
@@ -126,6 +127,7 @@ export default { { value: 'detail', label: 'Detail Only' }, { value: 'list-detail', label: 'List + Detail' } ], + extractFieldsLoading: false, previewLoading: false, saveLoading: false, dialogVisible: false @@ -210,6 +212,8 @@ export default { this.$message.success(this.$t(`Spider task has been scheduled`)) }) }) + }, + onExtractFields () { } }, created () { diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index e6db4795..9af096c9 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -48,13 +48,14 @@ export default { Submit: '提交', 'Import Spiders': '导入爬虫', 'Deploy All': '部署所有爬虫', - Refresh: '刷新', - View: '查看', - Edit: '编辑', - Remove: '删除', - Confirm: '确认', - Stop: '停止', - Preview: '预览', + 'Refresh': '刷新', + 'View': '查看', + 'Edit': '编辑', + 'Remove': '删除', + 'Confirm': '确认', + 'Stop': '停止', + 'Preview': '预览', + 'Extract Fields': '提取字段', // 主页 'Total Tasks': '总任务数',