diff --git a/crawlab/.gitignore b/crawlab/.gitignore new file mode 100644 index 00000000..e043754d --- /dev/null +++ b/crawlab/.gitignore @@ -0,0 +1,112 @@ +.idea/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# node_modules +node_modules/ + +# egg-info +*.egg-info diff --git a/Dockerfile b/crawlab/Dockerfile similarity index 100% rename from Dockerfile rename to crawlab/Dockerfile diff --git a/bin/run_flower.py b/crawlab/bin/run_flower.py similarity index 100% rename from bin/run_flower.py rename to crawlab/bin/run_flower.py diff --git a/bin/run_worker.py b/crawlab/bin/run_worker.py similarity index 100% rename from bin/run_worker.py rename to crawlab/bin/run_worker.py diff --git a/bin/start_flower.sh b/crawlab/bin/start_flower.sh similarity index 100% rename from bin/start_flower.sh rename to crawlab/bin/start_flower.sh diff --git a/config.py b/crawlab/config.py similarity index 100% rename from config.py rename to crawlab/config.py diff --git a/model/__init__.py b/crawlab/constants/__init__.py similarity index 100% rename from model/__init__.py rename to crawlab/constants/__init__.py diff --git a/constants/file.py b/crawlab/constants/file.py similarity index 100% rename from constants/file.py rename to crawlab/constants/file.py diff --git a/constants/lang.py b/crawlab/constants/lang.py similarity index 100% rename from constants/lang.py rename to crawlab/constants/lang.py diff --git a/crawlab/constants/manage.py b/crawlab/constants/manage.py new file mode 100644 index 00000000..f1e60691 --- /dev/null +++ b/crawlab/constants/manage.py @@ -0,0 +1,4 @@ +class ActionType: + APP = 'app' + FLOWER = 'flower' + RUN_ALL = 'run_all' diff --git a/constants/node.py b/crawlab/constants/node.py similarity index 100% rename from constants/node.py rename to crawlab/constants/node.py diff --git a/constants/spider.py b/crawlab/constants/spider.py similarity index 100% rename from constants/spider.py rename to crawlab/constants/spider.py diff --git a/constants/task.py b/crawlab/constants/task.py similarity index 100% rename from constants/task.py rename to crawlab/constants/task.py diff --git a/spiders/taobao/taobao/__init__.py b/crawlab/db/__init__.py similarity index 100% rename from spiders/taobao/taobao/__init__.py rename to crawlab/db/__init__.py diff --git a/db/manager.py b/crawlab/db/manager.py similarity index 100% rename from db/manager.py rename to crawlab/db/manager.py diff --git a/docker-compose.yml b/crawlab/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to crawlab/docker-compose.yml diff --git a/app.py b/crawlab/manage.py similarity index 60% rename from app.py rename to crawlab/manage.py index 7efd3af8..65272099 100644 --- a/app.py +++ b/crawlab/manage.py @@ -1,11 +1,14 @@ import os -import shutil +import subprocess +from multiprocessing import Process -from flask import Flask, logging +import click +from flask import Flask from flask_cors import CORS from flask_restful import Api -from config import FLASK_HOST, FLASK_PORT, PROJECT_LOGS_FOLDER +from config import FLASK_HOST, FLASK_PORT, PROJECT_LOGS_FOLDER, BROKER_URL +from constants.manage import ActionType from routes.deploys import DeployApi from routes.files import FileApi from routes.nodes import NodeApi @@ -52,9 +55,36 @@ api.add_resource(StatsApi, '/api/stats', '/api/stats/') -# create folder if it does not exist -if not os.path.exists(PROJECT_LOGS_FOLDER): - os.makedirs(PROJECT_LOGS_FOLDER) + +def run_app(): + # create folder if it does not exist + if not os.path.exists(PROJECT_LOGS_FOLDER): + os.makedirs(PROJECT_LOGS_FOLDER) + + # run app instance + app.run(host=FLASK_HOST, port=FLASK_PORT) + + +def run_flower(): + p = subprocess.Popen(['celery', 'flower', '-b', BROKER_URL], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + for line in iter(p.stdout.readline, 'b'): + if line.decode('utf-8') != '': + print(line.decode('utf-8')) + + +@click.command() +@click.argument('action', type=click.Choice([ActionType.APP, ActionType.FLOWER, ActionType.RUN_ALL])) +def main(action): + if action == ActionType.APP: + run_app() + elif action == ActionType.FLOWER: + run_flower() + elif action == ActionType.RUN_ALL: + p_flower = Process(target=run_flower) + p_flower.start() + p_app = Process(target=run_app) + p_app.start() + if __name__ == '__main__': - app.run(host=FLASK_HOST, port=FLASK_PORT) + main() diff --git a/tasks/__init__.py b/crawlab/model/__init__.py similarity index 100% rename from tasks/__init__.py rename to crawlab/model/__init__.py diff --git a/model/base.py b/crawlab/model/base.py similarity index 100% rename from model/base.py rename to crawlab/model/base.py diff --git a/model/deploy.py b/crawlab/model/deploy.py similarity index 100% rename from model/deploy.py rename to crawlab/model/deploy.py diff --git a/model/node.py b/crawlab/model/node.py similarity index 100% rename from model/node.py rename to crawlab/model/node.py diff --git a/model/spider.py b/crawlab/model/spider.py similarity index 100% rename from model/spider.py rename to crawlab/model/spider.py diff --git a/model/task.py b/crawlab/model/task.py similarity index 100% rename from model/task.py rename to crawlab/model/task.py diff --git a/requirements.txt b/crawlab/requirements.txt similarity index 100% rename from requirements.txt rename to crawlab/requirements.txt diff --git a/routes/__init__.py b/crawlab/routes/__init__.py similarity index 100% rename from routes/__init__.py rename to crawlab/routes/__init__.py diff --git a/routes/base.py b/crawlab/routes/base.py similarity index 100% rename from routes/base.py rename to crawlab/routes/base.py diff --git a/routes/deploys.py b/crawlab/routes/deploys.py similarity index 100% rename from routes/deploys.py rename to crawlab/routes/deploys.py diff --git a/routes/files.py b/crawlab/routes/files.py similarity index 100% rename from routes/files.py rename to crawlab/routes/files.py diff --git a/routes/nodes.py b/crawlab/routes/nodes.py similarity index 100% rename from routes/nodes.py rename to crawlab/routes/nodes.py diff --git a/routes/spiders.py b/crawlab/routes/spiders.py similarity index 100% rename from routes/spiders.py rename to crawlab/routes/spiders.py diff --git a/routes/stats.py b/crawlab/routes/stats.py similarity index 100% rename from routes/stats.py rename to crawlab/routes/stats.py diff --git a/routes/tasks.py b/crawlab/routes/tasks.py similarity index 100% rename from routes/tasks.py rename to crawlab/routes/tasks.py diff --git a/setup.py b/crawlab/setup.py similarity index 100% rename from setup.py rename to crawlab/setup.py diff --git a/test/__init__.py b/crawlab/tasks/__init__.py similarity index 100% rename from test/__init__.py rename to crawlab/tasks/__init__.py diff --git a/tasks/celery.py b/crawlab/tasks/celery.py similarity index 100% rename from tasks/celery.py rename to crawlab/tasks/celery.py diff --git a/tasks/deploy.py b/crawlab/tasks/deploy.py similarity index 100% rename from tasks/deploy.py rename to crawlab/tasks/deploy.py diff --git a/tasks/node.py b/crawlab/tasks/node.py similarity index 100% rename from tasks/node.py rename to crawlab/tasks/node.py diff --git a/tasks/spider.py b/crawlab/tasks/spider.py similarity index 100% rename from tasks/spider.py rename to crawlab/tasks/spider.py diff --git a/crawlab/test/__init__.py b/crawlab/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/test.http b/crawlab/test/test.http similarity index 100% rename from test/test.http rename to crawlab/test/test.http diff --git a/utils/__init__.py b/crawlab/utils/__init__.py similarity index 100% rename from utils/__init__.py rename to crawlab/utils/__init__.py diff --git a/utils/deploy.py b/crawlab/utils/deploy.py similarity index 100% rename from utils/deploy.py rename to crawlab/utils/deploy.py diff --git a/utils/file.py b/crawlab/utils/file.py similarity index 100% rename from utils/file.py rename to crawlab/utils/file.py diff --git a/utils/node.py b/crawlab/utils/node.py similarity index 100% rename from utils/node.py rename to crawlab/utils/node.py diff --git a/utils/spider.py b/crawlab/utils/spider.py similarity index 100% rename from utils/spider.py rename to crawlab/utils/spider.py