From ab2279c647d537c2b06c00c9947892fa58d9e04a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 4 Mar 2019 12:07:14 +0800 Subject: [PATCH] added github import --- .gitignore | 3 ++ app.py | 4 +- config.py | 1 + frontend/src/store/modules/spider.js | 13 ++++- frontend/src/views/spider/SpiderList.vue | 53 ++++++++++---------- routes/spiders.py | 63 +++++++++++++++++++++++- setup.py | 2 +- utils/file.py | 7 +-- 8 files changed, 112 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index c7a90ab9..e043754d 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,6 @@ venv.bak/ # node_modules node_modules/ + +# egg-info +*.egg-info diff --git a/app.py b/app.py index a9d9c5bf..ec94399c 100644 --- a/app.py +++ b/app.py @@ -9,7 +9,7 @@ from config import FLASK_HOST, FLASK_PORT, PROJECT_LOGS_FOLDER from routes.deploys import DeployApi from routes.files import FileApi from routes.nodes import NodeApi -from routes.spiders import SpiderApi +from routes.spiders import SpiderApi, SpiderImportApi from routes.stats import StatsApi from routes.tasks import TaskApi @@ -28,6 +28,8 @@ api.add_resource(NodeApi, '/api/nodes', '/api/nodes/', '/api/nodes//') +api.add_resource(SpiderImportApi, + '/api/spiders/import/') api.add_resource(SpiderApi, '/api/spiders', '/api/spiders/', diff --git a/config.py b/config.py index 4135016b..015c5c5d 100644 --- a/config.py +++ b/config.py @@ -13,6 +13,7 @@ CELERY_MONGODB_BACKEND_SETTINGS = { 'database': 'crawlab_test', 'taskmeta_collection': 'tasks_celery', } +CELERY_TIMEZONE = 'Asia/Shanghai' # flower variables FLOWER_API_ENDPOINT = 'http://localhost:5555/api' diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js index a91bf5e5..8d0ad719 100644 --- a/frontend/src/store/modules/spider.js +++ b/frontend/src/store/modules/spider.js @@ -12,7 +12,10 @@ const state = { activeNode: {}, // upload form for importing spiders - importForm: {} + importForm: { + url: '', + type: 'github' + } } const getters = {} @@ -114,8 +117,14 @@ const actions = { }).sort((a, b) => a.create_ts < b.create_ts ? 1 : -1), { root: true }) }) + }, + importGithub ({ state }) { + const url = state.importForm.url + return request.post('/spiders/import/github', { url }) + .then(response => { + console.log(response) + }) } - } export default { diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index fda3cb20..b6051c11 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -8,22 +8,22 @@ :before-close="onDialogClose"> - - - + + + Cancel - Import + Import @@ -36,23 +36,9 @@ @change="onSearch">
- - - Import Spiders - - - - -
Github
- - - Gitlab - - - SVN - - - + + Import Spiders + { + if (valid) { + this.importLoading = true + this.$store.dispatch('spider/importGithub') + .then(response => { + this.$message.success('Import repo sucessfully') + this.$store.dispatch('spider/getSpiderList') + }) + .catch(response => { + this.$message.error(response.data.error) + }) + .finally(() => { + this.dialogVisible = false + this.importLoading = false + }) + } + }) }, - openImportDialog (type) { - this.$store.commit('spider/SET_IMPORT_FORM', { type }) + openImportDialog () { this.dialogVisible = true } }, diff --git a/routes/spiders.py b/routes/spiders.py index c8b9b498..81b7617c 100644 --- a/routes/spiders.py +++ b/routes/spiders.py @@ -1,13 +1,14 @@ import json import os import shutil +import subprocess from datetime import datetime from random import random import requests from bson import ObjectId from flask import current_app, request -from flask_restful import reqparse +from flask_restful import reqparse, Resource from werkzeug.datastructures import FileStorage from config import PROJECT_DEPLOY_FILE_FOLDER, PROJECT_SOURCE_FILE_FOLDER, PROJECT_TMP_FOLDER @@ -302,3 +303,63 @@ class SpiderApi(BaseApi): 'status': 'ok', 'items': items }) + + +class SpiderImportApi(Resource): + parser = reqparse.RequestParser() + arguments = [ + ('url', str) + ] + + def __init__(self): + super(SpiderImportApi).__init__() + for arg, type in self.arguments: + self.parser.add_argument(arg, type=type) + + def post(self, platform=None): + if platform is None: + return { + 'status': 'ok', + 'code': 404, + 'error': 'platform invalid' + }, 404 + + if not hasattr(self, platform): + return { + 'status': 'ok', + 'code': 400, + 'error': 'platform "%s" invalid' % platform + }, 400 + + return getattr(self, platform)() + + def github(self): + self._git() + + def gitlab(self): + self._git() + + def _git(self): + args = self.parser.parse_args() + url = args.get('url') + if url is None: + return { + 'status': 'ok', + 'code': 400, + 'error': 'url should not be empty' + }, 400 + + try: + p = subprocess.Popen(['git', 'clone', url], cwd=PROJECT_SOURCE_FILE_FOLDER) + _stdout, _stderr = p.communicate() + except Exception as err: + return { + 'status': 'ok', + 'code': 500, + 'error': str(err) + }, 500 + + return { + 'status': 'ok', + 'message': 'success' + } diff --git a/setup.py b/setup.py index cf26d28c..22cda1de 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open('requirements.txt') as f: requirements = [l for l in f.read().splitlines() if l] setup( - name='crawlab', + name='crawlab-server', version='0.0.1', url='https://github.com/tikazyq/crawlab', install_requires=requirements, diff --git a/utils/file.py b/utils/file.py index 8ecc1c89..cec15ad5 100644 --- a/utils/file.py +++ b/utils/file.py @@ -2,7 +2,7 @@ import os import re from collections import defaultdict -SUFFIX_PATTERN = r'\.(\w{,10})$' +SUFFIX_PATTERN = r'\.([a-zA-Z]{,6})$' suffix_regex = re.compile(SUFFIX_PATTERN, re.IGNORECASE) SUFFIX_LANG_MAPPING = { @@ -20,7 +20,7 @@ def get_file_suffix(file_name: str): if m is not None: return m.groups()[0] else: - return file_name + return None def get_file_list(path): @@ -38,7 +38,8 @@ def get_file_suffix_stats(path) -> dict: stats = defaultdict(int) for file_path in get_file_list(path): suffix = get_file_suffix(file_path) - stats[suffix] += 1 + if suffix is not None: + stats[suffix] += 1 return stats