diff --git a/crawlab/bin/run_app.py b/crawlab/bin/run_app.py new file mode 100644 index 00000000..999d7278 --- /dev/null +++ b/crawlab/bin/run_app.py @@ -0,0 +1,17 @@ +import sys +import os + +# make sure the working directory is in system path +file_dir = os.path.dirname(os.path.realpath(__file__)) +root_path = os.path.abspath(os.path.join(file_dir, '..')) +sys.path.append(root_path) + +from config import PROJECT_LOGS_FOLDER, FLASK_HOST, FLASK_PORT +from manage import app + +# create folder if it does not exist +if not os.path.exists(PROJECT_LOGS_FOLDER): + os.makedirs(PROJECT_LOGS_FOLDER) + +# run app instance +app.run(host=FLASK_HOST, port=FLASK_PORT) diff --git a/crawlab/routes/spiders.py b/crawlab/routes/spiders.py index a2964400..f8a220e4 100644 --- a/crawlab/routes/spiders.py +++ b/crawlab/routes/spiders.py @@ -151,24 +151,12 @@ class SpiderApi(BaseApi): } def deploy(self, id): - args = self.parser.parse_args() - node_id = args.get('node_id') + spider = db_manager.get('spiders', id=id) + nodes = db_manager.list('nodes', {}) - # get spider given the id - spider = db_manager.get(col_name=self.col_name, id=id) - if spider is None: - return + for node in nodes: + node_id = node['_id'] - # get node given the node - node = db_manager.get(col_name='nodes', id=node_id) - - # make source / destination - src = spider.get('src') - - # copy files - # TODO: multi-node copy files - try: - # TODO: deploy spiders to other node rather than in local machine output_file_name = '%s_%s.zip' % ( datetime.now().strftime('%Y%m%d%H%M%S'), str(random())[2:12] @@ -176,7 +164,7 @@ class SpiderApi(BaseApi): output_file_path = os.path.join(PROJECT_TMP_FOLDER, output_file_name) # zip source folder to zip file - zip_file(source_dir=src, + zip_file(source_dir=spider['src'], output_filename=output_file_path) # upload to api @@ -188,27 +176,11 @@ class SpiderApi(BaseApi): node_id, ), files=files) - if r.status_code == 200: - return { - 'code': 200, - 'status': 'ok', - 'message': 'deploy success' - } - - else: - return { - 'code': r.status_code, - 'status': 'ok', - 'error': r.content.decode('utf-8') - }, r.status_code - - except Exception as err: - current_app.logger.error(err) - return { - 'code': 500, - 'status': 'ok', - 'error': str(err) - }, 500 + return { + 'code': 200, + 'status': 'ok', + 'message': 'deploy success' + } def deploy_file(self, id=None): args = parser.parse_args() diff --git a/crawlab/routes/tasks.py b/crawlab/routes/tasks.py index 2f496c17..398bee15 100644 --- a/crawlab/routes/tasks.py +++ b/crawlab/routes/tasks.py @@ -1,3 +1,7 @@ +import json + +import requests + from constants.task import TaskStatus from db.manager import db_manager from routes.base import BaseApi @@ -57,7 +61,7 @@ class TaskApi(BaseApi): 'items': items }) - def get_log(self, id): + def on_get_log(self, id): try: task = db_manager.get('tasks', id=id) with open(task['log_file_path']) as f: @@ -73,6 +77,28 @@ class TaskApi(BaseApi): 'error': str(err) }, 500 + def get_log(self, id): + task = db_manager.get('tasks', id=id) + node = db_manager.get('nodes', id=task['node_id']) + r = requests.get('http://%s:%s/api/tasks/%s/on_get_log' % ( + node['ip'], + node['port'], + id + )) + if r.status_code == 200: + data = json.loads(r.content.decode('utf-8')) + return { + 'status': 'ok', + 'log': data.get('log') + } + else: + data = json.loads(r.content) + return { + 'code': 500, + 'status': 'ok', + 'error': data['error'] + }, 500 + def get_results(self, id): task = db_manager.get('tasks', id=id) spider = db_manager.get('spiders', id=task['spider_id']) diff --git a/frontend/src/components/Common/DialogView.vue b/frontend/src/components/Common/DialogView.vue index 41917279..0f1f5932 100644 --- a/frontend/src/components/Common/DialogView.vue +++ b/frontend/src/components/Common/DialogView.vue @@ -133,10 +133,7 @@ export default { this.$store.commit('dialogView/SET_DIALOG_VISIBLE', false) }) } else if (this.dialogType === 'spiderRun') { - this.$store.dispatch('spider/crawlSpider', { - id: this.spiderForm._id.$oid, - nodeId: this.activeNode._id - }) + this.$store.dispatch('spider/crawlSpider', this.spiderForm._id.$oid) .then(() => { this.$message.success(`Spider "${this.spiderForm.name}" started to run on node "${this.activeNode._id}"`) }) diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index 417db03f..4265e58d 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -42,7 +42,7 @@ Run - + Deploy Save @@ -78,12 +78,12 @@ export default { const row = this.spiderForm this.$refs['spiderForm'].validate(res => { if (res) { - this.$confirm('Are you sure to run this spider', 'Notice', { + this.$confirm('Are you sure to run this spider?', 'Notice', { confirmButtonText: 'Confirm', cancelButtonText: 'Cancel' }) .then(() => { - this.$store.dispatch('spider/crawlSpider', { id: row._id.$oid }) + this.$store.dispatch('spider/crawlSpider', row._id.$oid) .then(() => { this.$message.success(`Running spider "${row._id.$oid}" has been scheduled`) }) @@ -92,8 +92,21 @@ export default { }) }, onDeploy () { - this.$store.commit('dialogView/SET_DIALOG_VISIBLE', true) - this.$store.commit('dialogView/SET_DIALOG_TYPE', 'spiderDeploy') + const row = this.spiderForm + this.$refs['spiderForm'].validate(res => { + if (res) { + this.$confirm('Are you sure to deploy this spider?', 'Notice', { + confirmButtonText: 'Confirm', + cancelButtonText: 'Cancel' + }) + .then(() => { + this.$store.dispatch('spider/crawlSpider', row._id.$oid) + .then(() => { + this.$message.success(`Spider "${row._id.$oid}" has been deployed`) + }) + }) + } + }) }, onSave () { this.$refs['spiderForm'].validate(res => { diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js index 9040a0f0..7da95e4f 100644 --- a/frontend/src/router/index.js +++ b/frontend/src/router/index.js @@ -166,8 +166,19 @@ export const constantRouterMap = [ { path: '*', redirect: '/404', hidden: true } ] -export default new Router({ +const router = new Router({ // mode: 'history', //后端支持可开 scrollBehavior: () => ({ y: 0 }), routes: constantRouterMap }) + +router.beforeEach((to, from, next) => { + if (to.meta && to.meta.title) { + window.document.title = `Crawlab - ${to.meta.title}` + } else { + window.document.title = 'Crawlab' + } + next() +}) + +export default router diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js index 548a26ba..9514e08f 100644 --- a/frontend/src/store/modules/spider.js +++ b/frontend/src/store/modules/spider.js @@ -79,16 +79,13 @@ const actions = { commit('SET_SPIDER_FORM', response.data) }) }, - deploySpider ({ state, dispatch }, { id, nodeId }) { - return request.post(`/spiders/${id}/deploy`, { - node_id: nodeId - }) + deploySpider ({ state, dispatch }, id) { + return request.post(`/spiders/${id}/deploy`) .then(response => { console.log(response.data) }) }, - crawlSpider ({ state, dispatch }, payload) { - const { id } = payload + crawlSpider ({ state, dispatch }, id) { return request.post(`/spiders/${id}/on_crawl`) .then(response => { console.log(response.data) diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index ce71222d..cdcfc497 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -241,9 +241,19 @@ export default { }) }, onDeploy (row) { - this.$store.dispatch('spider/getSpiderData', row._id.$oid) - this.$store.commit('dialogView/SET_DIALOG_VISIBLE', true) - this.$store.commit('dialogView/SET_DIALOG_TYPE', 'spiderDeploy') + this.$confirm('Are you sure to deploy this spider?', 'Notification', { + confirmButtonText: 'Confirm', + cancelButtonText: 'Cancel', + type: 'warning' + }).then(() => { + this.$store.dispatch('spider/deploySpider', row._id.$oid) + .then(() => { + this.$message({ + type: 'success', + message: 'Deployed successfully' + }) + }) + }) }, onCrawl (row) { this.$confirm('Are you sure to run this spider', 'Notice', { @@ -251,7 +261,7 @@ export default { cancelButtonText: 'Cancel' }) .then(() => { - this.$store.dispatch('spider/crawlSpider', { id: row._id.$oid }) + this.$store.dispatch('spider/crawlSpider', row._id.$oid) .then(() => { this.$message.success(`Running spider "${row._id.$oid}" has been scheduled`) })