git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

support multi-node log view

Browse Source
This commit is contained in:
Marvin Zhang
2019-03-06 15:43:14 +08:00
parent 437e4f7e3f
commit a71f9488ec
8 changed files with 102 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
crawlab/bin/run_app.py Normal file
View File

@@ -0,0 +1,17 @@
import sys
import os
# make sure the working directory is in system path
file_dir = os.path.dirname(os.path.realpath(__file__))
root_path = os.path.abspath(os.path.join(file_dir, '..'))
sys.path.append(root_path)
from config import PROJECT_LOGS_FOLDER, FLASK_HOST, FLASK_PORT
from manage import app
# create folder if it does not exist
if not os.path.exists(PROJECT_LOGS_FOLDER):
os.makedirs(PROJECT_LOGS_FOLDER)
# run app instance
app.run(host=FLASK_HOST, port=FLASK_PORT)

48
crawlab/routes/spiders.py
View File

@@ -151,24 +151,12 @@ class SpiderApi(BaseApi):
}
def deploy(self, id):
args = self.parser.parse_args()
node_id = args.get('node_id')
spider = db_manager.get('spiders', id=id)
nodes = db_manager.list('nodes', {})
# get spider given the id
spider = db_manager.get(col_name=self.col_name, id=id)
if spider is None:
return
for node in nodes:
node_id = node['_id']
# get node given the node
node = db_manager.get(col_name='nodes', id=node_id)
# make source / destination
src = spider.get('src')
# copy files
# TODO: multi-node copy files
try:
# TODO: deploy spiders to other node rather than in local machine
output_file_name = '%s_%s.zip' % (
datetime.now().strftime('%Y%m%d%H%M%S'),
str(random())[2:12]
@@ -176,7 +164,7 @@ class SpiderApi(BaseApi):
output_file_path = os.path.join(PROJECT_TMP_FOLDER, output_file_name)
# zip source folder to zip file
zip_file(source_dir=src,
zip_file(source_dir=spider['src'],
output_filename=output_file_path)
# upload to api
@@ -188,27 +176,11 @@ class SpiderApi(BaseApi):
node_id,
), files=files)
if r.status_code == 200:
return {
'code': 200,
'status': 'ok',
'message': 'deploy success'
}
else:
return {
'code': r.status_code,
'status': 'ok',
'error': r.content.decode('utf-8')
}, r.status_code
except Exception as err:
current_app.logger.error(err)
return {
'code': 500,
'status': 'ok',
'error': str(err)
}, 500
return {
'code': 200,
'status': 'ok',
'message': 'deploy success'
}
def deploy_file(self, id=None):
args = parser.parse_args()

28
crawlab/routes/tasks.py
View File

@@ -1,3 +1,7 @@
import json
import requests
from constants.task import TaskStatus
from db.manager import db_manager
from routes.base import BaseApi
@@ -57,7 +61,7 @@ class TaskApi(BaseApi):
'items': items
})
def get_log(self, id):
def on_get_log(self, id):
try:
task = db_manager.get('tasks', id=id)
with open(task['log_file_path']) as f:
@@ -73,6 +77,28 @@ class TaskApi(BaseApi):
'error': str(err)
}, 500
def get_log(self, id):
task = db_manager.get('tasks', id=id)
node = db_manager.get('nodes', id=task['node_id'])
r = requests.get('http://%s:%s/api/tasks/%s/on_get_log' % (
node['ip'],
node['port'],
id
))
if r.status_code == 200:
data = json.loads(r.content.decode('utf-8'))
return {
'status': 'ok',
'log': data.get('log')
}
else:
data = json.loads(r.content)
return {
'code': 500,
'status': 'ok',
'error': data['error']
}, 500
def get_results(self, id):
task = db_manager.get('tasks', id=id)
spider = db_manager.get('spiders', id=task['spider_id'])

5
frontend/src/components/Common/DialogView.vue
View File

@@ -133,10 +133,7 @@ export default {
this.$store.commit('dialogView/SET_DIALOG_VISIBLE', false)
})
} else if (this.dialogType === 'spiderRun') {
this.$store.dispatch('spider/crawlSpider', {
id: this.spiderForm._id.$oid,
nodeId: this.activeNode._id
})
this.$store.dispatch('spider/crawlSpider', this.spiderForm._id.$oid)
.then(() => {
this.$message.success(`Spider "${this.spiderForm.name}" started to run on node "${this.activeNode._id}"`)
})

23
frontend/src/components/InfoView/SpiderInfoView.vue
View File

@@ -42,7 +42,7 @@
</el-row>
<el-row class="button-container" v-if="!isView">
<el-button type="danger" @click="onRun">Run</el-button>
<!--<el-button type="primary" @click="onDeploy">Deploy</el-button>-->
<el-button type="primary" @click="onDeploy">Deploy</el-button>
<el-button type="success" @click="onSave">Save</el-button>
</el-row>
</div>
@@ -78,12 +78,12 @@ export default {
const row = this.spiderForm
this.$refs['spiderForm'].validate(res => {
if (res) {
this.$confirm('Are you sure to run this spider', 'Notice', {
this.$confirm('Are you sure to run this spider?', 'Notice', {
confirmButtonText: 'Confirm',
cancelButtonText: 'Cancel'
})
.then(() => {
this.$store.dispatch('spider/crawlSpider', { id: row._id.$oid })
this.$store.dispatch('spider/crawlSpider', row._id.$oid)
.then(() => {
this.$message.success(`Running spider "${row._id.$oid}" has been scheduled`)
})
@@ -92,8 +92,21 @@ export default {
})
},
onDeploy () {
this.$store.commit('dialogView/SET_DIALOG_VISIBLE', true)
this.$store.commit('dialogView/SET_DIALOG_TYPE', 'spiderDeploy')
const row = this.spiderForm
this.$refs['spiderForm'].validate(res => {
if (res) {
this.$confirm('Are you sure to deploy this spider?', 'Notice', {
confirmButtonText: 'Confirm',
cancelButtonText: 'Cancel'
})
.then(() => {
this.$store.dispatch('spider/crawlSpider', row._id.$oid)
.then(() => {
this.$message.success(`Spider "${row._id.$oid}" has been deployed`)
})
})
}
})
},
onSave () {
this.$refs['spiderForm'].validate(res => {

13
frontend/src/router/index.js
View File

@@ -166,8 +166,19 @@ export const constantRouterMap = [
{ path: '*', redirect: '/404', hidden: true }
]
export default new Router({
const router = new Router({
// mode: 'history', //后端支持可开
scrollBehavior: () => ({ y: 0 }),
routes: constantRouterMap
})
router.beforeEach((to, from, next) => {
if (to.meta && to.meta.title) {
window.document.title = `Crawlab - ${to.meta.title}`
} else {
window.document.title = 'Crawlab'
}
next()
})
export default router

9
frontend/src/store/modules/spider.js
View File

@@ -79,16 +79,13 @@ const actions = {
commit('SET_SPIDER_FORM', response.data)
})
},
deploySpider ({ state, dispatch }, { id, nodeId }) {
return request.post(`/spiders/${id}/deploy`, {
node_id: nodeId
})
deploySpider ({ state, dispatch }, id) {
return request.post(`/spiders/${id}/deploy`)
.then(response => {
console.log(response.data)
})
},
crawlSpider ({ state, dispatch }, payload) {
const { id } = payload
crawlSpider ({ state, dispatch }, id) {
return request.post(`/spiders/${id}/on_crawl`)
.then(response => {
console.log(response.data)

18
frontend/src/views/spider/SpiderList.vue
View File

@@ -241,9 +241,19 @@ export default {
})
},
onDeploy (row) {
this.$store.dispatch('spider/getSpiderData', row._id.$oid)
this.$store.commit('dialogView/SET_DIALOG_VISIBLE', true)
this.$store.commit('dialogView/SET_DIALOG_TYPE', 'spiderDeploy')
this.$confirm('Are you sure to deploy this spider?', 'Notification', {
confirmButtonText: 'Confirm',
cancelButtonText: 'Cancel',
type: 'warning'
}).then(() => {
this.$store.dispatch('spider/deploySpider', row._id.$oid)
.then(() => {
this.$message({
type: 'success',
message: 'Deployed successfully'
})
})
})
},
onCrawl (row) {
this.$confirm('Are you sure to run this spider', 'Notice', {
@@ -251,7 +261,7 @@ export default {
cancelButtonText: 'Cancel'
})
.then(() => {
this.$store.dispatch('spider/crawlSpider', { id: row._id.$oid })
this.$store.dispatch('spider/crawlSpider', row._id.$oid)
.then(() => {
this.$message.success(`Running spider "${row._id.$oid}" has been scheduled`)
})