git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

优化爬虫执行逻辑

Browse Source
This commit is contained in:
marvzhang
2020-02-11 15:37:06 +08:00
parent a0c47e54e8
commit fd62adc8ba
5 changed files with 51 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
backend/routes/task.go
View File

@@ -100,6 +100,9 @@ func PutTask(c *gin.Context) {
return
}
// 任务ID
var taskIds []string
if reqBody.RunType == constants.RunTypeAllNodes {
// 所有节点
nodes, err := model.GetNodeList(nil)
@@ -115,10 +118,13 @@ func PutTask(c *gin.Context) {
UserId: services.GetCurrentUser(c).Id,
}
if err := services.AddTask(t); err != nil {
id, err := services.AddTask(t);
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
taskIds = append(taskIds, id)
}
} else if reqBody.RunType == constants.RunTypeRandom {
// 随机
@@ -127,10 +133,12 @@ func PutTask(c *gin.Context) {
Param: reqBody.Param,
UserId: services.GetCurrentUser(c).Id,
}
if err := services.AddTask(t); err != nil {
id, err := services.AddTask(t);
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
taskIds = append(taskIds, id)
} else if reqBody.RunType == constants.RunTypeSelectedNodes {
// 指定节点
for _, nodeId := range reqBody.NodeIds {
@@ -141,16 +149,19 @@ func PutTask(c *gin.Context) {
UserId: services.GetCurrentUser(c).Id,
}
if err := services.AddTask(t); err != nil {
id, err := services.AddTask(t);
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
taskIds = append(taskIds, id)
}
} else {
HandleErrorF(http.StatusInternalServerError, c, "invalid run_type")
return
}
HandleSuccess(c)
HandleSuccessData(c, taskIds)
}
func DeleteTaskByStatus(c *gin.Context) {

6
backend/services/schedule.go
View File

@@ -37,7 +37,7 @@ func AddScheduleTask(s model.Schedule) func() {
UserId: s.UserId,
}
if err := AddTask(t); err != nil {
if _, err := AddTask(t); err != nil {
return
}
}
@@ -49,7 +49,7 @@ func AddScheduleTask(s model.Schedule) func() {
Param: s.Param,
UserId: s.UserId,
}
if err := AddTask(t); err != nil {
if _, err := AddTask(t); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
@@ -65,7 +65,7 @@ func AddScheduleTask(s model.Schedule) func() {
UserId: s.UserId,
}
if err := AddTask(t); err != nil {
if _, err := AddTask(t); err != nil {
return
}
}

8
backend/services/task.go
View File

@@ -666,7 +666,7 @@ func CancelTask(id string) (err error) {
return nil
}
func AddTask(t model.Task) error {
func AddTask(t model.Task) (string, error) {
// 生成任务ID
id := uuid.NewV4()
t.Id = id.String()
@@ -683,17 +683,17 @@ func AddTask(t model.Task) error {
if err := model.AddTask(t); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return err
return t.Id, err
}
// 加入任务队列
if err := AssignTask(t); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return err
return t.Id, err
}
return nil
return t.Id, nil
}
func GetTaskEmailMarkdownContent(t model.Task, s model.Spider) string {

37
frontend/src/components/Common/CrawlConfirmDialog.vue
View File

@@ -30,13 +30,22 @@
<el-input v-model="form.param" :placeholder="$t('Parameters')"></el-input>
</el-form-item>
<el-form-item class="disclaimer-wrapper">
<el-checkbox v-model="isAllowDisclaimer"/>
<span style="margin-left: 5px">我已阅读并同意 <a href="javascript:" @click="onClickDisclaimer">《免责声明》</a> 所有内容</span>
<div>
<el-checkbox v-model="isAllowDisclaimer"/>
<span style="margin-left: 5px">我已阅读并同意 <a href="javascript:" @click="onClickDisclaimer">《免责声明》</a> 所有内容</span>
</div>
<div>
<el-checkbox v-model="isRedirect"/>
<span style="margin-left: 5px">跳转到任务详情页</span>
</div>
</el-form-item>
<el-form-item>
</el-form-item>
</el-form>
<template slot="footer">
<el-button type="plain" size="small" @click="$emit('close')">{{$t('Cancel')}}</el-button>
<el-button type="primary" size="small" @click="onConfirm" :disabled="!isAllowDisclaimer">{{$t('Confirm')}}</el-button>
<el-button type="primary" size="small" @click="onConfirm" :disabled="!isAllowDisclaimer">{{$t('Confirm')}}
</el-button>
</template>
</el-dialog>
</template>
@@ -64,7 +73,8 @@ export default {
param: '',
nodeList: []
},
isAllowDisclaimer: true
isAllowDisclaimer: true,
isRedirect: true
}
},
methods: {
@@ -72,20 +82,27 @@ export default {
this.$emit('close')
},
onConfirm () {
this.$refs['form'].validate(res => {
if (!res) return
this.$refs['form'].validate(async valid => {
if (!valid) return
this.$store.dispatch('spider/crawlSpider', {
const res = await this.$store.dispatch('spider/crawlSpider', {
spiderId: this.spiderId,
nodeIds: this.form.nodeIds,
param: this.form.param,
runType: this.form.runType
})
.then(() => {
this.$message.success(this.$t('A task has been scheduled successfully'))
})
const id = res.data.data[0]
this.$message.success(this.$t('A task has been scheduled successfully'))
this.$emit('close')
this.$st.sendEv('爬虫确认', '确认运行', this.form.runType)
if (this.isRedirect) {
this.$router.push('/tasks/' + id)
this.$st.sendEv('爬虫确认', '跳转到任务详情')
}
})
},
onClickDisclaimer () {

6
frontend/src/views/task/TaskDetail.vue
View File

@@ -10,14 +10,12 @@
<!--./tour-->
<!--tabs-->
<el-tabs v-model="activeTabName" @tab-click="onTabClick" type="card">
<el-tabs v-model="activeTabName" @tab-click="onTabClick" type="border-card">
<el-tab-pane :label="$t('Overview')" name="overview">
<task-overview/>
</el-tab-pane>
<el-tab-pane :label="$t('Log')" name="log">
<el-card>
<log-view :data="taskLog"/>
</el-card>
<log-view :data="taskLog"/>
</el-tab-pane>
<el-tab-pane :label="$t('Results')" name="results">
<div class="button-group">