git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #355 from wo10378931/develop

Browse Source 
Develop
This commit is contained in:
Marvin Zhang
2019-12-09 14:39:24 +08:00
committed by GitHub
parent 225930793b 069d861be9
commit 15fe92dac9
19 changed files with 364 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/constants/schedule.go Normal file
View File

@@ -0,0 +1,10 @@
package constants
const (
ScheduleStatusStop = "stop"
ScheduleStatusRunning = "running"
ScheduleStatusError = "error"
ScheduleStatusErrorNotFoundNode = "Not Found Node"
ScheduleStatusErrorNotFoundSpider = "Not Found Spider"
)

52
backend/database/redis.go
View File

@@ -4,10 +4,12 @@ import (
"context"
"crawlab/entity"
"crawlab/utils"
"errors"
"github.com/apex/log"
"github.com/gomodule/redigo/redis"
"github.com/spf13/viper"
"runtime/debug"
"strings"
"time"
)
@@ -17,9 +19,18 @@ type Redis struct {
pool *redis.Pool
}
type Mutex struct {
Name string
expiry time.Duration
tries int
delay time.Duration
value string
}
func NewRedisClient() *Redis {
return &Redis{pool: NewRedisPool()}
}
func (r *Redis) RPush(collection string, value interface{}) error {
c := r.pool.Get()
defer utils.Close(c)
@@ -143,3 +154,44 @@ func Sub(channel string, consume ConsumeFunc) error {
}
return nil
}
func (r *Redis) getLockKey(lockKey string) string {
lockKey = strings.ReplaceAll(lockKey, ":", "-")
return "nodes:lock:" + lockKey
}
func (r *Redis) Lock(lockKey string) error {
c := r.pool.Get()
defer utils.Close(c)
lockKey = r.getLockKey(lockKey)
ts := time.Now()
v, err := c.Do("SET", lockKey, ts, "NX", "PX", 30000)
if err != nil {
log.Errorf("get lock fail with error: %s", err.Error())
debug.PrintStack()
return err
}
if err == nil && v == nil {
log.Errorf("the lockKey is locked: key=%s", lockKey)
return errors.New("the lockKey is locked")
}
return nil
}
func (r *Redis) UnLock(lockKey string) {
c := r.pool.Get()
defer utils.Close(c)
lockKey = r.getLockKey(lockKey)
v, err := c.Do("DEL", lockKey)
if err != nil {
log.Errorf("unlock failed, error: %s", err.Error())
debug.PrintStack()
return
}
if v.(int64) == 0 {
log.Errorf("unlock failed: key=%s", lockKey)
return
}
}

13
backend/main.go
View File

@@ -154,17 +154,20 @@ func main() {
authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情
authGroup.PUT("/tasks", routes.PutTask) // 派发任务
authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务
authGroup.DELETE("/tasks_multiple", routes.DeleteMultipleTask) // 删除多个任务
authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) //删除指定状态的任务
authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务
authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志
authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果
authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果
// 定时任务
authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表
authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情
authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务
authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表
authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情
authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务
authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
authGroup.POST("/schedules/:id/stop", routes.StopSchedule) // 停止定时任务
authGroup.POST("/schedules/:id/run", routes.RunSchedule) // 运行定时任务
// 统计数据
authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据
// 用户

2
backend/model/node.go
View File

@@ -169,7 +169,7 @@ func GetNode(id bson.ObjectId) (Node, error) {
defer s.Close()
if err := c.FindId(id).One(&node); err != nil {
log.Errorf(err.Error())
log.Errorf("get node error: %s, id: %s", err.Error(), id.Hex())
debug.PrintStack()
return node, err
}

19
backend/model/schedule.go
View File

@@ -21,10 +21,13 @@ type Schedule struct {
Cron string `json:"cron" bson:"cron"`
EntryId cron.EntryID `json:"entry_id" bson:"entry_id"`
Param string `json:"param" bson:"param"`
// 状态
Status string `json:"status" bson:"status"`
// 前端展示
SpiderName string `json:"spider_name" bson:"spider_name"`
NodeName string `json:"node_name" bson:"node_name"`
Message string `json:"message" bson:"message"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
@@ -86,21 +89,23 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) {
// 选择单一节点
node, err := GetNode(schedule.NodeId)
if err != nil {
log.Errorf(err.Error())
continue
schedule.Status = constants.ScheduleStatusError
schedule.Message = constants.ScheduleStatusErrorNotFoundNode
} else {
schedule.NodeName = node.Name
}
schedule.NodeName = node.Name
}
// 获取爬虫名称
spider, err := GetSpider(schedule.SpiderId)
if err != nil && err == mgo.ErrNotFound {
log.Errorf("get spider by id: %s, error: %s", schedule.SpiderId.Hex(), err.Error())
debug.PrintStack()
_ = schedule.Delete()
continue
schedule.Status = constants.ScheduleStatusError
schedule.Message = constants.ScheduleStatusErrorNotFoundSpider
} else {
schedule.SpiderName = spider.Name
}
schedule.SpiderName = spider.Name
schs = append(schs, schedule)
}
return schs, nil

4
backend/model/task.go
View File

@@ -61,6 +61,7 @@ func (t *Task) Save() error {
defer s.Close()
t.UpdateTs = time.Now()
if err := c.UpdateId(t.Id, t); err != nil {
log.Errorf("update task error: %s", err.Error())
debug.PrintStack()
return err
}
@@ -152,14 +153,13 @@ func GetTask(id string) (Task, error) {
var task Task
if err := c.FindId(id).One(&task); err != nil {
log.Infof("get task error: %s, id: %s", err.Error(), id)
debug.PrintStack()
return task, err
}
return task, nil
}
func AddTask(item Task) error {
s, c := database.GetCol("tasks")
defer s.Close()

52
backend/routes/schedule.go
View File

@@ -14,11 +14,7 @@ func GetScheduleList(c *gin.Context) {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: results,
})
HandleSuccessData(c, results)
}
func GetSchedule(c *gin.Context) {
@@ -29,11 +25,8 @@ func GetSchedule(c *gin.Context) {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: result,
})
HandleSuccessData(c, result)
}
func PostSchedule(c *gin.Context) {
@@ -48,7 +41,7 @@ func PostSchedule(c *gin.Context) {
// 验证cron表达式
if err := services.ParserCron(newItem.Cron); err != nil {
HandleError(http.StatusOK, c, err)
HandleError(http.StatusInternalServerError, c, err)
return
}
@@ -65,10 +58,7 @@ func PostSchedule(c *gin.Context) {
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
HandleSuccess(c)
}
func PutSchedule(c *gin.Context) {
@@ -82,7 +72,7 @@ func PutSchedule(c *gin.Context) {
// 验证cron表达式
if err := services.ParserCron(item.Cron); err != nil {
HandleError(http.StatusOK, c, err)
HandleError(http.StatusInternalServerError, c, err)
return
}
@@ -98,10 +88,7 @@ func PutSchedule(c *gin.Context) {
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
HandleSuccess(c)
}
func DeleteSchedule(c *gin.Context) {
@@ -119,8 +106,25 @@ func DeleteSchedule(c *gin.Context) {
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
HandleSuccess(c)
}
// 停止定时任务
func StopSchedule(c *gin.Context) {
id := c.Param("id")
if err := services.Sched.Stop(bson.ObjectIdHex(id)); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
HandleSuccess(c)
}
// 运行定时任务
func RunSchedule(c *gin.Context) {
id := c.Param("id")
if err := services.Sched.Run(bson.ObjectIdHex(id)); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
HandleSuccess(c)
}

68
backend/routes/task.go
View File

@@ -29,7 +29,7 @@ func GetTaskList(c *gin.Context) {
// 绑定数据
data := TaskListRequestData{}
if err := c.ShouldBindQuery(&data); err != nil {
HandleError(http.StatusBadRequest, c, err)
HandleError(http.StatusInternalServerError, c, err)
return
}
if data.PageNum == 0 {
@@ -82,11 +82,7 @@ func GetTask(c *gin.Context) {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: result,
})
HandleSuccessData(c, result)
}
func PutTask(c *gin.Context) {
@@ -100,7 +96,7 @@ func PutTask(c *gin.Context) {
// 绑定数据
var reqBody TaskRequestBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleError(http.StatusBadRequest, c, err)
HandleError(http.StatusInternalServerError, c, err)
return
}
@@ -151,14 +147,10 @@ func PutTask(c *gin.Context) {
}
} else {
HandleErrorF(http.StatusBadRequest, c, "invalid run_type")
HandleErrorF(http.StatusInternalServerError, c, "invalid run_type")
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
HandleSuccess(c)
}
func DeleteTaskByStatus(c *gin.Context) {
@@ -176,12 +168,31 @@ func DeleteTaskByStatus(c *gin.Context) {
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
HandleSuccess(c)
}
// 删除多个任务
func DeleteMultipleTask(c *gin.Context) {
ids := make(map[string][]string)
if err := c.ShouldBindJSON(&ids); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
list := ids["ids"]
for _, id := range list {
if err := services.RemoveLogByTaskId(id); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
if err := model.RemoveTask(id); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
}
HandleSuccess(c)
}
// 删除单个任务
func DeleteTask(c *gin.Context) {
id := c.Param("id")
@@ -190,33 +201,22 @@ func DeleteTask(c *gin.Context) {
HandleError(http.StatusInternalServerError, c, err)
return
}
// 删除task
if err := model.RemoveTask(id); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
HandleSuccess(c)
}
func GetTaskLog(c *gin.Context) {
id := c.Param("id")
logStr, err := services.GetTaskLog(id)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: logStr,
})
HandleSuccessData(c, logStr)
}
func GetTaskResults(c *gin.Context) {
@@ -225,7 +225,7 @@ func GetTaskResults(c *gin.Context) {
// 绑定数据
data := TaskResultsRequestData{}
if err := c.ShouldBindQuery(&data); err != nil {
HandleError(http.StatusBadRequest, c, err)
HandleError(http.StatusInternalServerError, c, err)
return
}
@@ -327,9 +327,5 @@ func CancelTask(c *gin.Context) {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
HandleSuccess(c)
}

23
backend/routes/utils.go
View File

@@ -1,17 +1,15 @@
package routes
import (
"github.com/apex/log"
"github.com/gin-gonic/gin"
"net/http"
"runtime/debug"
)
func HandleError(statusCode int, c *gin.Context, err error) {
log.Errorf("handle error:" + err.Error())
debug.PrintStack()
c.AbortWithStatusJSON(statusCode, Response{
Status: "ok",
Message: "error",
Status: "error",
Message: "failure",
Error: err.Error(),
})
}
@@ -24,3 +22,18 @@ func HandleErrorF(statusCode int, c *gin.Context, err string) {
Error: err,
})
}
func HandleSuccess(c *gin.Context) {
c.AbortWithStatusJSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}
func HandleSuccessData(c *gin.Context, data interface{}) {
c.AbortWithStatusJSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: data,
})
}

15
backend/services/node.go
View File

@@ -95,19 +95,16 @@ func UpdateNodeStatus() {
}
func handleNodeInfo(key string, data Data) {
// 添加同步锁
if err := database.RedisClient.Lock(key); err != nil {
return
}
defer database.RedisClient.UnLock(key)
// 更新节点信息到数据库
s, c := database.GetCol("nodes")
defer s.Close()
// 同个key可能因为并发，被注册多次
var nodes []model.Node
_ = c.Find(bson.M{"key": key}).All(&nodes)
if len(nodes) > 1 {
for _, node := range nodes {
_ = c.RemoveId(node.Id)
}
}
var node model.Node
if err := c.Find(bson.M{"key": key}).One(&node); err != nil {
// 数据库不存在该节点

37
backend/services/schedule.go
View File

@@ -4,7 +4,9 @@ import (
"crawlab/constants"
"crawlab/lib/cron"
"crawlab/model"
"errors"
"github.com/apex/log"
"github.com/globalsign/mgo/bson"
"github.com/satori/go.uuid"
"runtime/debug"
)
@@ -106,6 +108,7 @@ func (s *Scheduler) AddJob(job model.Schedule) error {
// 更新EntryID
job.EntryId = eid
job.Status = constants.ScheduleStatusRunning
if err := job.Save(); err != nil {
log.Errorf("job save error: %s", err.Error())
debug.PrintStack()
@@ -134,6 +137,36 @@ func ParserCron(spec string) error {
return nil
}
// 停止定时任务
func (s *Scheduler) Stop(id bson.ObjectId) error {
schedule, err := model.GetSchedule(id)
if err != nil {
return err
}
if schedule.EntryId == 0 {
return errors.New("entry id not found")
}
s.cron.Remove(schedule.EntryId)
// 更新状态
schedule.Status = constants.ScheduleStatusStop
if err = schedule.Save(); err != nil {
return err
}
return nil
}
// 运行任务
func (s *Scheduler) Run(id bson.ObjectId) error {
schedule, err := model.GetSchedule(id)
if err != nil {
return err
}
if err := s.AddJob(schedule); err != nil {
return err
}
return nil
}
func (s *Scheduler) Update() error {
// 删除所有定时任务
s.RemoveAll()
@@ -151,6 +184,10 @@ func (s *Scheduler) Update() error {
// 单个任务
job := sList[i]
if job.Status == constants.ScheduleStatusStop {
continue
}
// 添加到定时任务
if err := s.AddJob(job); err != nil {
log.Errorf("add job error: %s, job: %s, cron: %s", err.Error(), job.Name, job.Cron)

18
backend/services/task.go
View File

@@ -418,15 +418,23 @@ func ExecuteTask(id int) {
t.Status = constants.StatusRunning // 任务状态
t.WaitDuration = t.StartTs.Sub(t.CreateTs).Seconds() // 等待时长
// 判断爬虫文件是否存在
gfFile := model.GetGridFs(spider.FileId)
if gfFile == nil {
t.Error = "找不到爬虫文件，请重新上传"
t.Status = constants.StatusError
t.FinishTs = time.Now() // 结束时间
t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长
t.TotalDuration = t.FinishTs.Sub(t.CreateTs).Seconds() // 总时长
_ = t.Save()
return
}
// 开始执行任务
log.Infof(GetWorkerPrefix(id) + "开始执行任务(ID:" + t.Id + ")")
// 储存任务
if err := t.Save(); err != nil {
log.Errorf(err.Error())
HandleTaskError(t, err)
return
}
_ = t.Save()
// 起一个cron执行器来统计任务结果数
if spider.Col != "" {

2
frontend/src/api/request.js
View File

@@ -63,7 +63,7 @@ const put = (path, data) => {
}
const del = (path, data) => {
return request('DELETE', path)
return request('DELETE', path, {}, data)
}
export default {

5
frontend/src/i18n/zh.js
View File

@@ -212,6 +212,11 @@ export default {
'Schedule Description': '定时任务描述',
'Parameters': '参数',
'Add Schedule': '添加定时任务',
'stop': '暂停',
'running': '运行',
'error': '错误',
'Not Found Node': '节点配置错误',
'Not Found Spider': '爬虫配置错误',
// 网站
'Site': '网站',

7
frontend/src/store/modules/schedule.js
View File

@@ -1,5 +1,4 @@
import request from '../../api/request'
const state = {
scheduleList: [],
scheduleForm: {}
@@ -31,6 +30,12 @@ const actions = {
},
removeSchedule ({ state }, id) {
request.delete(`/schedules/${id}`)
},
stopSchedule ({ state, dispatch }, id) {
return request.post(`/schedules/${id}/stop`)
},
runSchedule ({ state, dispatch }, id) {
return request.post(`/schedules/${id}/run`)
}
}

1
frontend/src/store/modules/spider.js
View File

@@ -1,6 +1,5 @@
import Vue from 'vue'
import request from '../../api/request'
import axisModelCommonMixin from 'echarts/src/coord/axisModelCommonMixin'
const state = {
// list of spiders

5
frontend/src/store/modules/task.js
View File

@@ -102,6 +102,11 @@ const actions = {
dispatch('getTaskList')
})
},
deleteTaskMultiple ({ state }, ids) {
return request.delete(`/tasks_multiple`, {
ids: ids
})
},
getTaskLog ({ state, commit }, id) {
commit('SET_TASK_LOG', '')
return request.get(`/tasks/${id}/log`)

114
frontend/src/views/schedule/ScheduleList.vue
View File

@@ -102,26 +102,46 @@
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
border>
<template v-for="col in columns">
<el-table-column :key="col.name"
<el-table-column v-if="col.name === 'status'"
:key="col.name"
:property="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{$t(scope.row[col.name])}}
<el-tooltip v-if="scope.row[col.name] === 'error'" :content="$t(scope.row['message'])" placement="top">
<el-tag class="status-tag" type="danger">
{{scope.row[col.name] ? $t(scope.row[col.name]) : $t('NA')}}
</el-tag>
</el-tooltip>
<el-tag class="status-tag" v-else>
{{scope.row[col.name] ? $t(scope.row[col.name]) : $t('NA')}}
</el-tag>
</template>
</el-table-column>
<el-table-column v-else :key="col.name"
:property="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{scope.row[col.name]}}
</template>
</el-table-column>
</template>
<el-table-column :label="$t('Action')" align="left" width="150px" fixed="right">
<el-table-column :label="$t('Action')" align="left" width="180px" fixed="right">
<template slot-scope="scope">
<!-- 编辑 -->
<el-tooltip :content="$t('Edit')" placement="top">
<el-button type="warning" icon="el-icon-edit" size="mini" @click="onEdit(scope.row)"></el-button>
</el-tooltip>
<!-- 删除 -->
<el-tooltip :content="$t('Remove')" placement="top">
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
</el-tooltip>
<el-tooltip v-if="isShowRun(scope.row)" :content="$t('Run')" placement="top">
<el-tooltip content="暂停/运行" placement="top">
<el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
</el-tooltip>
</template>
@@ -149,7 +169,8 @@ export default {
{ name: 'node_name', label: 'Node', width: '150' },
{ name: 'spider_name', label: 'Spider', width: '150' },
{ name: 'param', label: 'Parameters', width: '150' },
{ name: 'description', label: 'Description', width: 'auto' }
{ name: 'description', label: 'Description', width: 'auto' },
{ name: 'status', label: 'Status', width: 'auto' }
],
isEdit: false,
dialogTitle: '',
@@ -216,7 +237,7 @@ export default {
})
this.$st.sendEv('定时任务', '提交')
},
isShowRun () {
isShowRun (row) {
},
onEdit (row) {
this.$store.commit('schedule/SET_SCHEDULE_FORM', row)
@@ -225,34 +246,62 @@ export default {
this.$st.sendEv('定时任务', '修改', 'id', row._id)
},
onRemove (row) {
this.$store.dispatch('schedule/removeSchedule', row._id)
.then(() => {
setTimeout(() => {
this.$store.dispatch('schedule/getScheduleList')
this.$message.success(`Schedule "${row.name}" has been removed`)
}, 100)
})
this.$confirm('确定删除定时任务?', '提示', {
confirmButtonText: '确定',
cancelButtonText: '取消',
type: 'warning'
}).then(() => {
this.$store.dispatch('schedule/removeSchedule', row._id)
.then(() => {
setTimeout(() => {
this.$store.dispatch('schedule/getScheduleList')
this.$message.success(`Schedule "${row.name}" has been removed`)
}, 100)
})
}).catch(() => {})
this.$st.sendEv('定时任务', '删除', 'id', row._id)
},
onCrawl () {
},
onCrontabFill (value) {
value = value.replace(/[?]/g, '*')
this.$set(this.scheduleForm, 'cron', value)
this.$st.sendEv('定时任务', '提交生成Cron', 'cron', this.scheduleForm.cron)
},
onShowCronDialog () {
this.showCron = true
if (this.expression.split(' ').length < 7) {
// this.expression = (this.scheduleForm.cron + ' ').replace(/[?]/g, '*')
this.expression = this.scheduleForm.cron + ' '
} else {
// this.expression = this.scheduleForm.cron.replace(/[?]/g, '*')
this.expression = this.scheduleForm.cron
onCrawl (row) {
// 停止定时任务
if (!row.status || row.status === 'running') {
this.$confirm('确定停止定时任务?', '提示', {
confirmButtonText: '确定',
cancelButtonText: '取消',
type: 'warning'
}).then(() => {
this.$store.dispatch('schedule/stopSchedule', row._id)
.then((resp) => {
if (resp.data.status === 'ok') {
this.$store.dispatch('schedule/getScheduleList')
return
}
this.$message({
type: 'error',
message: resp.data.error
})
})
}).catch(() => {})
}
// 运行定时任务
if (row.status === 'stop') {
this.$confirm('确定运行定时任务?', '提示', {
confirmButtonText: '确定',
cancelButtonText: '取消',
type: 'warning'
}).then(() => {
this.$store.dispatch('schedule/runSchedule', row._id)
.then((resp) => {
if (resp.data.status === 'ok') {
this.$store.dispatch('schedule/getScheduleList')
return
}
this.$message({
type: 'error',
message: resp.data.error
})
})
}).catch(() => {})
}
this.$st.sendEv('定时任务', '点击生成Cron', 'cron', this.scheduleForm.cron)
}
},
created () {
@@ -289,4 +338,7 @@ export default {
min-height: 360px;
margin-top: 10px;
}
.status-tag {
cursor: pointer;
}
</style>

85
frontend/src/views/task/TaskList.vue
View File

@@ -4,31 +4,12 @@
<!--filter-->
<div class="filter">
<div class="left">
<!--<el-select size="small" class="filter-select"-->
<!--v-model="filter.node_id"-->
<!--:placeholder="$t('Node')"-->
<!--filterable-->
<!--clearable-->
<!--@change="onSelectNode">-->
<!--<el-option v-for="op in nodeList" :key="op._id" :value="op._id" :label="op.name"></el-option>-->
<!--</el-select>-->
<!--<el-select size="small" class="filter-select"-->
<!--v-model="filter.spider_id"-->
<!--:placeholder="$t('Spider')"-->
<!--filterable-->
<!--clearable-->
<!--@change="onSelectSpider">-->
<!--<el-option v-for="op in spiderList" :key="op._id" :value="op._id" :label="op.name"></el-option>-->
<!--</el-select>-->
<!--<el-button size="small" type="success"-->
<!--icon="el-icon-search"-->
<!--class="refresh"-->
<!--@click="onRefresh">-->
<!--{{$t('Search')}}-->
<!--</el-button>-->
</div>
<!--<div class="right">-->
<!--</div>-->
<div class="right">
<el-button @click="onRemoveMultipleTask" size="small" type="danger">
删除任务
</el-button>
</div>
</div>
<!--./filter-->
@@ -38,7 +19,9 @@
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
border
@row-click="onRowClick"
@selection-change="onSelectionChange">
>
<el-table-column type="selection" width="55"/>
<template v-for="col in columns">
<el-table-column v-if="col.name === 'spider_name'"
:key="col.name"
@@ -181,7 +164,9 @@ export default {
{ name: 'total_duration', label: 'Total Duration (sec)', width: '80', align: 'right' },
{ name: 'result_count', label: 'Results Count', width: '80' }
// { name: 'avg_num_results', label: 'Average Results Count per Second', width: '80' }
]
],
multipleSelection: []
}
},
computed: {
@@ -228,12 +213,6 @@ export default {
}
return false
})
// .filter((d, index) => {
// // pagination
// const pageNum = this.pageNum
// const pageSize = this.pageSize
// return (pageSize * (pageNum - 1) <= index) && (index < pageSize * pageNum)
// })
}
},
methods: {
@@ -244,11 +223,35 @@ export default {
this.$store.dispatch('task/getTaskList')
this.$st.sendEv('任务', '搜索')
},
onSelectNode () {
this.$st.sendEv('任务', '选择节点')
},
onSelectSpider () {
this.$st.sendEv('任务', '选择爬虫')
onRemoveMultipleTask () {
if (this.multipleSelection.length === 0) {
this.$message({
type: 'error',
message: '请选择要删除的任务'
})
return
}
this.$confirm('确定删除任务', '提示', {
confirmButtonText: '确定',
cancelButtonText: '取消',
type: 'warning'
}).then(() => {
let ids = this.multipleSelection.map(item => item._id)
this.$store.dispatch('task/deleteTaskMultiple', ids).then((resp) => {
if (resp.data.status === 'ok') {
this.$message({
type: 'success',
message: '删除任务成功'
})
this.$store.dispatch('task/getTaskList')
return
}
this.$message({
type: 'error',
message: resp.data.error
})
})
}).catch(() => {})
},
onRemove (row, ev) {
ev.stopPropagation()
@@ -304,6 +307,9 @@ export default {
if (column.label !== this.$t('Action')) {
this.onView(row)
}
},
onSelectionChange (val) {
this.multipleSelection = val
}
},
created () {
@@ -312,10 +318,9 @@ export default {
this.$store.dispatch('node/getNodeList')
},
mounted () {
// request task list every 5 seconds
this.handle = setInterval(() => {
this.$store.dispatch('task/getTaskList')
}, 5000)
// this.handle = setInterval(() => {
// this.$store.dispatch('task/getTaskList')
// }, 5000)
},
destroyed () {
clearInterval(this.handle)