mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-22 17:31:03 +01:00
added spider stats
This commit is contained in:
@@ -95,6 +95,7 @@ func main() {
|
||||
app.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
|
||||
app.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫目录写入
|
||||
app.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
|
||||
app.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
|
||||
// 任务
|
||||
app.GET("/tasks", routes.GetTaskList) // 任务列表
|
||||
app.GET("/tasks/:id", routes.GetTask) // 任务详情
|
||||
@@ -110,6 +111,8 @@ func main() {
|
||||
app.PUT("/schedules", routes.PutSchedule) // 创建定时任务
|
||||
app.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
|
||||
app.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
|
||||
// 统计数据
|
||||
app.GET("/stats/home", routes.GetHomeStats) // 首页统计数据
|
||||
}
|
||||
|
||||
// 路由ping
|
||||
|
||||
@@ -137,3 +137,15 @@ func GetNodeTaskList(id bson.ObjectId) ([]Task, error) {
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
func GetNodeCount(query interface{}) (int, error) {
|
||||
s, c := database.GetCol("nodes")
|
||||
defer s.Close()
|
||||
|
||||
count, err := c.Find(query).Count()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
@@ -129,3 +129,15 @@ func RemoveSchedule(id bson.ObjectId) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetScheduleCount() (int, error) {
|
||||
s, c := database.GetCol("schedules")
|
||||
defer s.Close()
|
||||
|
||||
count, err := c.Count()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
@@ -92,6 +92,8 @@ func (spider *Spider) GetLastTask() (Task, error) {
|
||||
return tasks[0], nil
|
||||
}
|
||||
|
||||
|
||||
|
||||
func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, error) {
|
||||
s, c := database.GetCol("spiders")
|
||||
defer s.Close()
|
||||
@@ -165,3 +167,15 @@ func RemoveSpider(id bson.ObjectId) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetSpiderCount() (int, error) {
|
||||
s, c := database.GetCol("spiders")
|
||||
defer s.Close()
|
||||
|
||||
count, err := c.Count()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"crawlab/constants"
|
||||
"crawlab/database"
|
||||
"github.com/apex/log"
|
||||
"github.com/globalsign/mgo"
|
||||
@@ -10,25 +11,34 @@ import (
|
||||
)
|
||||
|
||||
type Task struct {
|
||||
Id string `json:"_id" bson:"_id"`
|
||||
SpiderId bson.ObjectId `json:"spider_id" bson:"spider_id"`
|
||||
StartTs time.Time `json:"start_ts" bson:"start_ts"`
|
||||
FinishTs time.Time `json:"finish_ts" bson:"finish_ts"`
|
||||
Status string `json:"status" bson:"status"`
|
||||
NodeId bson.ObjectId `json:"node_id" bson:"node_id"`
|
||||
LogPath string `json:"log_path" bson:"log_path"`
|
||||
Cmd string `json:"cmd" bson:"cmd"`
|
||||
Error string `json:"error" bson:"error"`
|
||||
Id string `json:"_id" bson:"_id"`
|
||||
SpiderId bson.ObjectId `json:"spider_id" bson:"spider_id"`
|
||||
StartTs time.Time `json:"start_ts" bson:"start_ts"`
|
||||
FinishTs time.Time `json:"finish_ts" bson:"finish_ts"`
|
||||
Status string `json:"status" bson:"status"`
|
||||
NodeId bson.ObjectId `json:"node_id" bson:"node_id"`
|
||||
LogPath string `json:"log_path" bson:"log_path"`
|
||||
Cmd string `json:"cmd" bson:"cmd"`
|
||||
Error string `json:"error" bson:"error"`
|
||||
ResultCount int `json:"result_count" bson:"result_count"`
|
||||
WaitDuration float64 `json:"wait_duration" bson:"wait_duration"`
|
||||
RuntimeDuration float64 `json:"runtime_duration" bson:"runtime_duration"`
|
||||
TotalDuration float64 `json:"total_duration" bson:"total_duration"`
|
||||
|
||||
// 前端数据
|
||||
SpiderName string `json:"spider_name"`
|
||||
NodeName string `json:"node_name"`
|
||||
NumResults int `json:"num_results"`
|
||||
|
||||
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
|
||||
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
|
||||
}
|
||||
|
||||
type TaskDailyItem struct {
|
||||
Date string `json:"date" bson:"_id"`
|
||||
TaskCount int `json:"task_count" bson:"task_count"`
|
||||
AvgRuntimeDuration float64 `json:"avg_runtime_duration" bson:"avg_runtime_duration"`
|
||||
}
|
||||
|
||||
func (t *Task) GetSpider() (Spider, error) {
|
||||
spider, err := GetSpider(t.SpiderId)
|
||||
if err != nil {
|
||||
@@ -123,17 +133,6 @@ func GetTaskList(filter interface{}, skip int, limit int, sortKey string) ([]Tas
|
||||
} else {
|
||||
tasks[i].NodeName = node.Name
|
||||
}
|
||||
|
||||
// 获取结果数
|
||||
if spider.Col == "" {
|
||||
continue
|
||||
}
|
||||
s, c := database.GetCol(spider.Col)
|
||||
tasks[i].NumResults, err = c.Find(bson.M{"task_id": task.Id}).Count()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
s.Close()
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
@@ -190,3 +189,141 @@ func RemoveTask(id string) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetTaskCount(query interface{}) (int, error) {
|
||||
s, c := database.GetCol("tasks")
|
||||
defer s.Close()
|
||||
|
||||
count, err := c.Find(query).Count()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func GetDailyTaskStats(query bson.M) ([]TaskDailyItem, error) {
|
||||
s, c := database.GetCol("tasks")
|
||||
defer s.Close()
|
||||
|
||||
// 起始日期
|
||||
startDate := time.Now().Add(- 30 * 24 * time.Hour)
|
||||
endDate := time.Now()
|
||||
|
||||
// query
|
||||
query["create_ts"] = bson.M{
|
||||
"$gte": startDate,
|
||||
"$lt": endDate,
|
||||
}
|
||||
|
||||
// match
|
||||
op1 := bson.M{
|
||||
"$match": query,
|
||||
}
|
||||
|
||||
// project
|
||||
op2 := bson.M{
|
||||
"$project": bson.M{
|
||||
"date": bson.M{
|
||||
"$dateToString": bson.M{
|
||||
"format": "%Y%m%d",
|
||||
"date": "$create_ts",
|
||||
"timezone": "Asia/Shanghai",
|
||||
},
|
||||
},
|
||||
"success_count": bson.M{
|
||||
"$cond": []interface{}{
|
||||
bson.M{
|
||||
"$eq": []string{
|
||||
"$status",
|
||||
constants.StatusFinished,
|
||||
},
|
||||
},
|
||||
1,
|
||||
0,
|
||||
},
|
||||
},
|
||||
"runtime_duration": "$runtime_duration",
|
||||
},
|
||||
}
|
||||
|
||||
// group
|
||||
op3 := bson.M{
|
||||
"$group": bson.M{
|
||||
"_id": "$date",
|
||||
"task_count": bson.M{"$sum": 1},
|
||||
"runtime_duration": bson.M{"$sum": "$runtime_duration"},
|
||||
},
|
||||
}
|
||||
|
||||
op4 := bson.M{
|
||||
"$project": bson.M{
|
||||
"task_count": "$task_count",
|
||||
"date": "$date",
|
||||
"avg_runtime_duration": bson.M{
|
||||
"$divide": []string{"$runtime_duration", "$task_count"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// run aggregation
|
||||
var items []TaskDailyItem
|
||||
if err := c.Pipe([]bson.M{op1, op2, op3, op4}).All(&items); err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
// 缓存每日数据
|
||||
dict := make(map[string]TaskDailyItem)
|
||||
for _, item := range items {
|
||||
dict[item.Date] = item
|
||||
}
|
||||
|
||||
// 遍历日期
|
||||
var dailyItems []TaskDailyItem
|
||||
for date := startDate; endDate.Sub(date) > 0; date = date.Add(24 * time.Hour) {
|
||||
dateStr := date.Format("20060102")
|
||||
dailyItems = append(dailyItems, TaskDailyItem{
|
||||
Date: dateStr,
|
||||
TaskCount: dict[dateStr].TaskCount,
|
||||
AvgRuntimeDuration: dict[dateStr].AvgRuntimeDuration,
|
||||
})
|
||||
}
|
||||
|
||||
return dailyItems, nil
|
||||
}
|
||||
|
||||
func UpdateTaskResultCount(id string) (err error) {
|
||||
// 获取任务
|
||||
task, err := GetTask(id)
|
||||
if err != nil {
|
||||
log.Errorf(err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// 获取爬虫
|
||||
spider, err := GetSpider(task.SpiderId)
|
||||
if err != nil {
|
||||
log.Errorf(err.Error())
|
||||
debug.PrintStack()
|
||||
return err
|
||||
}
|
||||
|
||||
// 获取结果数量
|
||||
s, c := database.GetCol(spider.Col)
|
||||
defer s.Close()
|
||||
resultCount, err := c.Find(bson.M{"task_id": task.Id}).Count()
|
||||
if err != nil {
|
||||
log.Errorf(err.Error())
|
||||
debug.PrintStack()
|
||||
return err
|
||||
}
|
||||
|
||||
// 保存结果数量
|
||||
task.ResultCount = resultCount
|
||||
if err := task.Save(); err != nil {
|
||||
log.Errorf(err.Error())
|
||||
debug.PrintStack()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
72
backend/routes/stats.go
Normal file
72
backend/routes/stats.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"crawlab/constants"
|
||||
"crawlab/model"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/globalsign/mgo/bson"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func GetHomeStats(c *gin.Context) {
|
||||
type DataOverview struct {
|
||||
TaskCount int `json:"task_count"`
|
||||
SpiderCount int `json:"spider_count"`
|
||||
ActiveNodeCount int `json:"active_node_count"`
|
||||
ScheduleCount int `json:"schedule_count"`
|
||||
}
|
||||
|
||||
type Data struct {
|
||||
Overview DataOverview `json:"overview"`
|
||||
Daily []model.TaskDailyItem `json:"daily"`
|
||||
}
|
||||
|
||||
// 任务总数
|
||||
taskCount, err := model.GetTaskCount(nil)
|
||||
if err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// 在线节点总数
|
||||
activeNodeCount, err := model.GetNodeCount(bson.M{"status": constants.StatusOnline})
|
||||
if err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// 爬虫总数
|
||||
spiderCount, err := model.GetSpiderCount()
|
||||
if err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// 定时任务数
|
||||
scheduleCount, err := model.GetScheduleCount()
|
||||
if err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// 每日任务数
|
||||
items, err := model.GetDailyTaskStats(bson.M{})
|
||||
if err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, Response{
|
||||
Status: "ok",
|
||||
Message: "success",
|
||||
Data: Data{
|
||||
Overview: DataOverview{
|
||||
ActiveNodeCount: activeNodeCount,
|
||||
TaskCount: taskCount,
|
||||
SpiderCount: spiderCount,
|
||||
ScheduleCount: scheduleCount,
|
||||
},
|
||||
Daily: items,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -104,6 +104,27 @@ func GetCurrentNode() (model.Node, error) {
|
||||
|
||||
// 如果获取失败
|
||||
if err != nil {
|
||||
// 如果为主节点,表示为第一次注册,插入节点信息
|
||||
if IsMaster() {
|
||||
// 获取本机IP地址
|
||||
ip, err := GetIp()
|
||||
if err != nil {
|
||||
debug.PrintStack()
|
||||
return model.Node{}, err
|
||||
}
|
||||
// 生成节点
|
||||
node = model.Node{
|
||||
Id: bson.NewObjectId(),
|
||||
Ip: ip,
|
||||
Name: mac,
|
||||
Mac: mac,
|
||||
IsMaster: true,
|
||||
}
|
||||
if err := node.Add(); err != nil {
|
||||
return node, err
|
||||
}
|
||||
return node, nil
|
||||
}
|
||||
// 增加错误次数
|
||||
errNum++
|
||||
|
||||
|
||||
@@ -142,9 +142,6 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e
|
||||
return
|
||||
}
|
||||
t.Status = constants.StatusCancelled
|
||||
} else if signal == constants.TaskFinish {
|
||||
// 完成进程
|
||||
t.Status = constants.StatusFinished
|
||||
}
|
||||
|
||||
// 保存任务
|
||||
@@ -205,6 +202,17 @@ func GetWorkerPrefix(id int) string {
|
||||
return "[Worker " + strconv.Itoa(id) + "] "
|
||||
}
|
||||
|
||||
// 统计任务结果数
|
||||
func SaveTaskResultCount(id string) func() {
|
||||
return func() {
|
||||
if err := model.UpdateTaskResultCount(id); err != nil {
|
||||
log.Errorf(err.Error())
|
||||
debug.PrintStack()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 执行任务
|
||||
func ExecuteTask(id int) {
|
||||
if LockList[id] {
|
||||
@@ -315,9 +323,10 @@ func ExecuteTask(id int) {
|
||||
}
|
||||
|
||||
// 任务赋值
|
||||
t.NodeId = node.Id // 任务节点信息
|
||||
t.StartTs = time.Now() // 任务开始时间
|
||||
t.Status = constants.StatusRunning // 任务状态
|
||||
t.NodeId = node.Id // 任务节点信息
|
||||
t.StartTs = time.Now() // 任务开始时间
|
||||
t.Status = constants.StatusRunning // 任务状态
|
||||
t.WaitDuration = t.StartTs.Sub(t.CreateTs).Seconds() // 等待时长
|
||||
|
||||
// 开始执行任务
|
||||
log.Infof(GetWorkerPrefix(id) + "开始执行任务(ID:" + t.Id + ")")
|
||||
@@ -329,12 +338,45 @@ func ExecuteTask(id int) {
|
||||
return
|
||||
}
|
||||
|
||||
// 起一个cron执行器来统计任务结果数
|
||||
cronExec := cron.New(cron.WithSeconds())
|
||||
_, err = cronExec.AddFunc("*/5 * * * * *", SaveTaskResultCount(t.Id))
|
||||
if err != nil {
|
||||
log.Errorf(GetWorkerPrefix(id) + err.Error())
|
||||
return
|
||||
}
|
||||
cronExec.Start()
|
||||
defer cronExec.Stop()
|
||||
|
||||
// 执行Shell命令
|
||||
if err := ExecuteShellCmd(cmd, cwd, t, spider); err != nil {
|
||||
log.Errorf(GetWorkerPrefix(id) + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// 更新任务结果数
|
||||
if err := model.UpdateTaskResultCount(t.Id); err != nil {
|
||||
log.Errorf(GetWorkerPrefix(id) + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// 完成进程
|
||||
t, err = model.GetTask(t.Id)
|
||||
if err != nil {
|
||||
log.Errorf(GetWorkerPrefix(id) + err.Error())
|
||||
return
|
||||
}
|
||||
t.Status = constants.StatusFinished // 任务状态: 已完成
|
||||
t.FinishTs = time.Now() // 结束时间
|
||||
t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长
|
||||
t.TotalDuration = t.FinishTs.Sub(t.CreateTs).Seconds() // 总时长
|
||||
|
||||
// 保存任务
|
||||
if err := t.Save(); err != nil {
|
||||
log.Errorf(GetWorkerPrefix(id) + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// 结束计时
|
||||
toc := time.Now()
|
||||
|
||||
|
||||
@@ -74,4 +74,8 @@ export default {
|
||||
width: 100%;
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.el-table {
|
||||
min-height: 360px;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -87,23 +87,9 @@ export default {
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
getIcon (type) {
|
||||
if (type === 1) {
|
||||
return 'fa-file-o'
|
||||
} else if (type === 2) {
|
||||
return 'fa-folder'
|
||||
}
|
||||
},
|
||||
onEdit () {
|
||||
this.isEdit = true
|
||||
},
|
||||
onChange (path) {
|
||||
this.$store.commit('file/SET_CURRENT_PATH', path)
|
||||
},
|
||||
onChangeSubmit () {
|
||||
this.isEdit = false
|
||||
this.$store.dispatch('file/getFileList', { path: this.currentPath })
|
||||
},
|
||||
onItemClick (item) {
|
||||
if (item.is_dir) {
|
||||
// 目录
|
||||
|
||||
@@ -13,24 +13,33 @@
|
||||
<status-tag :status="taskForm.status"/>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Log File Path')">
|
||||
<el-input v-model="taskForm.log_stdout_path" placeholder="Log File Path" disabled></el-input>
|
||||
<el-input v-model="taskForm.log_path" placeholder="Log File Path" disabled></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Create Timestamp')">
|
||||
<el-input v-model="taskForm.create_ts" placeholder="Create Timestamp" disabled></el-input>
|
||||
<el-form-item :label="$t('Create Time')">
|
||||
<el-input :value="getTime(taskForm.create_ts)" placeholder="Create Time" disabled></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Finish Timestamp')">
|
||||
<el-input v-model="taskForm.finish_ts" placeholder="Finish Timestamp" disabled></el-input>
|
||||
<el-form-item :label="$t('Start Time')">
|
||||
<el-input :value="getTime(taskForm.start_ts)" placeholder="Start Time" disabled></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Duration (sec)')">
|
||||
<el-input v-model="taskForm.duration" placeholder="Duration" disabled></el-input>
|
||||
<el-form-item :label="$t('Finish Time')">
|
||||
<el-input :value="getTime(taskForm.finish_ts)" placeholder="Finish Time" disabled></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Wait Duration (sec)')">
|
||||
<el-input :value="getWaitDuration(taskForm)" placeholder="Wait Duration" disabled></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Runtime Duration (sec)')">
|
||||
<el-input :value="getRuntimeDuration(taskForm)" placeholder="Runtime Duration" disabled></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Total Duration (sec)')">
|
||||
<el-input :value="getTotalDuration(taskForm)" placeholder="Runtime Duration" disabled></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Results Count')">
|
||||
<el-input v-model="taskForm.num_results" placeholder="Results Count" disabled></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item :label="$t('Average Results Count per Second')">
|
||||
<el-input v-model="taskForm.avg_num_results" placeholder="Average Results Count per Second" disabled>
|
||||
</el-input>
|
||||
<el-input v-model="taskForm.result_count" placeholder="Results Count" disabled></el-input>
|
||||
</el-form-item>
|
||||
<!--<el-form-item :label="$t('Average Results Count per Second')">-->
|
||||
<!--<el-input v-model="taskForm.avg_num_results" placeholder="Average Results Count per Second" disabled>-->
|
||||
<!--</el-input>-->
|
||||
<!--</el-form-item>-->
|
||||
<el-form-item :label="$t('Error Message')" v-if="taskForm.status === 'error'">
|
||||
<div class="error-message">
|
||||
{{ taskForm.error }}
|
||||
@@ -50,6 +59,7 @@ import {
|
||||
mapState
|
||||
} from 'vuex'
|
||||
import StatusTag from '../Status/StatusTag'
|
||||
import dayjs from 'dayjs'
|
||||
|
||||
export default {
|
||||
name: 'NodeInfoView',
|
||||
@@ -70,6 +80,22 @@ export default {
|
||||
.then(() => {
|
||||
this.$message.success(`Task "${this.$route.params.id}" has been sent signal to stop`)
|
||||
})
|
||||
},
|
||||
getTime (str) {
|
||||
if (!str || str.match('^0001')) return 'NA'
|
||||
return dayjs(str).format('YYYY-MM-DD HH:mm:ss')
|
||||
},
|
||||
getWaitDuration (row) {
|
||||
if (row.start_ts.match('^0001')) return 'NA'
|
||||
return dayjs(row.start_ts).diff(row.create_ts, 'second')
|
||||
},
|
||||
getRuntimeDuration (row) {
|
||||
if (row.finish_ts.match('^0001')) return 'NA'
|
||||
return dayjs(row.finish_ts).diff(row.start_ts, 'second')
|
||||
},
|
||||
getTotalDuration (row) {
|
||||
if (row.finish_ts.match('^0001')) return 'NA'
|
||||
return dayjs(row.finish_ts).diff(row.create_ts, 'second')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,14 +100,14 @@ export default {
|
||||
if (this.masterNode.id === this.nodes[i].id) continue
|
||||
|
||||
// master
|
||||
links.push({
|
||||
source: this.masterNode.id,
|
||||
target: this.nodes[i].id,
|
||||
value: 0.5,
|
||||
lineStyle: {
|
||||
color: '#409EFF'
|
||||
}
|
||||
})
|
||||
// links.push({
|
||||
// source: this.masterNode.id,
|
||||
// target: this.nodes[i].id,
|
||||
// value: 0.5,
|
||||
// lineStyle: {
|
||||
// color: '#409EFF'
|
||||
// }
|
||||
// })
|
||||
}
|
||||
return links
|
||||
}
|
||||
@@ -120,6 +120,7 @@ export default {
|
||||
},
|
||||
tooltip: {
|
||||
formatter: params => {
|
||||
if (!params.data.name) return
|
||||
let str = '<span style="margin-right:5px;display:inline-block;height:12px;width:12px;border-radius:6px;background:' + params.color + '"></span>'
|
||||
if (params.data.name) str += '<span>' + params.data.name + '</span><br>'
|
||||
if (params.data.ip) str += '<span>IP: ' + params.data.ip + '</span><br>'
|
||||
|
||||
@@ -27,7 +27,6 @@ export default {
|
||||
default: ''
|
||||
},
|
||||
value: {
|
||||
type: String,
|
||||
default: ''
|
||||
},
|
||||
type: {
|
||||
|
||||
@@ -17,20 +17,14 @@
|
||||
type="success"/>
|
||||
<metric-card :label="$t('Avg Duration (sec)')"
|
||||
icon="fa fa-hourglass"
|
||||
:value="getDecimal(overviewStats.avg_duration)"
|
||||
:value="getDecimal(overviewStats.avg_runtime_duration)"
|
||||
type="warning"/>
|
||||
</div>
|
||||
</el-row>
|
||||
<!--./overall stats-->
|
||||
|
||||
<el-row>
|
||||
<el-col :span="8">
|
||||
<el-card class="chart-wrapper" style="margin-right: 20px;">
|
||||
<h4>{{$t('Tasks by Status')}}</h4>
|
||||
<div id="task-pie-status" class="chart"></div>
|
||||
</el-card>
|
||||
</el-col>
|
||||
<el-col :span="16">
|
||||
<el-col :span="24">
|
||||
<el-card class="chart-wrapper">
|
||||
<h4>{{$t('Daily Tasks')}}</h4>
|
||||
<div id="task-line" class="chart"></div>
|
||||
@@ -39,13 +33,7 @@
|
||||
</el-row>
|
||||
|
||||
<el-row>
|
||||
<el-col :span="8">
|
||||
<el-card class="chart-wrapper" style="margin-right: 20px;">
|
||||
<h4>{{$t('Tasks by Node')}}</h4>
|
||||
<div id="task-pie-node" class="chart"></div>
|
||||
</el-card>
|
||||
</el-col>
|
||||
<el-col :span="16">
|
||||
<el-col :span="24">
|
||||
<el-card class="chart-wrapper">
|
||||
<h4>{{$t('Daily Avg Duration (sec)')}}</h4>
|
||||
<div id="duration-line" class="chart"></div>
|
||||
@@ -71,64 +59,6 @@ export default {
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
renderTaskPieStatus () {
|
||||
const chart = echarts.init(this.$el.querySelector('#task-pie-status'))
|
||||
const option = {
|
||||
tooltip: {
|
||||
show: true
|
||||
},
|
||||
series: [{
|
||||
name: '',
|
||||
type: 'pie',
|
||||
// radius: ['50%', '70%'],
|
||||
data: this.statusStats.map(d => {
|
||||
let color
|
||||
if (d.name === 'SUCCESS') {
|
||||
color = '#67c23a'
|
||||
} else if (d.name === 'STARTED') {
|
||||
color = '#e6a23c'
|
||||
} else if (d.name === 'FAILURE') {
|
||||
color = '#f56c6c'
|
||||
} else {
|
||||
color = 'grey'
|
||||
}
|
||||
return {
|
||||
name: this.$t(d.name),
|
||||
value: d.value,
|
||||
itemStyle: {
|
||||
color
|
||||
}
|
||||
}
|
||||
})
|
||||
}]
|
||||
}
|
||||
chart.setOption(option)
|
||||
},
|
||||
|
||||
renderTaskPieNode () {
|
||||
const chart = echarts.init(this.$el.querySelector('#task-pie-node'))
|
||||
const option = {
|
||||
tooltip: {
|
||||
show: true
|
||||
},
|
||||
series: [{
|
||||
name: '',
|
||||
type: 'pie',
|
||||
// radius: ['50%', '70%'],
|
||||
data: this.nodeStats.map(d => {
|
||||
return {
|
||||
name: d.name,
|
||||
value: d.value
|
||||
// itemStyle: {
|
||||
// color
|
||||
// }
|
||||
}
|
||||
})
|
||||
}]
|
||||
}
|
||||
chart.setOption(option)
|
||||
},
|
||||
|
||||
renderTaskLine () {
|
||||
const chart = echarts.init(this.$el.querySelector('#task-line'))
|
||||
const option = {
|
||||
@@ -145,7 +75,7 @@ export default {
|
||||
},
|
||||
series: [{
|
||||
type: 'line',
|
||||
data: this.dailyStats.map(d => d.count),
|
||||
data: this.dailyStats.map(d => d.task_count),
|
||||
areaStyle: {},
|
||||
smooth: true
|
||||
}],
|
||||
@@ -173,7 +103,7 @@ export default {
|
||||
},
|
||||
series: [{
|
||||
type: 'line',
|
||||
data: this.dailyStats.map(d => d.duration),
|
||||
data: this.dailyStats.map(d => d.avg_runtime_duration),
|
||||
areaStyle: {},
|
||||
smooth: true
|
||||
}],
|
||||
@@ -186,9 +116,7 @@ export default {
|
||||
},
|
||||
|
||||
render () {
|
||||
this.renderTaskPieStatus()
|
||||
this.renderTaskLine()
|
||||
this.renderTaskPieNode()
|
||||
this.renderDurationLine()
|
||||
},
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<h5 class="title">{{title}}</h5>
|
||||
<el-button type="success" plain class="small-btn" size="mini" icon="fa fa-refresh" @click="onRefresh"></el-button>
|
||||
</el-row>
|
||||
<el-table border height="480px" :data="taskList">
|
||||
<el-table border height="480px" :data="taskList" @row-click="onClickTask">
|
||||
<el-table-column property="node" :label="$t('Node')" width="120" align="left">
|
||||
<template slot-scope="scope">
|
||||
<a class="a-tag" @click="onClickNode(scope.row)">{{scope.row.node_name}}</a>
|
||||
@@ -15,6 +15,11 @@
|
||||
<a class="a-tag" @click="onClickSpider(scope.row)">{{scope.row.spider_name}}</a>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column property="result_count" :label="$t('Results Count')" width="60" align="right">
|
||||
<template slot-scope="scope">
|
||||
<span>{{scope.row.result_count}}</span>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column :label="$t('Status')"
|
||||
align="left"
|
||||
width="100">
|
||||
@@ -23,7 +28,7 @@
|
||||
</template>
|
||||
</el-table-column>
|
||||
<!--<el-table-column property="create_ts" label="Create Time" width="auto" align="center"></el-table-column>-->
|
||||
<el-table-column property="create_ts" :label="$t('Create Time')" width="auto" align="left">
|
||||
<el-table-column property="create_ts" :label="$t('Create Time')" width="150" align="left">
|
||||
<template slot-scope="scope">
|
||||
<a href="javascript:" class="a-tag" @click="onClickTask(scope.row)">
|
||||
{{getTime(scope.row.create_ts).format('YYYY-MM-DD HH:mm:ss')}}
|
||||
@@ -48,7 +53,9 @@ export default {
|
||||
data () {
|
||||
return {
|
||||
// setInterval handle
|
||||
handle: undefined
|
||||
handle: undefined,
|
||||
// 防抖
|
||||
clicked: false
|
||||
}
|
||||
},
|
||||
props: {
|
||||
@@ -64,13 +71,21 @@ export default {
|
||||
},
|
||||
methods: {
|
||||
onClickSpider (row) {
|
||||
this.clicked = true
|
||||
setTimeout(() => {
|
||||
this.clicked = false
|
||||
}, 100)
|
||||
this.$router.push(`/spiders/${row.spider_id}`)
|
||||
},
|
||||
onClickNode (row) {
|
||||
this.clicked = true
|
||||
setTimeout(() => {
|
||||
this.clicked = false
|
||||
}, 100)
|
||||
this.$router.push(`/nodes/${row.node_id}`)
|
||||
},
|
||||
onClickTask (row) {
|
||||
this.$router.push(`/tasks/${row._id}`)
|
||||
if (!this.clicked) this.$router.push(`/tasks/${row._id}`)
|
||||
},
|
||||
onRefresh () {
|
||||
if (this.$route.path.split('/')[1] === 'spiders') {
|
||||
@@ -114,4 +129,8 @@ export default {
|
||||
margin: 0;
|
||||
padding: 5px;
|
||||
}
|
||||
|
||||
.el-table >>> tr {
|
||||
cursor: pointer;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -166,6 +166,9 @@ export default {
|
||||
'Error Message': '错误信息',
|
||||
'Results Count': '结果数',
|
||||
'Average Results Count per Second': '抓取速度(个/秒)',
|
||||
'Wait Duration (sec)': '等待时长(秒)',
|
||||
'Runtime Duration (sec)': '运行时长(秒)',
|
||||
'Total Duration (sec)': '总时长(秒)',
|
||||
|
||||
// 任务列表
|
||||
'Node': '节点',
|
||||
@@ -231,5 +234,6 @@ export default {
|
||||
'Are you sure to delete this spider?': '你确定要删除该爬虫?',
|
||||
'Spider info has been saved successfully': '爬虫信息已成功保存',
|
||||
'Do you allow us to collect some statistics to improve Crawlab?': '您允许我们收集统计数据以更好地优化Crawlab?',
|
||||
'Saved file successfully': '成功保存文件'
|
||||
'Saved file successfully': '成功保存文件',
|
||||
'An error happened when fetching the data': '请求数据时出错'
|
||||
}
|
||||
|
||||
@@ -129,12 +129,12 @@ const actions = {
|
||||
return request.post('/spiders/import/github', { url })
|
||||
},
|
||||
getSpiderStats ({ state, commit }) {
|
||||
return request.get('/stats/get_spider_stats?spider_id=' + state.spiderForm._id)
|
||||
return request.get(`/spiders/${state.spiderForm._id}/stats`)
|
||||
.then(response => {
|
||||
commit('SET_OVERVIEW_STATS', response.data.overview)
|
||||
commit('SET_STATUS_STATS', response.data.task_count_by_status)
|
||||
commit('SET_DAILY_STATS', response.data.daily_stats)
|
||||
commit('SET_NODE_STATS', response.data.task_count_by_node)
|
||||
commit('SET_OVERVIEW_STATS', response.data.data.overview)
|
||||
// commit('SET_STATUS_STATS', response.data.task_count_by_status)
|
||||
commit('SET_DAILY_STATS', response.data.data.daily)
|
||||
// commit('SET_NODE_STATS', response.data.task_count_by_node)
|
||||
})
|
||||
},
|
||||
getPreviewCrawlData ({ state, commit }) {
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
<li class="metric-item" v-for="m in metrics" @click="onClickMetric(m)" :key="m.name">
|
||||
<el-card class="metric-card" shadow="hover">
|
||||
<el-col :span="6" class="icon-col">
|
||||
<i :class="m.icon" :style="{color:m.color}"></i>
|
||||
<font-awesome-icon :icon="m.icon" :color="m.color"/>
|
||||
<!--<i :class="m.icon" :style="{color:m.color}"></i>-->
|
||||
</el-col>
|
||||
<el-col :span="18" class="text-col">
|
||||
<el-row>
|
||||
@@ -40,10 +41,10 @@ export default {
|
||||
overviewStats: {},
|
||||
dailyTasks: [],
|
||||
metrics: [
|
||||
{ name: 'task_count', label: 'Total Tasks', icon: 'fa fa-play', color: '#f56c6c', path: 'tasks' },
|
||||
{ name: 'spider_count', label: 'Spiders', icon: 'fa fa-bug', color: '#67c23a', path: 'spiders' },
|
||||
{ name: 'node_count', label: 'Active Nodes', icon: 'fa fa-server', color: '#409EFF', path: 'nodes' },
|
||||
{ name: 'deploy_count', label: 'Total Deploys', icon: 'fa fa-cloud', color: '#409EFF', path: 'deploys' }
|
||||
{ name: 'task_count', label: 'Total Tasks', icon: ['fa', 'play'], color: '#f56c6c', path: 'tasks' },
|
||||
{ name: 'spider_count', label: 'Spiders', icon: ['fa', 'bug'], color: '#67c23a', path: 'spiders' },
|
||||
{ name: 'active_node_count', label: 'Active Nodes', icon: ['fa', 'server'], color: '#409EFF', path: 'nodes' },
|
||||
{ name: 'schedule_count', label: 'Schedules', icon: ['fa', 'clock'], color: '#409EFF', path: 'schedules' }
|
||||
]
|
||||
}
|
||||
},
|
||||
@@ -58,7 +59,7 @@ export default {
|
||||
type: 'value'
|
||||
},
|
||||
series: [{
|
||||
data: this.dailyTasks.map(d => d.count),
|
||||
data: this.dailyTasks.map(d => d.task_count),
|
||||
type: 'line',
|
||||
areaStyle: {},
|
||||
smooth: true
|
||||
@@ -76,13 +77,13 @@ export default {
|
||||
}
|
||||
},
|
||||
created () {
|
||||
request.get('/stats/get_home_stats')
|
||||
request.get('/stats/home')
|
||||
.then(response => {
|
||||
// overview stats
|
||||
this.overviewStats = response.data.overview_stats
|
||||
this.overviewStats = response.data.data.overview
|
||||
|
||||
// daily tasks
|
||||
this.dailyTasks = response.data.daily_tasks
|
||||
this.dailyTasks = response.data.data.daily
|
||||
this.initEchartsDailyTasks()
|
||||
})
|
||||
},
|
||||
@@ -160,4 +161,9 @@ export default {
|
||||
.el-card {
|
||||
/*border: 1px solid lightgrey;*/
|
||||
}
|
||||
|
||||
.svg-inline--fa {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -80,49 +80,53 @@
|
||||
<vcrontab @hide="showCron=false" @fill="onCrontabFill" :expression="expression"></vcrontab>
|
||||
</el-dialog>
|
||||
|
||||
<!--filter-->
|
||||
<div class="filter">
|
||||
<div class="right">
|
||||
<el-button type="primary"
|
||||
icon="el-icon-plus"
|
||||
class="refresh"
|
||||
@click="onAdd">
|
||||
{{$t('Add Schedule')}}
|
||||
</el-button>
|
||||
<el-card style="border-radius: 0">
|
||||
<!--filter-->
|
||||
<div class="filter">
|
||||
<div class="right">
|
||||
<el-button type="primary"
|
||||
icon="el-icon-plus"
|
||||
class="refresh"
|
||||
@click="onAdd">
|
||||
{{$t('Add Schedule')}}
|
||||
</el-button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!--./filter-->
|
||||
|
||||
<!--table list-->
|
||||
<el-table :data="filteredTableData"
|
||||
class="table"
|
||||
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
|
||||
border>
|
||||
<template v-for="col in columns">
|
||||
<el-table-column :key="col.name"
|
||||
:property="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<!--table list-->
|
||||
<el-table :data="filteredTableData"
|
||||
class="table"
|
||||
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
|
||||
border>
|
||||
<template v-for="col in columns">
|
||||
<el-table-column :key="col.name"
|
||||
:property="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
{{$t(scope.row[col.name])}}
|
||||
</template>
|
||||
</el-table-column>
|
||||
</template>
|
||||
<el-table-column :label="$t('Action')" align="left" width="250" fixed="right">
|
||||
<template slot-scope="scope">
|
||||
{{$t(scope.row[col.name])}}
|
||||
<el-tooltip :content="$t('Edit')" placement="top">
|
||||
<el-button type="warning" icon="el-icon-edit" size="mini" @click="onEdit(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip :content="$t('Remove')" placement="top">
|
||||
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip v-if="isShowRun(scope.row)" :content="$t('Run')" placement="top">
|
||||
<el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</template>
|
||||
<el-table-column :label="$t('Action')" align="left" width="250" fixed="right">
|
||||
<template slot-scope="scope">
|
||||
<el-tooltip :content="$t('Edit')" placement="top">
|
||||
<el-button type="warning" icon="el-icon-edit" size="mini" @click="onEdit(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip :content="$t('Remove')" placement="top">
|
||||
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip v-if="isShowRun(scope.row)" :content="$t('Run')" placement="top">
|
||||
<el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
</el-table>
|
||||
<!--./table list-->
|
||||
</el-card>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@@ -276,6 +280,7 @@ export default {
|
||||
}
|
||||
|
||||
.table {
|
||||
min-height: 360px;
|
||||
margin-top: 10px;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
</div>
|
||||
|
||||
<!--tabs-->
|
||||
<el-tabs v-model="activeTabName" @tab-click="onTabClick" type="card">
|
||||
<el-tabs v-model="activeTabName" @tab-click="onTabClick" type="border-card">
|
||||
<el-tab-pane :label="$t('Overview')" name="overview">
|
||||
<spider-overview/>
|
||||
</el-tab-pane>
|
||||
@@ -107,18 +107,31 @@ export default {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
position: absolute;
|
||||
right: 20px;
|
||||
right: 48px;
|
||||
/*float: right;*/
|
||||
z-index: 999;
|
||||
margin-top: -7px;
|
||||
margin-top: 5px;
|
||||
}
|
||||
|
||||
.selector .el-select {
|
||||
height: 30px;
|
||||
line-height: 30px;
|
||||
padding-left: 10px;
|
||||
width: 180px;
|
||||
border-radius: 0;
|
||||
}
|
||||
|
||||
.selector .el-select >>> .el-input__icon,
|
||||
.selector .el-select >>> .el-input__inner {
|
||||
border-radius: 0;
|
||||
height: 30px;
|
||||
line-height: 30px;
|
||||
}
|
||||
|
||||
.label {
|
||||
text-align: right;
|
||||
width: 80px;
|
||||
color: #909399;
|
||||
font-weight: 100;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -94,41 +94,6 @@
|
||||
</el-dialog>
|
||||
<!--./customized spider dialog-->
|
||||
|
||||
<!--filter-->
|
||||
<div class="filter">
|
||||
<!--<el-input prefix-icon="el-icon-search"-->
|
||||
<!--:placeholder="$t('Search')"-->
|
||||
<!--class="filter-search"-->
|
||||
<!--v-model="filter.keyword"-->
|
||||
<!--@change="onSearch">-->
|
||||
<!--</el-input>-->
|
||||
<div class="left">
|
||||
<el-autocomplete v-model="filterSite"
|
||||
:placeholder="$t('Site')"
|
||||
clearable
|
||||
:fetch-suggestions="fetchSiteSuggestions"
|
||||
@select="onSiteSelect">
|
||||
</el-autocomplete>
|
||||
</div>
|
||||
<div class="right">
|
||||
<el-button type="primary" icon="fa fa-download" @click="openImportDialog">
|
||||
{{$t('Import Spiders')}}
|
||||
</el-button>
|
||||
<el-button type="success"
|
||||
icon="el-icon-plus"
|
||||
class="btn add"
|
||||
@click="onAdd">
|
||||
{{$t('Add Spider')}}
|
||||
</el-button>
|
||||
<el-button type="success"
|
||||
icon="el-icon-refresh"
|
||||
class="btn refresh"
|
||||
@click="onRefresh">
|
||||
{{$t('Refresh')}}
|
||||
</el-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!--crawl confirm dialog-->
|
||||
<crawl-confirm-dialog
|
||||
:visible="crawlConfirmDialogVisible"
|
||||
@@ -137,89 +102,128 @@
|
||||
/>
|
||||
<!--./crawl confirm dialog-->
|
||||
|
||||
<!--table list-->
|
||||
<el-table :data="filteredTableData"
|
||||
class="table"
|
||||
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
|
||||
border>
|
||||
<template v-for="col in columns">
|
||||
<el-table-column v-if="col.name === 'type'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
align="left"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
<el-tag type="success" v-if="scope.row.type === 'configurable'">{{$t('Configurable')}}</el-tag>
|
||||
<el-tag type="primary" v-else-if="scope.row.type === 'customized'">{{$t('Customized')}}</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'last_5_errors'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:width="col.width"
|
||||
align="center">
|
||||
<template slot-scope="scope">
|
||||
<div :style="{color:scope.row[col.name]>0?'red':''}">
|
||||
{{scope.row[col.name]}}
|
||||
</div>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'cmd'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:width="col.width"
|
||||
align="left">
|
||||
<template slot-scope="scope">
|
||||
<el-input v-model="scope.row[col.name]"></el-input>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name.match(/_ts$/)"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
{{getTime(scope.row[col.name])}}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else
|
||||
:key="col.name"
|
||||
:property="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align || 'left'"
|
||||
:width="col.width">
|
||||
</el-table-column>
|
||||
</template>
|
||||
<el-table-column :label="$t('Action')" align="left" width="auto" fixed="right">
|
||||
<template slot-scope="scope">
|
||||
<el-tooltip :content="$t('View')" placement="top">
|
||||
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip :content="$t('Remove')" placement="top">
|
||||
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip v-if="!isShowRun(scope.row)" :content="$t('No command line')" placement="top">
|
||||
<el-button disabled type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip v-else :content="$t('Run')" placement="top">
|
||||
<el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-card style="border-radius: 0">
|
||||
<!--filter-->
|
||||
<div class="filter">
|
||||
<!--<el-input prefix-icon="el-icon-search"-->
|
||||
<!--:placeholder="$t('Search')"-->
|
||||
<!--class="filter-search"-->
|
||||
<!--v-model="filter.keyword"-->
|
||||
<!--@change="onSearch">-->
|
||||
<!--</el-input>-->
|
||||
<div class="left">
|
||||
<el-autocomplete v-model="filterSite"
|
||||
:placeholder="$t('Site')"
|
||||
clearable
|
||||
:fetch-suggestions="fetchSiteSuggestions"
|
||||
@select="onSiteSelect">
|
||||
</el-autocomplete>
|
||||
</div>
|
||||
<div class="right">
|
||||
<el-button v-if="false" type="primary" icon="fa fa-download" @click="openImportDialog">
|
||||
{{$t('Import Spiders')}}
|
||||
</el-button>
|
||||
<el-button type="success"
|
||||
icon="el-icon-plus"
|
||||
class="btn add"
|
||||
@click="onAdd">
|
||||
{{$t('Add Spider')}}
|
||||
</el-button>
|
||||
<el-button type="success"
|
||||
icon="el-icon-refresh"
|
||||
class="btn refresh"
|
||||
@click="onRefresh">
|
||||
{{$t('Refresh')}}
|
||||
</el-button>
|
||||
</div>
|
||||
</div>
|
||||
<!--./filter-->
|
||||
|
||||
<!--table list-->
|
||||
<el-table :data="filteredTableData"
|
||||
class="table"
|
||||
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
|
||||
border>
|
||||
<template v-for="col in columns">
|
||||
<el-table-column v-if="col.name === 'type'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
align="left"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
<el-tag type="success" v-if="scope.row.type === 'configurable'">{{$t('Configurable')}}</el-tag>
|
||||
<el-tag type="primary" v-else-if="scope.row.type === 'customized'">{{$t('Customized')}}</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'last_5_errors'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:width="col.width"
|
||||
align="center">
|
||||
<template slot-scope="scope">
|
||||
<div :style="{color:scope.row[col.name]>0?'red':''}">
|
||||
{{scope.row[col.name]}}
|
||||
</div>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'cmd'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:width="col.width"
|
||||
align="left">
|
||||
<template slot-scope="scope">
|
||||
<el-input v-model="scope.row[col.name]"></el-input>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name.match(/_ts$/)"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
{{getTime(scope.row[col.name])}}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else
|
||||
:key="col.name"
|
||||
:property="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align || 'left'"
|
||||
:width="col.width">
|
||||
</el-table-column>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
<div class="pagination">
|
||||
<el-pagination
|
||||
@current-change="onPageChange"
|
||||
@size-change="onPageChange"
|
||||
:current-page.sync="pagination.pageNum"
|
||||
:page-sizes="[10, 20, 50, 100]"
|
||||
:page-size.sync="pagination.pageSize"
|
||||
layout="sizes, prev, pager, next"
|
||||
:total="spiderList.length">
|
||||
</el-pagination>
|
||||
</div>
|
||||
<el-table-column :label="$t('Action')" align="left" width="auto" fixed="right">
|
||||
<template slot-scope="scope">
|
||||
<el-tooltip :content="$t('View')" placement="top">
|
||||
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip :content="$t('Remove')" placement="top">
|
||||
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip v-if="!isShowRun(scope.row)" :content="$t('No command line')" placement="top">
|
||||
<el-button disabled type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip v-else :content="$t('Run')" placement="top">
|
||||
<el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
<div class="pagination">
|
||||
<el-pagination
|
||||
@current-change="onPageChange"
|
||||
@size-change="onPageChange"
|
||||
:current-page.sync="pagination.pageNum"
|
||||
:page-sizes="[10, 20, 50, 100]"
|
||||
:page-size.sync="pagination.pageSize"
|
||||
layout="sizes, prev, pager, next"
|
||||
:total="spiderList.length">
|
||||
</el-pagination>
|
||||
</div>
|
||||
<!--./table list-->
|
||||
</el-card>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
|
||||
@@ -1,142 +1,146 @@
|
||||
<template>
|
||||
<div class="app-container">
|
||||
<!--filter-->
|
||||
<div class="filter">
|
||||
<div class="left">
|
||||
<el-select class="filter-select"
|
||||
v-model="filter.node_id"
|
||||
:placeholder="$t('Node')"
|
||||
filterable
|
||||
clearable
|
||||
@change="onSelectNode">
|
||||
<el-option v-for="op in nodeList" :key="op._id" :value="op._id" :label="op.name"></el-option>
|
||||
</el-select>
|
||||
<el-select class="filter-select"
|
||||
v-model="filter.spider_id"
|
||||
:placeholder="$t('Spider')"
|
||||
filterable
|
||||
clearable
|
||||
@change="onSelectSpider">
|
||||
<el-option v-for="op in spiderList" :key="op._id" :value="op._id" :label="op.name"></el-option>
|
||||
</el-select>
|
||||
<el-button type="success"
|
||||
icon="el-icon-search"
|
||||
class="refresh"
|
||||
@click="onRefresh">
|
||||
{{$t('Search')}}
|
||||
</el-button>
|
||||
<el-card style="border-radius: 0">
|
||||
<!--filter-->
|
||||
<div class="filter">
|
||||
<div class="left">
|
||||
<el-select class="filter-select"
|
||||
v-model="filter.node_id"
|
||||
:placeholder="$t('Node')"
|
||||
filterable
|
||||
clearable
|
||||
@change="onSelectNode">
|
||||
<el-option v-for="op in nodeList" :key="op._id" :value="op._id" :label="op.name"></el-option>
|
||||
</el-select>
|
||||
<el-select class="filter-select"
|
||||
v-model="filter.spider_id"
|
||||
:placeholder="$t('Spider')"
|
||||
filterable
|
||||
clearable
|
||||
@change="onSelectSpider">
|
||||
<el-option v-for="op in spiderList" :key="op._id" :value="op._id" :label="op.name"></el-option>
|
||||
</el-select>
|
||||
<el-button type="success"
|
||||
icon="el-icon-search"
|
||||
class="refresh"
|
||||
@click="onRefresh">
|
||||
{{$t('Search')}}
|
||||
</el-button>
|
||||
</div>
|
||||
<!--<div class="right">-->
|
||||
<!--</div>-->
|
||||
</div>
|
||||
<!--<div class="right">-->
|
||||
<!--</div>-->
|
||||
</div>
|
||||
<!--./filter-->
|
||||
|
||||
<!--table list-->
|
||||
<el-table :data="filteredTableData"
|
||||
class="table"
|
||||
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
|
||||
border>
|
||||
<template v-for="col in columns">
|
||||
<el-table-column v-if="col.name === 'spider_name'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
<a href="javascript:" class="a-tag" @click="onClickSpider(scope.row)">{{scope.row[col.name]}}</a>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name.match(/_ts$/)"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
{{getTime(scope.row[col.name])}}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'wait_duration'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
{{getWaitDuration(scope.row)}}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'runtime_duration'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
{{getRuntimeDuration(scope.row)}}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'total_duration'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
{{getTotalDuration(scope.row)}}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'node_name'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
<a href="javascript:" class="a-tag" @click="onClickNode(scope.row)">{{scope.row[col.name]}}</a>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'status'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
<status-tag :status="scope.row[col.name]"/>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else
|
||||
:key="col.name"
|
||||
:property="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
</el-table-column>
|
||||
</template>
|
||||
<el-table-column :label="$t('Action')" align="left" width="150" fixed="right">
|
||||
<template slot-scope="scope">
|
||||
<el-tooltip :content="$t('View')" placement="top">
|
||||
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip :content="$t('Remove')" placement="top">
|
||||
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<!--table list-->
|
||||
<el-table :data="filteredTableData"
|
||||
class="table"
|
||||
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
|
||||
border>
|
||||
<template v-for="col in columns">
|
||||
<el-table-column v-if="col.name === 'spider_name'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
<a href="javascript:" class="a-tag" @click="onClickSpider(scope.row)">{{scope.row[col.name]}}</a>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name.match(/_ts$/)"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
{{getTime(scope.row[col.name])}}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'wait_duration'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
{{getWaitDuration(scope.row)}}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'runtime_duration'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
{{getRuntimeDuration(scope.row)}}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'total_duration'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
{{getTotalDuration(scope.row)}}
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'node_name'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
<a href="javascript:" class="a-tag" @click="onClickNode(scope.row)">{{scope.row[col.name]}}</a>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else-if="col.name === 'status'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
<template slot-scope="scope">
|
||||
<status-tag :status="scope.row[col.name]"/>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else
|
||||
:key="col.name"
|
||||
:property="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align"
|
||||
:width="col.width">
|
||||
</el-table-column>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
<div class="pagination">
|
||||
<el-pagination
|
||||
@current-change="onPageChange"
|
||||
@size-change="onPageChange"
|
||||
:current-page.sync="pageNum"
|
||||
:page-sizes="[10, 20, 50, 100]"
|
||||
:page-size.sync="pageSize"
|
||||
layout="sizes, prev, pager, next"
|
||||
:total="taskListTotalCount">
|
||||
</el-pagination>
|
||||
</div>
|
||||
<el-table-column :label="$t('Action')" align="left" width="150" fixed="right">
|
||||
<template slot-scope="scope">
|
||||
<el-tooltip :content="$t('View')" placement="top">
|
||||
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip :content="$t('Remove')" placement="top">
|
||||
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
<div class="pagination">
|
||||
<el-pagination
|
||||
@current-change="onPageChange"
|
||||
@size-change="onPageChange"
|
||||
:current-page.sync="pageNum"
|
||||
:page-sizes="[10, 20, 50, 100]"
|
||||
:page-size.sync="pageSize"
|
||||
layout="sizes, prev, pager, next"
|
||||
:total="taskListTotalCount">
|
||||
</el-pagination>
|
||||
</div>
|
||||
<!--./table list-->
|
||||
</el-card>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@@ -172,7 +176,7 @@ export default {
|
||||
{ name: 'wait_duration', label: 'Wait Duration (sec)', width: '80', align: 'right' },
|
||||
{ name: 'runtime_duration', label: 'Runtime Duration (sec)', width: '80', align: 'right' },
|
||||
{ name: 'total_duration', label: 'Total Duration (sec)', width: '80', align: 'right' },
|
||||
{ name: 'num_results', label: 'Results Count', width: '80' }
|
||||
{ name: 'result_count', label: 'Results Count', width: '80' }
|
||||
// { name: 'avg_num_results', label: 'Average Results Count per Second', width: '80' }
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user