added spider stats

This commit is contained in:
Marvin Zhang
2019-07-26 17:07:24 +08:00
parent 1e8afe6923
commit 946bb7ceba
22 changed files with 763 additions and 451 deletions

View File

@@ -95,6 +95,7 @@ func main() {
app.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
app.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫目录写入
app.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
app.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
// 任务
app.GET("/tasks", routes.GetTaskList) // 任务列表
app.GET("/tasks/:id", routes.GetTask) // 任务详情
@@ -110,6 +111,8 @@ func main() {
app.PUT("/schedules", routes.PutSchedule) // 创建定时任务
app.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
app.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
// 统计数据
app.GET("/stats/home", routes.GetHomeStats) // 首页统计数据
}
// 路由ping

View File

@@ -137,3 +137,15 @@ func GetNodeTaskList(id bson.ObjectId) ([]Task, error) {
}
return tasks, nil
}
func GetNodeCount(query interface{}) (int, error) {
s, c := database.GetCol("nodes")
defer s.Close()
count, err := c.Find(query).Count()
if err != nil {
return 0, err
}
return count, nil
}

View File

@@ -129,3 +129,15 @@ func RemoveSchedule(id bson.ObjectId) error {
return nil
}
func GetScheduleCount() (int, error) {
s, c := database.GetCol("schedules")
defer s.Close()
count, err := c.Count()
if err != nil {
return 0, err
}
return count, nil
}

View File

@@ -92,6 +92,8 @@ func (spider *Spider) GetLastTask() (Task, error) {
return tasks[0], nil
}
func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, error) {
s, c := database.GetCol("spiders")
defer s.Close()
@@ -165,3 +167,15 @@ func RemoveSpider(id bson.ObjectId) error {
return nil
}
func GetSpiderCount() (int, error) {
s, c := database.GetCol("spiders")
defer s.Close()
count, err := c.Count()
if err != nil {
return 0, err
}
return count, nil
}

View File

@@ -1,6 +1,7 @@
package model
import (
"crawlab/constants"
"crawlab/database"
"github.com/apex/log"
"github.com/globalsign/mgo"
@@ -10,25 +11,34 @@ import (
)
type Task struct {
Id string `json:"_id" bson:"_id"`
SpiderId bson.ObjectId `json:"spider_id" bson:"spider_id"`
StartTs time.Time `json:"start_ts" bson:"start_ts"`
FinishTs time.Time `json:"finish_ts" bson:"finish_ts"`
Status string `json:"status" bson:"status"`
NodeId bson.ObjectId `json:"node_id" bson:"node_id"`
LogPath string `json:"log_path" bson:"log_path"`
Cmd string `json:"cmd" bson:"cmd"`
Error string `json:"error" bson:"error"`
Id string `json:"_id" bson:"_id"`
SpiderId bson.ObjectId `json:"spider_id" bson:"spider_id"`
StartTs time.Time `json:"start_ts" bson:"start_ts"`
FinishTs time.Time `json:"finish_ts" bson:"finish_ts"`
Status string `json:"status" bson:"status"`
NodeId bson.ObjectId `json:"node_id" bson:"node_id"`
LogPath string `json:"log_path" bson:"log_path"`
Cmd string `json:"cmd" bson:"cmd"`
Error string `json:"error" bson:"error"`
ResultCount int `json:"result_count" bson:"result_count"`
WaitDuration float64 `json:"wait_duration" bson:"wait_duration"`
RuntimeDuration float64 `json:"runtime_duration" bson:"runtime_duration"`
TotalDuration float64 `json:"total_duration" bson:"total_duration"`
// 前端数据
SpiderName string `json:"spider_name"`
NodeName string `json:"node_name"`
NumResults int `json:"num_results"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
}
type TaskDailyItem struct {
Date string `json:"date" bson:"_id"`
TaskCount int `json:"task_count" bson:"task_count"`
AvgRuntimeDuration float64 `json:"avg_runtime_duration" bson:"avg_runtime_duration"`
}
func (t *Task) GetSpider() (Spider, error) {
spider, err := GetSpider(t.SpiderId)
if err != nil {
@@ -123,17 +133,6 @@ func GetTaskList(filter interface{}, skip int, limit int, sortKey string) ([]Tas
} else {
tasks[i].NodeName = node.Name
}
// 获取结果数
if spider.Col == "" {
continue
}
s, c := database.GetCol(spider.Col)
tasks[i].NumResults, err = c.Find(bson.M{"task_id": task.Id}).Count()
if err != nil {
continue
}
s.Close()
}
return tasks, nil
}
@@ -190,3 +189,141 @@ func RemoveTask(id string) error {
return nil
}
func GetTaskCount(query interface{}) (int, error) {
s, c := database.GetCol("tasks")
defer s.Close()
count, err := c.Find(query).Count()
if err != nil {
return 0, err
}
return count, nil
}
func GetDailyTaskStats(query bson.M) ([]TaskDailyItem, error) {
s, c := database.GetCol("tasks")
defer s.Close()
// 起始日期
startDate := time.Now().Add(- 30 * 24 * time.Hour)
endDate := time.Now()
// query
query["create_ts"] = bson.M{
"$gte": startDate,
"$lt": endDate,
}
// match
op1 := bson.M{
"$match": query,
}
// project
op2 := bson.M{
"$project": bson.M{
"date": bson.M{
"$dateToString": bson.M{
"format": "%Y%m%d",
"date": "$create_ts",
"timezone": "Asia/Shanghai",
},
},
"success_count": bson.M{
"$cond": []interface{}{
bson.M{
"$eq": []string{
"$status",
constants.StatusFinished,
},
},
1,
0,
},
},
"runtime_duration": "$runtime_duration",
},
}
// group
op3 := bson.M{
"$group": bson.M{
"_id": "$date",
"task_count": bson.M{"$sum": 1},
"runtime_duration": bson.M{"$sum": "$runtime_duration"},
},
}
op4 := bson.M{
"$project": bson.M{
"task_count": "$task_count",
"date": "$date",
"avg_runtime_duration": bson.M{
"$divide": []string{"$runtime_duration", "$task_count"},
},
},
}
// run aggregation
var items []TaskDailyItem
if err := c.Pipe([]bson.M{op1, op2, op3, op4}).All(&items); err != nil {
return items, err
}
// 缓存每日数据
dict := make(map[string]TaskDailyItem)
for _, item := range items {
dict[item.Date] = item
}
// 遍历日期
var dailyItems []TaskDailyItem
for date := startDate; endDate.Sub(date) > 0; date = date.Add(24 * time.Hour) {
dateStr := date.Format("20060102")
dailyItems = append(dailyItems, TaskDailyItem{
Date: dateStr,
TaskCount: dict[dateStr].TaskCount,
AvgRuntimeDuration: dict[dateStr].AvgRuntimeDuration,
})
}
return dailyItems, nil
}
func UpdateTaskResultCount(id string) (err error) {
// 获取任务
task, err := GetTask(id)
if err != nil {
log.Errorf(err.Error())
return err
}
// 获取爬虫
spider, err := GetSpider(task.SpiderId)
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return err
}
// 获取结果数量
s, c := database.GetCol(spider.Col)
defer s.Close()
resultCount, err := c.Find(bson.M{"task_id": task.Id}).Count()
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return err
}
// 保存结果数量
task.ResultCount = resultCount
if err := task.Save(); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return err
}
return nil
}

72
backend/routes/stats.go Normal file
View File

@@ -0,0 +1,72 @@
package routes
import (
"crawlab/constants"
"crawlab/model"
"github.com/gin-gonic/gin"
"github.com/globalsign/mgo/bson"
"net/http"
)
func GetHomeStats(c *gin.Context) {
type DataOverview struct {
TaskCount int `json:"task_count"`
SpiderCount int `json:"spider_count"`
ActiveNodeCount int `json:"active_node_count"`
ScheduleCount int `json:"schedule_count"`
}
type Data struct {
Overview DataOverview `json:"overview"`
Daily []model.TaskDailyItem `json:"daily"`
}
// 任务总数
taskCount, err := model.GetTaskCount(nil)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
// 在线节点总数
activeNodeCount, err := model.GetNodeCount(bson.M{"status": constants.StatusOnline})
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
// 爬虫总数
spiderCount, err := model.GetSpiderCount()
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
// 定时任务数
scheduleCount, err := model.GetScheduleCount()
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
// 每日任务数
items, err := model.GetDailyTaskStats(bson.M{})
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: Data{
Overview: DataOverview{
ActiveNodeCount: activeNodeCount,
TaskCount: taskCount,
SpiderCount: spiderCount,
ScheduleCount: scheduleCount,
},
Daily: items,
},
})
}

View File

@@ -104,6 +104,27 @@ func GetCurrentNode() (model.Node, error) {
// 如果获取失败
if err != nil {
// 如果为主节点,表示为第一次注册,插入节点信息
if IsMaster() {
// 获取本机IP地址
ip, err := GetIp()
if err != nil {
debug.PrintStack()
return model.Node{}, err
}
// 生成节点
node = model.Node{
Id: bson.NewObjectId(),
Ip: ip,
Name: mac,
Mac: mac,
IsMaster: true,
}
if err := node.Add(); err != nil {
return node, err
}
return node, nil
}
// 增加错误次数
errNum++

View File

@@ -142,9 +142,6 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e
return
}
t.Status = constants.StatusCancelled
} else if signal == constants.TaskFinish {
// 完成进程
t.Status = constants.StatusFinished
}
// 保存任务
@@ -205,6 +202,17 @@ func GetWorkerPrefix(id int) string {
return "[Worker " + strconv.Itoa(id) + "] "
}
// 统计任务结果数
func SaveTaskResultCount(id string) func() {
return func() {
if err := model.UpdateTaskResultCount(id); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
}
}
}
// 执行任务
func ExecuteTask(id int) {
if LockList[id] {
@@ -315,9 +323,10 @@ func ExecuteTask(id int) {
}
// 任务赋值
t.NodeId = node.Id // 任务节点信息
t.StartTs = time.Now() // 任务开始时间
t.Status = constants.StatusRunning // 任务状态
t.NodeId = node.Id // 任务节点信息
t.StartTs = time.Now() // 任务开始时间
t.Status = constants.StatusRunning // 任务状态
t.WaitDuration = t.StartTs.Sub(t.CreateTs).Seconds() // 等待时长
// 开始执行任务
log.Infof(GetWorkerPrefix(id) + "开始执行任务(ID:" + t.Id + ")")
@@ -329,12 +338,45 @@ func ExecuteTask(id int) {
return
}
// 起一个cron执行器来统计任务结果数
cronExec := cron.New(cron.WithSeconds())
_, err = cronExec.AddFunc("*/5 * * * * *", SaveTaskResultCount(t.Id))
if err != nil {
log.Errorf(GetWorkerPrefix(id) + err.Error())
return
}
cronExec.Start()
defer cronExec.Stop()
// 执行Shell命令
if err := ExecuteShellCmd(cmd, cwd, t, spider); err != nil {
log.Errorf(GetWorkerPrefix(id) + err.Error())
return
}
// 更新任务结果数
if err := model.UpdateTaskResultCount(t.Id); err != nil {
log.Errorf(GetWorkerPrefix(id) + err.Error())
return
}
// 完成进程
t, err = model.GetTask(t.Id)
if err != nil {
log.Errorf(GetWorkerPrefix(id) + err.Error())
return
}
t.Status = constants.StatusFinished // 任务状态: 已完成
t.FinishTs = time.Now() // 结束时间
t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长
t.TotalDuration = t.FinishTs.Sub(t.CreateTs).Seconds() // 总时长
// 保存任务
if err := t.Save(); err != nil {
log.Errorf(GetWorkerPrefix(id) + err.Error())
return
}
// 结束计时
toc := time.Now()

View File

@@ -74,4 +74,8 @@ export default {
width: 100%;
text-align: right;
}
.el-table {
min-height: 360px;
}
</style>

View File

@@ -87,23 +87,9 @@ export default {
}
},
methods: {
getIcon (type) {
if (type === 1) {
return 'fa-file-o'
} else if (type === 2) {
return 'fa-folder'
}
},
onEdit () {
this.isEdit = true
},
onChange (path) {
this.$store.commit('file/SET_CURRENT_PATH', path)
},
onChangeSubmit () {
this.isEdit = false
this.$store.dispatch('file/getFileList', { path: this.currentPath })
},
onItemClick (item) {
if (item.is_dir) {
// 目录

View File

@@ -13,24 +13,33 @@
<status-tag :status="taskForm.status"/>
</el-form-item>
<el-form-item :label="$t('Log File Path')">
<el-input v-model="taskForm.log_stdout_path" placeholder="Log File Path" disabled></el-input>
<el-input v-model="taskForm.log_path" placeholder="Log File Path" disabled></el-input>
</el-form-item>
<el-form-item :label="$t('Create Timestamp')">
<el-input v-model="taskForm.create_ts" placeholder="Create Timestamp" disabled></el-input>
<el-form-item :label="$t('Create Time')">
<el-input :value="getTime(taskForm.create_ts)" placeholder="Create Time" disabled></el-input>
</el-form-item>
<el-form-item :label="$t('Finish Timestamp')">
<el-input v-model="taskForm.finish_ts" placeholder="Finish Timestamp" disabled></el-input>
<el-form-item :label="$t('Start Time')">
<el-input :value="getTime(taskForm.start_ts)" placeholder="Start Time" disabled></el-input>
</el-form-item>
<el-form-item :label="$t('Duration (sec)')">
<el-input v-model="taskForm.duration" placeholder="Duration" disabled></el-input>
<el-form-item :label="$t('Finish Time')">
<el-input :value="getTime(taskForm.finish_ts)" placeholder="Finish Time" disabled></el-input>
</el-form-item>
<el-form-item :label="$t('Wait Duration (sec)')">
<el-input :value="getWaitDuration(taskForm)" placeholder="Wait Duration" disabled></el-input>
</el-form-item>
<el-form-item :label="$t('Runtime Duration (sec)')">
<el-input :value="getRuntimeDuration(taskForm)" placeholder="Runtime Duration" disabled></el-input>
</el-form-item>
<el-form-item :label="$t('Total Duration (sec)')">
<el-input :value="getTotalDuration(taskForm)" placeholder="Runtime Duration" disabled></el-input>
</el-form-item>
<el-form-item :label="$t('Results Count')">
<el-input v-model="taskForm.num_results" placeholder="Results Count" disabled></el-input>
</el-form-item>
<el-form-item :label="$t('Average Results Count per Second')">
<el-input v-model="taskForm.avg_num_results" placeholder="Average Results Count per Second" disabled>
</el-input>
<el-input v-model="taskForm.result_count" placeholder="Results Count" disabled></el-input>
</el-form-item>
<!--<el-form-item :label="$t('Average Results Count per Second')">-->
<!--<el-input v-model="taskForm.avg_num_results" placeholder="Average Results Count per Second" disabled>-->
<!--</el-input>-->
<!--</el-form-item>-->
<el-form-item :label="$t('Error Message')" v-if="taskForm.status === 'error'">
<div class="error-message">
{{ taskForm.error }}
@@ -50,6 +59,7 @@ import {
mapState
} from 'vuex'
import StatusTag from '../Status/StatusTag'
import dayjs from 'dayjs'
export default {
name: 'NodeInfoView',
@@ -70,6 +80,22 @@ export default {
.then(() => {
this.$message.success(`Task "${this.$route.params.id}" has been sent signal to stop`)
})
},
getTime (str) {
if (!str || str.match('^0001')) return 'NA'
return dayjs(str).format('YYYY-MM-DD HH:mm:ss')
},
getWaitDuration (row) {
if (row.start_ts.match('^0001')) return 'NA'
return dayjs(row.start_ts).diff(row.create_ts, 'second')
},
getRuntimeDuration (row) {
if (row.finish_ts.match('^0001')) return 'NA'
return dayjs(row.finish_ts).diff(row.start_ts, 'second')
},
getTotalDuration (row) {
if (row.finish_ts.match('^0001')) return 'NA'
return dayjs(row.finish_ts).diff(row.create_ts, 'second')
}
}
}

View File

@@ -100,14 +100,14 @@ export default {
if (this.masterNode.id === this.nodes[i].id) continue
// master
links.push({
source: this.masterNode.id,
target: this.nodes[i].id,
value: 0.5,
lineStyle: {
color: '#409EFF'
}
})
// links.push({
// source: this.masterNode.id,
// target: this.nodes[i].id,
// value: 0.5,
// lineStyle: {
// color: '#409EFF'
// }
// })
}
return links
}
@@ -120,6 +120,7 @@ export default {
},
tooltip: {
formatter: params => {
if (!params.data.name) return
let str = '<span style="margin-right:5px;display:inline-block;height:12px;width:12px;border-radius:6px;background:' + params.color + '"></span>'
if (params.data.name) str += '<span>' + params.data.name + '</span><br>'
if (params.data.ip) str += '<span>IP: ' + params.data.ip + '</span><br>'

View File

@@ -27,7 +27,6 @@ export default {
default: ''
},
value: {
type: String,
default: ''
},
type: {

View File

@@ -17,20 +17,14 @@
type="success"/>
<metric-card :label="$t('Avg Duration (sec)')"
icon="fa fa-hourglass"
:value="getDecimal(overviewStats.avg_duration)"
:value="getDecimal(overviewStats.avg_runtime_duration)"
type="warning"/>
</div>
</el-row>
<!--./overall stats-->
<el-row>
<el-col :span="8">
<el-card class="chart-wrapper" style="margin-right: 20px;">
<h4>{{$t('Tasks by Status')}}</h4>
<div id="task-pie-status" class="chart"></div>
</el-card>
</el-col>
<el-col :span="16">
<el-col :span="24">
<el-card class="chart-wrapper">
<h4>{{$t('Daily Tasks')}}</h4>
<div id="task-line" class="chart"></div>
@@ -39,13 +33,7 @@
</el-row>
<el-row>
<el-col :span="8">
<el-card class="chart-wrapper" style="margin-right: 20px;">
<h4>{{$t('Tasks by Node')}}</h4>
<div id="task-pie-node" class="chart"></div>
</el-card>
</el-col>
<el-col :span="16">
<el-col :span="24">
<el-card class="chart-wrapper">
<h4>{{$t('Daily Avg Duration (sec)')}}</h4>
<div id="duration-line" class="chart"></div>
@@ -71,64 +59,6 @@ export default {
}
},
methods: {
renderTaskPieStatus () {
const chart = echarts.init(this.$el.querySelector('#task-pie-status'))
const option = {
tooltip: {
show: true
},
series: [{
name: '',
type: 'pie',
// radius: ['50%', '70%'],
data: this.statusStats.map(d => {
let color
if (d.name === 'SUCCESS') {
color = '#67c23a'
} else if (d.name === 'STARTED') {
color = '#e6a23c'
} else if (d.name === 'FAILURE') {
color = '#f56c6c'
} else {
color = 'grey'
}
return {
name: this.$t(d.name),
value: d.value,
itemStyle: {
color
}
}
})
}]
}
chart.setOption(option)
},
renderTaskPieNode () {
const chart = echarts.init(this.$el.querySelector('#task-pie-node'))
const option = {
tooltip: {
show: true
},
series: [{
name: '',
type: 'pie',
// radius: ['50%', '70%'],
data: this.nodeStats.map(d => {
return {
name: d.name,
value: d.value
// itemStyle: {
// color
// }
}
})
}]
}
chart.setOption(option)
},
renderTaskLine () {
const chart = echarts.init(this.$el.querySelector('#task-line'))
const option = {
@@ -145,7 +75,7 @@ export default {
},
series: [{
type: 'line',
data: this.dailyStats.map(d => d.count),
data: this.dailyStats.map(d => d.task_count),
areaStyle: {},
smooth: true
}],
@@ -173,7 +103,7 @@ export default {
},
series: [{
type: 'line',
data: this.dailyStats.map(d => d.duration),
data: this.dailyStats.map(d => d.avg_runtime_duration),
areaStyle: {},
smooth: true
}],
@@ -186,9 +116,7 @@ export default {
},
render () {
this.renderTaskPieStatus()
this.renderTaskLine()
this.renderTaskPieNode()
this.renderDurationLine()
},

View File

@@ -4,7 +4,7 @@
<h5 class="title">{{title}}</h5>
<el-button type="success" plain class="small-btn" size="mini" icon="fa fa-refresh" @click="onRefresh"></el-button>
</el-row>
<el-table border height="480px" :data="taskList">
<el-table border height="480px" :data="taskList" @row-click="onClickTask">
<el-table-column property="node" :label="$t('Node')" width="120" align="left">
<template slot-scope="scope">
<a class="a-tag" @click="onClickNode(scope.row)">{{scope.row.node_name}}</a>
@@ -15,6 +15,11 @@
<a class="a-tag" @click="onClickSpider(scope.row)">{{scope.row.spider_name}}</a>
</template>
</el-table-column>
<el-table-column property="result_count" :label="$t('Results Count')" width="60" align="right">
<template slot-scope="scope">
<span>{{scope.row.result_count}}</span>
</template>
</el-table-column>
<el-table-column :label="$t('Status')"
align="left"
width="100">
@@ -23,7 +28,7 @@
</template>
</el-table-column>
<!--<el-table-column property="create_ts" label="Create Time" width="auto" align="center"></el-table-column>-->
<el-table-column property="create_ts" :label="$t('Create Time')" width="auto" align="left">
<el-table-column property="create_ts" :label="$t('Create Time')" width="150" align="left">
<template slot-scope="scope">
<a href="javascript:" class="a-tag" @click="onClickTask(scope.row)">
{{getTime(scope.row.create_ts).format('YYYY-MM-DD HH:mm:ss')}}
@@ -48,7 +53,9 @@ export default {
data () {
return {
// setInterval handle
handle: undefined
handle: undefined,
// 防抖
clicked: false
}
},
props: {
@@ -64,13 +71,21 @@ export default {
},
methods: {
onClickSpider (row) {
this.clicked = true
setTimeout(() => {
this.clicked = false
}, 100)
this.$router.push(`/spiders/${row.spider_id}`)
},
onClickNode (row) {
this.clicked = true
setTimeout(() => {
this.clicked = false
}, 100)
this.$router.push(`/nodes/${row.node_id}`)
},
onClickTask (row) {
this.$router.push(`/tasks/${row._id}`)
if (!this.clicked) this.$router.push(`/tasks/${row._id}`)
},
onRefresh () {
if (this.$route.path.split('/')[1] === 'spiders') {
@@ -114,4 +129,8 @@ export default {
margin: 0;
padding: 5px;
}
.el-table >>> tr {
cursor: pointer;
}
</style>

View File

@@ -166,6 +166,9 @@ export default {
'Error Message': '错误信息',
'Results Count': '结果数',
'Average Results Count per Second': '抓取速度(个/秒)',
'Wait Duration (sec)': '等待时长(秒)',
'Runtime Duration (sec)': '运行时长(秒)',
'Total Duration (sec)': '总时长(秒)',
// 任务列表
'Node': '节点',
@@ -231,5 +234,6 @@ export default {
'Are you sure to delete this spider?': '你确定要删除该爬虫?',
'Spider info has been saved successfully': '爬虫信息已成功保存',
'Do you allow us to collect some statistics to improve Crawlab?': '您允许我们收集统计数据以更好地优化Crawlab',
'Saved file successfully': '成功保存文件'
'Saved file successfully': '成功保存文件',
'An error happened when fetching the data': '请求数据时出错'
}

View File

@@ -129,12 +129,12 @@ const actions = {
return request.post('/spiders/import/github', { url })
},
getSpiderStats ({ state, commit }) {
return request.get('/stats/get_spider_stats?spider_id=' + state.spiderForm._id)
return request.get(`/spiders/${state.spiderForm._id}/stats`)
.then(response => {
commit('SET_OVERVIEW_STATS', response.data.overview)
commit('SET_STATUS_STATS', response.data.task_count_by_status)
commit('SET_DAILY_STATS', response.data.daily_stats)
commit('SET_NODE_STATS', response.data.task_count_by_node)
commit('SET_OVERVIEW_STATS', response.data.data.overview)
// commit('SET_STATUS_STATS', response.data.task_count_by_status)
commit('SET_DAILY_STATS', response.data.data.daily)
// commit('SET_NODE_STATS', response.data.task_count_by_node)
})
},
getPreviewCrawlData ({ state, commit }) {

View File

@@ -5,7 +5,8 @@
<li class="metric-item" v-for="m in metrics" @click="onClickMetric(m)" :key="m.name">
<el-card class="metric-card" shadow="hover">
<el-col :span="6" class="icon-col">
<i :class="m.icon" :style="{color:m.color}"></i>
<font-awesome-icon :icon="m.icon" :color="m.color"/>
<!--<i :class="m.icon" :style="{color:m.color}"></i>-->
</el-col>
<el-col :span="18" class="text-col">
<el-row>
@@ -40,10 +41,10 @@ export default {
overviewStats: {},
dailyTasks: [],
metrics: [
{ name: 'task_count', label: 'Total Tasks', icon: 'fa fa-play', color: '#f56c6c', path: 'tasks' },
{ name: 'spider_count', label: 'Spiders', icon: 'fa fa-bug', color: '#67c23a', path: 'spiders' },
{ name: 'node_count', label: 'Active Nodes', icon: 'fa fa-server', color: '#409EFF', path: 'nodes' },
{ name: 'deploy_count', label: 'Total Deploys', icon: 'fa fa-cloud', color: '#409EFF', path: 'deploys' }
{ name: 'task_count', label: 'Total Tasks', icon: ['fa', 'play'], color: '#f56c6c', path: 'tasks' },
{ name: 'spider_count', label: 'Spiders', icon: ['fa', 'bug'], color: '#67c23a', path: 'spiders' },
{ name: 'active_node_count', label: 'Active Nodes', icon: ['fa', 'server'], color: '#409EFF', path: 'nodes' },
{ name: 'schedule_count', label: 'Schedules', icon: ['fa', 'clock'], color: '#409EFF', path: 'schedules' }
]
}
},
@@ -58,7 +59,7 @@ export default {
type: 'value'
},
series: [{
data: this.dailyTasks.map(d => d.count),
data: this.dailyTasks.map(d => d.task_count),
type: 'line',
areaStyle: {},
smooth: true
@@ -76,13 +77,13 @@ export default {
}
},
created () {
request.get('/stats/get_home_stats')
request.get('/stats/home')
.then(response => {
// overview stats
this.overviewStats = response.data.overview_stats
this.overviewStats = response.data.data.overview
// daily tasks
this.dailyTasks = response.data.daily_tasks
this.dailyTasks = response.data.data.daily
this.initEchartsDailyTasks()
})
},
@@ -160,4 +161,9 @@ export default {
.el-card {
/*border: 1px solid lightgrey;*/
}
.svg-inline--fa {
width: 100%;
height: 100%;
}
</style>

View File

@@ -80,49 +80,53 @@
<vcrontab @hide="showCron=false" @fill="onCrontabFill" :expression="expression"></vcrontab>
</el-dialog>
<!--filter-->
<div class="filter">
<div class="right">
<el-button type="primary"
icon="el-icon-plus"
class="refresh"
@click="onAdd">
{{$t('Add Schedule')}}
</el-button>
<el-card style="border-radius: 0">
<!--filter-->
<div class="filter">
<div class="right">
<el-button type="primary"
icon="el-icon-plus"
class="refresh"
@click="onAdd">
{{$t('Add Schedule')}}
</el-button>
</div>
</div>
</div>
<!--./filter-->
<!--table list-->
<el-table :data="filteredTableData"
class="table"
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
border>
<template v-for="col in columns">
<el-table-column :key="col.name"
:property="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<!--table list-->
<el-table :data="filteredTableData"
class="table"
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
border>
<template v-for="col in columns">
<el-table-column :key="col.name"
:property="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{$t(scope.row[col.name])}}
</template>
</el-table-column>
</template>
<el-table-column :label="$t('Action')" align="left" width="250" fixed="right">
<template slot-scope="scope">
{{$t(scope.row[col.name])}}
<el-tooltip :content="$t('Edit')" placement="top">
<el-button type="warning" icon="el-icon-edit" size="mini" @click="onEdit(scope.row)"></el-button>
</el-tooltip>
<el-tooltip :content="$t('Remove')" placement="top">
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
</el-tooltip>
<el-tooltip v-if="isShowRun(scope.row)" :content="$t('Run')" placement="top">
<el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
</el-tooltip>
</template>
</el-table-column>
</template>
<el-table-column :label="$t('Action')" align="left" width="250" fixed="right">
<template slot-scope="scope">
<el-tooltip :content="$t('Edit')" placement="top">
<el-button type="warning" icon="el-icon-edit" size="mini" @click="onEdit(scope.row)"></el-button>
</el-tooltip>
<el-tooltip :content="$t('Remove')" placement="top">
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
</el-tooltip>
<el-tooltip v-if="isShowRun(scope.row)" :content="$t('Run')" placement="top">
<el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
</el-tooltip>
</template>
</el-table-column>
</el-table>
</el-table>
<!--./table list-->
</el-card>
</div>
</template>
@@ -276,6 +280,7 @@ export default {
}
.table {
min-height: 360px;
margin-top: 10px;
}
</style>

View File

@@ -9,7 +9,7 @@
</div>
<!--tabs-->
<el-tabs v-model="activeTabName" @tab-click="onTabClick" type="card">
<el-tabs v-model="activeTabName" @tab-click="onTabClick" type="border-card">
<el-tab-pane :label="$t('Overview')" name="overview">
<spider-overview/>
</el-tab-pane>
@@ -107,18 +107,31 @@ export default {
display: flex;
align-items: center;
position: absolute;
right: 20px;
right: 48px;
/*float: right;*/
z-index: 999;
margin-top: -7px;
margin-top: 5px;
}
.selector .el-select {
height: 30px;
line-height: 30px;
padding-left: 10px;
width: 180px;
border-radius: 0;
}
.selector .el-select >>> .el-input__icon,
.selector .el-select >>> .el-input__inner {
border-radius: 0;
height: 30px;
line-height: 30px;
}
.label {
text-align: right;
width: 80px;
color: #909399;
font-weight: 100;
}
</style>

View File

@@ -94,41 +94,6 @@
</el-dialog>
<!--./customized spider dialog-->
<!--filter-->
<div class="filter">
<!--<el-input prefix-icon="el-icon-search"-->
<!--:placeholder="$t('Search')"-->
<!--class="filter-search"-->
<!--v-model="filter.keyword"-->
<!--@change="onSearch">-->
<!--</el-input>-->
<div class="left">
<el-autocomplete v-model="filterSite"
:placeholder="$t('Site')"
clearable
:fetch-suggestions="fetchSiteSuggestions"
@select="onSiteSelect">
</el-autocomplete>
</div>
<div class="right">
<el-button type="primary" icon="fa fa-download" @click="openImportDialog">
{{$t('Import Spiders')}}
</el-button>
<el-button type="success"
icon="el-icon-plus"
class="btn add"
@click="onAdd">
{{$t('Add Spider')}}
</el-button>
<el-button type="success"
icon="el-icon-refresh"
class="btn refresh"
@click="onRefresh">
{{$t('Refresh')}}
</el-button>
</div>
</div>
<!--crawl confirm dialog-->
<crawl-confirm-dialog
:visible="crawlConfirmDialogVisible"
@@ -137,89 +102,128 @@
/>
<!--./crawl confirm dialog-->
<!--table list-->
<el-table :data="filteredTableData"
class="table"
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
border>
<template v-for="col in columns">
<el-table-column v-if="col.name === 'type'"
:key="col.name"
:label="$t(col.label)"
align="left"
:width="col.width">
<template slot-scope="scope">
<el-tag type="success" v-if="scope.row.type === 'configurable'">{{$t('Configurable')}}</el-tag>
<el-tag type="primary" v-else-if="scope.row.type === 'customized'">{{$t('Customized')}}</el-tag>
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'last_5_errors'"
:key="col.name"
:label="$t(col.label)"
:width="col.width"
align="center">
<template slot-scope="scope">
<div :style="{color:scope.row[col.name]>0?'red':''}">
{{scope.row[col.name]}}
</div>
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'cmd'"
:key="col.name"
:label="$t(col.label)"
:width="col.width"
align="left">
<template slot-scope="scope">
<el-input v-model="scope.row[col.name]"></el-input>
</template>
</el-table-column>
<el-table-column v-else-if="col.name.match(/_ts$/)"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{getTime(scope.row[col.name])}}
</template>
</el-table-column>
<el-table-column v-else
:key="col.name"
:property="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align || 'left'"
:width="col.width">
</el-table-column>
</template>
<el-table-column :label="$t('Action')" align="left" width="auto" fixed="right">
<template slot-scope="scope">
<el-tooltip :content="$t('View')" placement="top">
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
</el-tooltip>
<el-tooltip :content="$t('Remove')" placement="top">
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
</el-tooltip>
<el-tooltip v-if="!isShowRun(scope.row)" :content="$t('No command line')" placement="top">
<el-button disabled type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
</el-tooltip>
<el-tooltip v-else :content="$t('Run')" placement="top">
<el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
</el-tooltip>
<el-card style="border-radius: 0">
<!--filter-->
<div class="filter">
<!--<el-input prefix-icon="el-icon-search"-->
<!--:placeholder="$t('Search')"-->
<!--class="filter-search"-->
<!--v-model="filter.keyword"-->
<!--@change="onSearch">-->
<!--</el-input>-->
<div class="left">
<el-autocomplete v-model="filterSite"
:placeholder="$t('Site')"
clearable
:fetch-suggestions="fetchSiteSuggestions"
@select="onSiteSelect">
</el-autocomplete>
</div>
<div class="right">
<el-button v-if="false" type="primary" icon="fa fa-download" @click="openImportDialog">
{{$t('Import Spiders')}}
</el-button>
<el-button type="success"
icon="el-icon-plus"
class="btn add"
@click="onAdd">
{{$t('Add Spider')}}
</el-button>
<el-button type="success"
icon="el-icon-refresh"
class="btn refresh"
@click="onRefresh">
{{$t('Refresh')}}
</el-button>
</div>
</div>
<!--./filter-->
<!--table list-->
<el-table :data="filteredTableData"
class="table"
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
border>
<template v-for="col in columns">
<el-table-column v-if="col.name === 'type'"
:key="col.name"
:label="$t(col.label)"
align="left"
:width="col.width">
<template slot-scope="scope">
<el-tag type="success" v-if="scope.row.type === 'configurable'">{{$t('Configurable')}}</el-tag>
<el-tag type="primary" v-else-if="scope.row.type === 'customized'">{{$t('Customized')}}</el-tag>
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'last_5_errors'"
:key="col.name"
:label="$t(col.label)"
:width="col.width"
align="center">
<template slot-scope="scope">
<div :style="{color:scope.row[col.name]>0?'red':''}">
{{scope.row[col.name]}}
</div>
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'cmd'"
:key="col.name"
:label="$t(col.label)"
:width="col.width"
align="left">
<template slot-scope="scope">
<el-input v-model="scope.row[col.name]"></el-input>
</template>
</el-table-column>
<el-table-column v-else-if="col.name.match(/_ts$/)"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{getTime(scope.row[col.name])}}
</template>
</el-table-column>
<el-table-column v-else
:key="col.name"
:property="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align || 'left'"
:width="col.width">
</el-table-column>
</template>
</el-table-column>
</el-table>
<div class="pagination">
<el-pagination
@current-change="onPageChange"
@size-change="onPageChange"
:current-page.sync="pagination.pageNum"
:page-sizes="[10, 20, 50, 100]"
:page-size.sync="pagination.pageSize"
layout="sizes, prev, pager, next"
:total="spiderList.length">
</el-pagination>
</div>
<el-table-column :label="$t('Action')" align="left" width="auto" fixed="right">
<template slot-scope="scope">
<el-tooltip :content="$t('View')" placement="top">
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
</el-tooltip>
<el-tooltip :content="$t('Remove')" placement="top">
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
</el-tooltip>
<el-tooltip v-if="!isShowRun(scope.row)" :content="$t('No command line')" placement="top">
<el-button disabled type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
</el-tooltip>
<el-tooltip v-else :content="$t('Run')" placement="top">
<el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row)"></el-button>
</el-tooltip>
</template>
</el-table-column>
</el-table>
<div class="pagination">
<el-pagination
@current-change="onPageChange"
@size-change="onPageChange"
:current-page.sync="pagination.pageNum"
:page-sizes="[10, 20, 50, 100]"
:page-size.sync="pagination.pageSize"
layout="sizes, prev, pager, next"
:total="spiderList.length">
</el-pagination>
</div>
<!--./table list-->
</el-card>
</div>
</template>

View File

@@ -1,142 +1,146 @@
<template>
<div class="app-container">
<!--filter-->
<div class="filter">
<div class="left">
<el-select class="filter-select"
v-model="filter.node_id"
:placeholder="$t('Node')"
filterable
clearable
@change="onSelectNode">
<el-option v-for="op in nodeList" :key="op._id" :value="op._id" :label="op.name"></el-option>
</el-select>
<el-select class="filter-select"
v-model="filter.spider_id"
:placeholder="$t('Spider')"
filterable
clearable
@change="onSelectSpider">
<el-option v-for="op in spiderList" :key="op._id" :value="op._id" :label="op.name"></el-option>
</el-select>
<el-button type="success"
icon="el-icon-search"
class="refresh"
@click="onRefresh">
{{$t('Search')}}
</el-button>
<el-card style="border-radius: 0">
<!--filter-->
<div class="filter">
<div class="left">
<el-select class="filter-select"
v-model="filter.node_id"
:placeholder="$t('Node')"
filterable
clearable
@change="onSelectNode">
<el-option v-for="op in nodeList" :key="op._id" :value="op._id" :label="op.name"></el-option>
</el-select>
<el-select class="filter-select"
v-model="filter.spider_id"
:placeholder="$t('Spider')"
filterable
clearable
@change="onSelectSpider">
<el-option v-for="op in spiderList" :key="op._id" :value="op._id" :label="op.name"></el-option>
</el-select>
<el-button type="success"
icon="el-icon-search"
class="refresh"
@click="onRefresh">
{{$t('Search')}}
</el-button>
</div>
<!--<div class="right">-->
<!--</div>-->
</div>
<!--<div class="right">-->
<!--</div>-->
</div>
<!--./filter-->
<!--table list-->
<el-table :data="filteredTableData"
class="table"
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
border>
<template v-for="col in columns">
<el-table-column v-if="col.name === 'spider_name'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
<a href="javascript:" class="a-tag" @click="onClickSpider(scope.row)">{{scope.row[col.name]}}</a>
</template>
</el-table-column>
<el-table-column v-else-if="col.name.match(/_ts$/)"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{getTime(scope.row[col.name])}}
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'wait_duration'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{getWaitDuration(scope.row)}}
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'runtime_duration'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{getRuntimeDuration(scope.row)}}
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'total_duration'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{getTotalDuration(scope.row)}}
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'node_name'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
<a href="javascript:" class="a-tag" @click="onClickNode(scope.row)">{{scope.row[col.name]}}</a>
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'status'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
<status-tag :status="scope.row[col.name]"/>
</template>
</el-table-column>
<el-table-column v-else
:key="col.name"
:property="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
</el-table-column>
</template>
<el-table-column :label="$t('Action')" align="left" width="150" fixed="right">
<template slot-scope="scope">
<el-tooltip :content="$t('View')" placement="top">
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
</el-tooltip>
<el-tooltip :content="$t('Remove')" placement="top">
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
</el-tooltip>
<!--table list-->
<el-table :data="filteredTableData"
class="table"
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
border>
<template v-for="col in columns">
<el-table-column v-if="col.name === 'spider_name'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
<a href="javascript:" class="a-tag" @click="onClickSpider(scope.row)">{{scope.row[col.name]}}</a>
</template>
</el-table-column>
<el-table-column v-else-if="col.name.match(/_ts$/)"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{getTime(scope.row[col.name])}}
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'wait_duration'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{getWaitDuration(scope.row)}}
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'runtime_duration'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{getRuntimeDuration(scope.row)}}
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'total_duration'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
{{getTotalDuration(scope.row)}}
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'node_name'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
<a href="javascript:" class="a-tag" @click="onClickNode(scope.row)">{{scope.row[col.name]}}</a>
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'status'"
:key="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
<template slot-scope="scope">
<status-tag :status="scope.row[col.name]"/>
</template>
</el-table-column>
<el-table-column v-else
:key="col.name"
:property="col.name"
:label="$t(col.label)"
:sortable="col.sortable"
:align="col.align"
:width="col.width">
</el-table-column>
</template>
</el-table-column>
</el-table>
<div class="pagination">
<el-pagination
@current-change="onPageChange"
@size-change="onPageChange"
:current-page.sync="pageNum"
:page-sizes="[10, 20, 50, 100]"
:page-size.sync="pageSize"
layout="sizes, prev, pager, next"
:total="taskListTotalCount">
</el-pagination>
</div>
<el-table-column :label="$t('Action')" align="left" width="150" fixed="right">
<template slot-scope="scope">
<el-tooltip :content="$t('View')" placement="top">
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
</el-tooltip>
<el-tooltip :content="$t('Remove')" placement="top">
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
</el-tooltip>
</template>
</el-table-column>
</el-table>
<div class="pagination">
<el-pagination
@current-change="onPageChange"
@size-change="onPageChange"
:current-page.sync="pageNum"
:page-sizes="[10, 20, 50, 100]"
:page-size.sync="pageSize"
layout="sizes, prev, pager, next"
:total="taskListTotalCount">
</el-pagination>
</div>
<!--./table list-->
</el-card>
</div>
</template>
@@ -172,7 +176,7 @@ export default {
{ name: 'wait_duration', label: 'Wait Duration (sec)', width: '80', align: 'right' },
{ name: 'runtime_duration', label: 'Runtime Duration (sec)', width: '80', align: 'right' },
{ name: 'total_duration', label: 'Total Duration (sec)', width: '80', align: 'right' },
{ name: 'num_results', label: 'Results Count', width: '80' }
{ name: 'result_count', label: 'Results Count', width: '80' }
// { name: 'avg_num_results', label: 'Average Results Count per Second', width: '80' }
]
}