diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 01815325..e83d7ed8 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -1,10 +1,14 @@ package routes import ( + "crawlab/constants" + "crawlab/database" "crawlab/model" "crawlab/services" "crawlab/utils" + "github.com/apex/log" "github.com/gin-gonic/gin" + "github.com/globalsign/mgo" "github.com/globalsign/mgo/bson" "github.com/pkg/errors" uuid "github.com/satori/go.uuid" @@ -14,7 +18,9 @@ import ( "os" "path/filepath" "runtime/debug" + "strconv" "strings" + "time" ) func GetSpiderList(c *gin.Context) { @@ -342,3 +348,123 @@ func PostSpiderFile(c *gin.Context) { Message: "success", }) } + +func GetSpiderStats(c *gin.Context) { + type Overview struct { + TaskCount int `json:"task_count" bson:"task_count"` + ResultCount int `json:"result_count" bson:"result_count"` + SuccessCount int `json:"success_count" bson:"success_count"` + SuccessRate float64 `json:"success_rate"` + TotalWaitDuration float64 `json:"wait_duration" bson:"wait_duration"` + TotalRuntimeDuration float64 `json:"runtime_duration" bson:"runtime_duration"` + AvgWaitDuration float64 `json:"avg_wait_duration"` + AvgRuntimeDuration float64 `json:"avg_runtime_duration"` + } + + type Data struct { + Overview Overview `json:"overview"` + Daily []model.TaskDailyItem `json:"daily"` + } + + id := c.Param("id") + + spider, err := model.GetSpider(bson.ObjectIdHex(id)) + if err != nil { + log.Errorf(err.Error()) + HandleError(http.StatusInternalServerError, c, err) + return + } + + s, col := database.GetCol("tasks") + defer s.Close() + + // 起始日期 + startDate := time.Now().Add(-time.Hour * 24 * 30) + endDate := time.Now() + + // match + op1 := bson.M{ + "$match": bson.M{ + "spider_id": spider.Id, + "create_ts": bson.M{ + "$gte": startDate, + "$lt": endDate, + }, + }, + } + + // project + op2 := bson.M{ + "$project": bson.M{ + "success_count": bson.M{ + "$cond": []interface{}{ + bson.M{ + "$eq": []string{ + "$status", + constants.StatusFinished, + }, + }, + 1, + 0, + }, + }, + "result_count": "$result_count", + "wait_duration": "$wait_duration", + "runtime_duration": "$runtime_duration", + }, + } + + // group + op3 := bson.M{ + "$group": bson.M{ + "_id": nil, + "task_count": bson.M{"$sum": 1}, + "success_count": bson.M{"$sum": "$success_count"}, + "result_count": bson.M{"$sum": "$result_count"}, + "wait_duration": bson.M{"$sum": "$wait_duration"}, + "runtime_duration": bson.M{"$sum": "$runtime_duration"}, + }, + } + + // run aggregation pipeline + var overview Overview + if err := col.Pipe([]bson.M{op1, op2, op3}).One(&overview); err != nil { + if err == mgo.ErrNotFound { + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: Data{ + Overview: overview, + Daily: []model.TaskDailyItem{}, + }, + }) + return + } + log.Errorf(err.Error()) + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 后续处理 + successCount, _ := strconv.ParseFloat(strconv.Itoa(overview.SuccessCount), 64) + taskCount, _ := strconv.ParseFloat(strconv.Itoa(overview.TaskCount), 64) + overview.SuccessRate = successCount / taskCount + overview.AvgWaitDuration = overview.TotalWaitDuration / taskCount + overview.AvgRuntimeDuration = overview.TotalRuntimeDuration / taskCount + + items, err := model.GetDailyTaskStats(bson.M{"spider_id": spider.Id}) + if err != nil { + log.Errorf(err.Error()) + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: Data{ + Overview: overview, + Daily: items, + }, + }) +} diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index be6168d4..7b3e589c 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -235,5 +235,20 @@ export default { 'Spider info has been saved successfully': '爬虫信息已成功保存', 'Do you allow us to collect some statistics to improve Crawlab?': '您允许我们收集统计数据以更好地优化Crawlab?', 'Saved file successfully': '成功保存文件', - 'An error happened when fetching the data': '请求数据时出错' + 'An error happened when fetching the data': '请求数据时出错', + + // 登录 + 'Sign in': '登录', + 'Sign-in': '登录', + 'Sign out': '退出登录', + 'Sign-out': '退出登录', + 'Sign up': '注册', + 'Sign-up': '注册', + 'Forgot Password': '忘记密码', + 'Has Account': '已有账号', + 'New to Crawlab': 'Crawlab新用户', + 'Initial Username/Password': '初始用户名/密码', + 'Username': '用户名', + 'Password': '密码', + 'Confirm Password': '确认密码' } diff --git a/frontend/src/views/login/index.vue b/frontend/src/views/login/index.vue index 70e28313..3a0b79d9 100644 --- a/frontend/src/views/login/index.vue +++ b/frontend/src/views/login/index.vue @@ -1,38 +1,65 @@