From 5a8da7ed29af5ab41a27fb73328fc78baef364f2 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Sun, 19 Apr 2020 11:07:39 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=97=A5=E5=BF=97=E5=BC=82?= =?UTF-8?q?=E5=B8=B8=E5=B1=95=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/task.go | 39 +++++++++++++++++++++++++++++++++++++-- backend/model/user.go | 1 + backend/routes/task.go | 3 ++- backend/routes/user.go | 6 +++--- backend/services/task.go | 15 +++++++++++++-- 5 files changed, 56 insertions(+), 8 deletions(-) diff --git a/backend/model/task.go b/backend/model/task.go index 6fefb42b..415a0234 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -21,6 +21,7 @@ type Task struct { Param string `json:"param" bson:"param"` Error string `json:"error" bson:"error"` ResultCount int `json:"result_count" bson:"result_count"` + ErrorLogCount int `json:"error_log_count" bson:"error_log_count"` WaitDuration float64 `json:"wait_duration" bson:"wait_duration"` RuntimeDuration float64 `json:"runtime_duration" bson:"runtime_duration"` TotalDuration float64 `json:"total_duration" bson:"total_duration"` @@ -127,7 +128,7 @@ func (t *Task) GetLogItems(keyword string, page int, pageSize int) (logItems []L return logItems, logTotal, nil } -func (t *Task) GetErrorLogItems() (errLogItems []ErrorLogItem, err error) { +func (t *Task) GetErrorLogItems(n int) (errLogItems []ErrorLogItem, err error) { s, c := database.GetCol("error_logs") defer s.Close() @@ -135,7 +136,7 @@ func (t *Task) GetErrorLogItems() (errLogItems []ErrorLogItem, err error) { "task_id": t.Id, } - if err := c.Find(query).All(&errLogItems); err != nil { + if err := c.Find(query).Limit(n).All(&errLogItems); err != nil { log.Errorf("find error logs error: " + err.Error()) debug.PrintStack() return errLogItems, err @@ -407,6 +408,40 @@ func UpdateTaskResultCount(id string) (err error) { return nil } +// update error log count +func UpdateErrorLogCount(id string) (err error) { + s, c := database.GetCol("error_logs") + defer s.Close() + + query := bson.M{ + "task_id": id, + } + count, err := c.Find(query).Count() + if err != nil { + log.Errorf("update error log count error: " + err.Error()) + debug.PrintStack() + return err + } + + st, ct := database.GetCol("tasks") + defer st.Close() + + task, err := GetTask(id) + if err != nil { + log.Errorf(err.Error()) + return err + } + task.ErrorLogCount = count + + if err := ct.UpdateId(id, task); err != nil { + log.Errorf("update error log count error: " + err.Error()) + debug.PrintStack() + return err + } + + return nil +} + // convert all running tasks to abnormal tasks func UpdateTaskToAbnormal(nodeId bson.ObjectId) error { s, c := database.GetCol("tasks") diff --git a/backend/model/user.go b/backend/model/user.go index 7e97af49..e730db38 100644 --- a/backend/model/user.go +++ b/backend/model/user.go @@ -30,6 +30,7 @@ type UserSetting struct { WechatRobotWebhook string `json:"wechat_robot_webhook" bson:"wechat_robot_webhook"` EnabledNotifications []string `json:"enabled_notifications" bson:"enabled_notifications"` ErrorRegexPattern string `json:"error_regex_pattern" bson:"error_regex_pattern"` + MaxErrorLog int `json:"max_error_log" bson:"max_error_log"` } func (user *User) Save() error { diff --git a/backend/routes/task.go b/backend/routes/task.go index 2cbd6426..2484b300 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -260,7 +260,8 @@ func GetTaskLog(c *gin.Context) { func GetTaskErrorLog(c *gin.Context) { id := c.Param("id") - errLogItems, err := services.GetTaskErrorLog(id) + u := services.GetCurrentUser(c) + errLogItems, err := services.GetTaskErrorLog(id, u.Setting.MaxErrorLog) if err != nil { HandleError(http.StatusInternalServerError, c, err) return diff --git a/backend/routes/user.go b/backend/routes/user.go index 47951428..d4ce3d08 100644 --- a/backend/routes/user.go +++ b/backend/routes/user.go @@ -237,10 +237,10 @@ func PostMe(c *gin.Context) { user.Setting.WechatRobotWebhook = reqBody.Setting.WechatRobotWebhook } user.Setting.EnabledNotifications = reqBody.Setting.EnabledNotifications - if reqBody.Setting.ErrorRegexPattern != "" { - user.Setting.ErrorRegexPattern = reqBody.Setting.ErrorRegexPattern - } user.Setting.ErrorRegexPattern = reqBody.Setting.ErrorRegexPattern + if reqBody.Setting.MaxErrorLog != 0 { + user.Setting.MaxErrorLog = reqBody.Setting.MaxErrorLog + } if user.UserId.Hex() == "" { user.UserId = bson.ObjectIdHex(constants.ObjectIdNull) diff --git a/backend/services/task.go b/backend/services/task.go index 396d74fd..47f52037 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -431,6 +431,9 @@ func ScanErrorLogs(t model.Task) func() { if err := model.UpdateTaskErrorLogs(t.Id, u.Setting.ErrorRegexPattern); err != nil { return } + if err := model.UpdateErrorLogCount(t.Id); err != nil { + return + } } } @@ -643,6 +646,10 @@ func FinishUpTask(s model.Spider, t model.Task) { }() } +func MonitorTask(s model.Spider, t model.Task) { + +} + func SpiderFileCheck(t model.Task, spider model.Spider) error { // 判断爬虫文件是否存在 gfFile := model.GetGridFs(spider.FileId) @@ -681,12 +688,16 @@ func GetTaskLog(id string, keyword string, page int, pageSize int) (logItems []m return logItems, logTotal, nil } -func GetTaskErrorLog(id string) (errLogItems []model.ErrorLogItem, err error) { +func GetTaskErrorLog(id string, n int) (errLogItems []model.ErrorLogItem, err error) { + if n == 0 { + n = 1000 + } + task, err := model.GetTask(id) if err != nil { return } - errLogItems, err = task.GetErrorLogItems() + errLogItems, err = task.GetErrorLogItems(n) if err != nil { return }