优化日志异常展示

backend/model/task.go

@@ -21,6 +21,7 @@ type Task struct {
 	Param           string        `json:"param" bson:"param"`
 	Error           string        `json:"error" bson:"error"`
 	ResultCount     int           `json:"result_count" bson:"result_count"`
+	ErrorLogCount   int           `json:"error_log_count" bson:"error_log_count"`
 	WaitDuration    float64       `json:"wait_duration" bson:"wait_duration"`
 	RuntimeDuration float64       `json:"runtime_duration" bson:"runtime_duration"`
 	TotalDuration   float64       `json:"total_duration" bson:"total_duration"`
@@ -127,7 +128,7 @@ func (t *Task) GetLogItems(keyword string, page int, pageSize int) (logItems []L
 	return logItems, logTotal, nil
 }
 
-func (t *Task) GetErrorLogItems() (errLogItems []ErrorLogItem, err error) {
+func (t *Task) GetErrorLogItems(n int) (errLogItems []ErrorLogItem, err error) {
 	s, c := database.GetCol("error_logs")
 	defer s.Close()
 
@@ -135,7 +136,7 @@ func (t *Task) GetErrorLogItems() (errLogItems []ErrorLogItem, err error) {
 		"task_id": t.Id,
 	}
 
-	if err := c.Find(query).All(&errLogItems); err != nil {
+	if err := c.Find(query).Limit(n).All(&errLogItems); err != nil {
 		log.Errorf("find error logs error: " + err.Error())
 		debug.PrintStack()
 		return errLogItems, err
@@ -407,6 +408,40 @@ func UpdateTaskResultCount(id string) (err error) {
 	return nil
 }
 
+// update error log count
+func UpdateErrorLogCount(id string) (err error) {
+	s, c := database.GetCol("error_logs")
+	defer s.Close()
+
+	query := bson.M{
+		"task_id": id,
+	}
+	count, err := c.Find(query).Count()
+	if err != nil {
+		log.Errorf("update error log count error: " + err.Error())
+		debug.PrintStack()
+		return err
+	}
+
+	st, ct := database.GetCol("tasks")
+	defer st.Close()
+
+	task, err := GetTask(id)
+	if err != nil {
+		log.Errorf(err.Error())
+		return err
+	}
+	task.ErrorLogCount = count
+
+	if err := ct.UpdateId(id, task); err != nil {
+		log.Errorf("update error log count error: " + err.Error())
+		debug.PrintStack()
+		return err
+	}
+
+	return nil
+}
+
 // convert all running tasks to abnormal tasks
 func UpdateTaskToAbnormal(nodeId bson.ObjectId) error {
 	s, c := database.GetCol("tasks")

backend/model/user.go

@@ -30,6 +30,7 @@ type UserSetting struct {
 	WechatRobotWebhook   string   `json:"wechat_robot_webhook" bson:"wechat_robot_webhook"`
 	EnabledNotifications []string `json:"enabled_notifications" bson:"enabled_notifications"`
 	ErrorRegexPattern    string   `json:"error_regex_pattern" bson:"error_regex_pattern"`
+	MaxErrorLog          int      `json:"max_error_log" bson:"max_error_log"`
 }
 
 func (user *User) Save() error {

backend/routes/task.go

@@ -260,7 +260,8 @@ func GetTaskLog(c *gin.Context) {
 
 func GetTaskErrorLog(c *gin.Context) {
 	id := c.Param("id")
-	errLogItems, err := services.GetTaskErrorLog(id)
+	u := services.GetCurrentUser(c)
+	errLogItems, err := services.GetTaskErrorLog(id, u.Setting.MaxErrorLog)
 	if err != nil {
 		HandleError(http.StatusInternalServerError, c, err)
 		return

backend/routes/user.go

@@ -237,10 +237,10 @@ func PostMe(c *gin.Context) {
 		user.Setting.WechatRobotWebhook = reqBody.Setting.WechatRobotWebhook
 	}
 	user.Setting.EnabledNotifications = reqBody.Setting.EnabledNotifications
-	if reqBody.Setting.ErrorRegexPattern != "" {
-		user.Setting.ErrorRegexPattern = reqBody.Setting.ErrorRegexPattern
-	}
 	user.Setting.ErrorRegexPattern = reqBody.Setting.ErrorRegexPattern
+	if reqBody.Setting.MaxErrorLog != 0 {
+		user.Setting.MaxErrorLog = reqBody.Setting.MaxErrorLog
+	}
 
 	if user.UserId.Hex() == "" {
 		user.UserId = bson.ObjectIdHex(constants.ObjectIdNull)

backend/services/task.go

@@ -431,6 +431,9 @@ func ScanErrorLogs(t model.Task) func() {
 		if err := model.UpdateTaskErrorLogs(t.Id, u.Setting.ErrorRegexPattern); err != nil {
 			return
 		}
+		if err := model.UpdateErrorLogCount(t.Id); err != nil {
+			return
+		}
 	}
 }
 
@@ -643,6 +646,10 @@ func FinishUpTask(s model.Spider, t model.Task) {
 	}()
 }
 
+func MonitorTask(s model.Spider, t model.Task) {
+
+}
+
 func SpiderFileCheck(t model.Task, spider model.Spider) error {
 	// 判断爬虫文件是否存在
 	gfFile := model.GetGridFs(spider.FileId)
@@ -681,12 +688,16 @@ func GetTaskLog(id string, keyword string, page int, pageSize int) (logItems []m
 	return logItems, logTotal, nil
 }
 
-func GetTaskErrorLog(id string) (errLogItems []model.ErrorLogItem, err error) {
+func GetTaskErrorLog(id string, n int) (errLogItems []model.ErrorLogItem, err error) {
+	if n == 0 {
+		n = 1000
+	}
+
 	task, err := model.GetTask(id)
 	if err != nil {
 		return
 	}
-	errLogItems, err = task.GetErrorLogItems()
+	errLogItems, err = task.GetErrorLogItems(n)
 	if err != nil {
 		return
 	}