crawlab/backend/model/log.go

package model

import (
	"crawlab/database"
	"crawlab/utils"
	"github.com/apex/log"
	"github.com/globalsign/mgo"
	"github.com/globalsign/mgo/bson"
	"os"
	"runtime/debug"
	"time"
)

type LogItem struct {
	Id       bson.ObjectId `json:"_id" bson:"_id"`
	Message  string        `json:"msg" bson:"msg"`
	TaskId   string        `json:"task_id" bson:"task_id"`
	Seq      int64         `json:"seq" bson:"seq"`
	Ts       time.Time     `json:"ts" bson:"ts"`
	ExpireTs time.Time     `json:"expire_ts" bson:"expire_ts"`
}

type ErrorLogItem struct {
	Id       bson.ObjectId `json:"_id" bson:"_id"`
	TaskId   string        `json:"task_id" bson:"task_id"`
	Message  string        `json:"msg" bson:"msg"`
	LogId    bson.ObjectId `json:"log_id" bson:"log_id"`
	Seq      int64         `json:"seq" bson:"seq"`
	Ts       time.Time     `json:"ts" bson:"ts"`
	ExpireTs time.Time     `json:"expire_ts" bson:"expire_ts"`
}

// 获取本地日志
func GetLocalLog(logPath string) (fileBytes []byte, err error) {

	f, err := os.Open(logPath)
	if err != nil {
		log.Error(err.Error())
		debug.PrintStack()
		return nil, err
	}
	fi, err := f.Stat()
	if err != nil {
		log.Error(err.Error())
		debug.PrintStack()
		return nil, err
	}
	defer utils.Close(f)

	const bufLen = 2 * 1024 * 1024
	logBuf := make([]byte, bufLen)

	off := int64(0)
	if fi.Size() > int64(len(logBuf)) {
		off = fi.Size() - int64(len(logBuf))
	}
	n, err := f.ReadAt(logBuf, off)

	//到文件结尾会有EOF标识
	if err != nil && err.Error() != "EOF" {
		log.Error(err.Error())
		debug.PrintStack()
		return nil, err
	}
	logBuf = logBuf[:n]
	return logBuf, nil
}

func AddLogItem(l LogItem) error {
	s, c := database.GetCol("logs")
	defer s.Close()
	if err := c.Insert(l); err != nil {
		log.Errorf("insert log error: " + err.Error())
		debug.PrintStack()
		return err
	}
	return nil
}

func AddLogItems(ls []LogItem) error {
	if len(ls) == 0 {
		return nil
	}
	s, c := database.GetCol("logs")
	defer s.Close()
	var docs []interface{}
	for _, l := range ls {
		docs = append(docs, l)
	}
	if err := c.Insert(docs...); err != nil {
		log.Errorf("insert log error: " + err.Error())
		debug.PrintStack()
		return err
	}
	return nil
}

func AddErrorLogItem(e ErrorLogItem) error {
	s, c := database.GetCol("error_logs")
	defer s.Close()
	var l LogItem
	err := c.FindId(bson.M{"log_id": e.LogId}).One(&l)
	if err != nil && err == mgo.ErrNotFound {
		if err := c.Insert(e); err != nil {
			log.Errorf("insert log error: " + err.Error())
			debug.PrintStack()
			return err
		}
	}
	return nil
}

func GetLogItemList(query bson.M, keyword string, skip int, limit int, sortStr string) ([]LogItem, error) {
	s, c := database.GetCol("logs")
	defer s.Close()

	filter := query

	var logItems []LogItem
	if keyword == "" {
		filter["seq"] = bson.M{
			"$gte": skip,
			"$lt":  skip + limit,
		}
		if err := c.Find(filter).Sort(sortStr).All(&logItems); err != nil {
			debug.PrintStack()
			return logItems, err
		}
	} else {
		filter["msg"] = bson.M{
			"$regex": bson.RegEx{
				Pattern: keyword,
				Options: "i",
			},
		}
		if err := c.Find(filter).Sort(sortStr).Skip(skip).Limit(limit).All(&logItems); err != nil {
			debug.PrintStack()
			return logItems, err
		}
	}

	return logItems, nil
}

func GetLogItemTotal(query bson.M, keyword string) (int, error) {
	s, c := database.GetCol("logs")
	defer s.Close()

	filter := query

	if keyword != "" {
		filter["msg"] = bson.M{
			"$regex": bson.RegEx{
				Pattern: keyword,
				Options: "i",
			},
		}
	}

	total, err := c.Find(filter).Count()
	if err != nil {
		debug.PrintStack()
		return total, err
	}

	return total, nil
}