Merge branch 'develop' of https://github.com/crawlab-team/crawlab into v0.4.0

backend/conf/config.yml

@@ -15,6 +15,8 @@ redis:
 log:
   level: info
   path: "/var/logs/crawlab"
+  isDeletePeriodically: "Y"
+  deleteFrequency: "@hourly"
 server:
   host: 0.0.0.0
   port: 8000

backend/main.go

@@ -29,6 +29,15 @@ func main() {
 	}
 	log.Info("初始化日志设置成功")
 
+	if viper.GetString("log.isDeletePeriodically") == "Y" {
+		err := services.InitDeleteLogPeriodically()
+		if err != nil {
+			log.Error("Init DeletePeriodically Failed")
+			panic(err)
+		}
+		log.Info("初始化定期清理日志配置成功")
+	}
+
 	// 初始化Mongodb数据库
 	if err := database.InitMongo(); err != nil {
 		log.Error("init mongodb error:" + err.Error())

backend/services/log.go

@@ -3,11 +3,15 @@ package services
 import (
 	"crawlab/constants"
 	"crawlab/database"
+	"crawlab/lib/cron"
 	"crawlab/model"
 	"crawlab/utils"
 	"encoding/json"
 	"github.com/apex/log"
+	"github.com/spf13/viper"
 	"io/ioutil"
+	"os"
+	"path/filepath"
 	"runtime/debug"
 )
 
@@ -16,13 +20,29 @@ var TaskLogChanMap = utils.NewChanMap()
 
 // 获取本地日志
 func GetLocalLog(logPath string) (fileBytes []byte, err error) {
-	fileBytes, err = ioutil.ReadFile(logPath)
+
+	f, err := os.Open(logPath)
 	if err != nil {
-		log.Errorf(err.Error())
+		log.Error(err.Error())
 		debug.PrintStack()
-		return fileBytes, err
+		return nil, err
 	}
-	return fileBytes, nil
+	fi, err := f.Stat()
+	if err != nil {
+		log.Error(err.Error())
+		debug.PrintStack()
+		return nil, err
+	}
+	defer f.Close()
+	logBuf := make([]byte, 2048)
+	n, err := f.ReadAt(logBuf, fi.Size()-int64(len(logBuf)))
+	if err != nil {
+		log.Error(err.Error())
+		debug.PrintStack()
+		return nil, err
+	}
+	logBuf = logBuf[:n]
+	return logBuf, nil
 }
 
 // 获取远端日志
@@ -55,3 +75,36 @@ func GetRemoteLog(task model.Task) (logStr string, err error) {
 
 	return logStr, nil
 }
+
+func DeleteLogPeriodically() {
+	logDir := viper.GetString("log.path")
+	if !utils.Exists(logDir) {
+		log.Error("Can Not Set Delete Logs Periodically,No Log Dir")
+		return
+	}
+	rd, err := ioutil.ReadDir(logDir)
+	if err != nil {
+		log.Error("Read Log Dir Failed")
+		return
+	}
+
+	for _, fi := range rd {
+		if fi.IsDir() {
+			log.Info(filepath.Join(logDir, fi.Name()))
+			os.RemoveAll(filepath.Join(logDir, fi.Name()))
+			log.Info("Delete Log File Success")
+		}
+	}
+
+}
+
+func InitDeleteLogPeriodically() error {
+	c := cron.New(cron.WithSeconds())
+	if _, err := c.AddFunc(viper.GetString("log.deleteFrequency"), DeleteLogPeriodically); err != nil {
+		return err
+	}
+
+	c.Start()
+	return nil
+
+}

backend/services/log_test.go

@@ -0,0 +1,22 @@
+package services
+
+import (
+	"crawlab/config"
+	"github.com/apex/log"
+	. "github.com/smartystreets/goconvey/convey"
+	"github.com/spf13/viper"
+	"testing"
+)
+
+func TestDeleteLogPeriodically(t *testing.T) {
+	Convey("Test DeleteLogPeriodically", t, func() {
+		if err := config.InitConfig("../conf/config.yml"); err != nil {
+			log.Error("init config error:" + err.Error())
+			panic(err)
+		}
+		log.Info("初始化配置成功")
+		logDir := viper.GetString("log.path")
+		log.Info(logDir)
+		DeleteLogPeriodically()
+	})
+}