feat: added modules

core/task/log/constants.go

@@ -0,0 +1,12 @@
+package log
+
+const (
+	MetadataName = "metadata.json"
+)
+
+const (
+	DriverTypeFile  = "file"  // raw file
+	DriverTypeFs    = "fs"    // file system (SeaweedFS)
+	DriverTypeMongo = "mongo" // mongodb
+	DriverTypeEs    = "es"    // elastic search
+)

core/task/log/default.go

@@ -0,0 +1,5 @@
+package log
+
+import "time"
+
+var DefaultLogTtl = 30 * 24 * time.Hour

core/task/log/driver.go

@@ -0,0 +1,18 @@
+package log
+
+func GetLogDriver(logDriverType string) (driver Driver, err error) {
+	switch logDriverType {
+	case DriverTypeFile:
+		driver, err = GetFileLogDriver()
+		if err != nil {
+			return driver, err
+		}
+	case DriverTypeMongo:
+		return driver, ErrNotImplemented
+	case DriverTypeEs:
+		return driver, ErrNotImplemented
+	default:
+		return driver, ErrInvalidType
+	}
+	return driver, nil
+}

core/task/log/entity.go

@@ -0,0 +1,16 @@
+package log
+
+import "time"
+
+type Message struct {
+	Id  int64     `json:"id" bson:"id"`
+	Msg string    `json:"msg" bson:"msg"`
+	Ts  time.Time `json:"ts" bson:"ts"`
+}
+
+type Metadata struct {
+	Size       int64  `json:"size,omitempty" bson:"size"`
+	TotalLines int64  `json:"total_lines,omitempty" bson:"total_lines"`
+	TotalBytes int64  `json:"total_bytes,omitempty" bson:"total_bytes"`
+	Md5        string `json:"md5,omitempty" bson:"md5"`
+}

core/task/log/errors.go

@@ -0,0 +1,8 @@
+package log
+
+import "errors"
+
+var (
+	ErrInvalidType    = errors.New("invalid type")
+	ErrNotImplemented = errors.New("not implemented")
+)

core/task/log/file_driver.go

@@ -0,0 +1,284 @@
+package log
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"github.com/spf13/viper"
+	"io"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+type FileLogDriver struct {
+	// settings
+	logFileName string
+	rootPath    string
+
+	// internals
+	mu sync.Mutex
+}
+
+func (d *FileLogDriver) Init() (err error) {
+	go d.cleanup()
+
+	return nil
+}
+
+func (d *FileLogDriver) Close() (err error) {
+	return nil
+}
+
+func (d *FileLogDriver) WriteLine(id string, line string) (err error) {
+	d.initDir(id)
+
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	filePath := d.getLogFilePath(id, d.logFileName)
+
+	f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(0760))
+	if err != nil {
+		return trace.TraceError(err)
+	}
+	defer func(f *os.File) {
+		err := f.Close()
+		if err != nil {
+			log.Errorf("close file error: %s", err.Error())
+		}
+	}(f)
+
+	_, err = f.WriteString(line + "\n")
+	if err != nil {
+		return trace.TraceError(err)
+	}
+
+	return nil
+}
+
+func (d *FileLogDriver) WriteLines(id string, lines []string) (err error) {
+	linesString := strings.Join(lines, "\n")
+	if err := d.WriteLine(id, linesString); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *FileLogDriver) Find(id string, pattern string, skip int, limit int) (lines []string, err error) {
+	if pattern != "" {
+		return lines, errors.New("not implemented")
+	}
+	if !utils.Exists(d.getLogFilePath(id, d.logFileName)) {
+		return nil, nil
+	}
+
+	f, err := os.Open(d.getLogFilePath(id, d.logFileName))
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+	defer f.Close()
+
+	sc := bufio.NewReaderSize(f, 1024*1024*10)
+
+	i := -1
+	for {
+		line, err := sc.ReadString(byte('\n'))
+		if err != nil {
+			break
+		}
+		line = strings.TrimSuffix(line, "\n")
+
+		i++
+
+		if i < skip {
+			continue
+		}
+
+		if i >= skip+limit {
+			break
+		}
+
+		lines = append(lines, line)
+	}
+
+	return lines, nil
+}
+
+func (d *FileLogDriver) Count(id string, pattern string) (n int, err error) {
+	if pattern != "" {
+		return n, errors.New("not implemented")
+	}
+	if !utils.Exists(d.getLogFilePath(id, d.logFileName)) {
+		return 0, nil
+	}
+
+	f, err := os.Open(d.getLogFilePath(id, d.logFileName))
+	if err != nil {
+		return n, trace.TraceError(err)
+	}
+	return d.lineCounter(f)
+}
+
+func (d *FileLogDriver) Flush() (err error) {
+	return nil
+}
+
+func (d *FileLogDriver) getLogPath() (logPath string) {
+	return viper.GetString("log.path")
+}
+
+func (d *FileLogDriver) getBasePath(id string) (filePath string) {
+	return filepath.Join(d.getLogPath(), id)
+}
+
+func (d *FileLogDriver) getMetadataPath(id string) (filePath string) {
+	return filepath.Join(d.getBasePath(id), MetadataName)
+}
+
+func (d *FileLogDriver) getLogFilePath(id, fileName string) (filePath string) {
+	return filepath.Join(d.getBasePath(id), fileName)
+}
+
+func (d *FileLogDriver) getLogFiles(id string) (files []os.FileInfo) {
+	// 增加了对返回异常的捕获
+	files, err := utils.ListDir(d.getBasePath(id))
+	if err != nil {
+		trace.PrintError(err)
+		return nil
+	}
+	return
+}
+
+func (d *FileLogDriver) initDir(id string) {
+	if !utils.Exists(d.getBasePath(id)) {
+		if err := os.MkdirAll(d.getBasePath(id), os.FileMode(0770)); err != nil {
+			trace.PrintError(err)
+		}
+	}
+}
+
+func (d *FileLogDriver) lineCounter(r io.Reader) (n int, err error) {
+	buf := make([]byte, 32*1024)
+	count := 0
+	lineSep := []byte{'\n'}
+
+	for {
+		c, err := r.Read(buf)
+		count += bytes.Count(buf[:c], lineSep)
+
+		switch {
+		case err == io.EOF:
+			return count, nil
+
+		case err != nil:
+			return count, err
+		}
+	}
+}
+
+func (d *FileLogDriver) getTtl() time.Duration {
+	ttl := viper.GetString("log.ttl")
+	if ttl == "" {
+		return DefaultLogTtl
+	}
+
+	if strings.HasSuffix(ttl, "s") {
+		ttl = strings.TrimSuffix(ttl, "s")
+		n, err := strconv.Atoi(ttl)
+		if err != nil {
+			return DefaultLogTtl
+		}
+		return time.Duration(n) * time.Second
+	} else if strings.HasSuffix(ttl, "m") {
+		ttl = strings.TrimSuffix(ttl, "m")
+		n, err := strconv.Atoi(ttl)
+		if err != nil {
+			return DefaultLogTtl
+		}
+		return time.Duration(n) * time.Minute
+	} else if strings.HasSuffix(ttl, "h") {
+		ttl = strings.TrimSuffix(ttl, "h")
+		n, err := strconv.Atoi(ttl)
+		if err != nil {
+			return DefaultLogTtl
+		}
+		return time.Duration(n) * time.Hour
+
+	} else if strings.HasSuffix(ttl, "d") {
+		ttl = strings.TrimSuffix(ttl, "d")
+		n, err := strconv.Atoi(ttl)
+		if err != nil {
+			return DefaultLogTtl
+		}
+		return time.Duration(n) * 24 * time.Hour
+	} else {
+		return DefaultLogTtl
+	}
+}
+
+func (d *FileLogDriver) cleanup() {
+	if d.getLogPath() == "" {
+		return
+	}
+	if !utils.Exists(d.getLogPath()) {
+		if err := os.MkdirAll(d.getLogPath(), os.FileMode(0770)); err != nil {
+			log.Errorf("failed to create log directory: %s", d.getLogPath())
+			trace.PrintError(err)
+			return
+		}
+	}
+	for {
+		// 增加对目录不存在的判断
+		dirs, err := utils.ListDir(d.getLogPath())
+		if err != nil {
+			trace.PrintError(err)
+			time.Sleep(10 * time.Minute)
+			continue
+		}
+		for _, dir := range dirs {
+			if time.Now().After(dir.ModTime().Add(d.getTtl())) {
+				if err := os.RemoveAll(d.getBasePath(dir.Name())); err != nil {
+					trace.PrintError(err)
+					continue
+				}
+				log.Infof("removed outdated log directory: %s", d.getBasePath(dir.Name()))
+			}
+		}
+
+		time.Sleep(10 * time.Minute)
+	}
+}
+
+var logDriver Driver
+
+func newFileLogDriver() (driver Driver, err error) {
+	// driver
+	driver = &FileLogDriver{
+		logFileName: "log.txt",
+		mu:          sync.Mutex{},
+	}
+
+	// init
+	if err := driver.Init(); err != nil {
+		return nil, err
+	}
+
+	return driver, nil
+}
+
+func GetFileLogDriver() (driver Driver, err error) {
+	if logDriver != nil {
+		return logDriver, nil
+	}
+	logDriver, err = newFileLogDriver()
+	if err != nil {
+		return nil, err
+	}
+	return logDriver, nil
+}

core/task/log/file_driver_test.go

@@ -0,0 +1,132 @@
+package log
+
+import (
+	"fmt"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"os"
+	"strings"
+	"testing"
+)
+
+func setupFileDriverTest() {
+	cleanupFileDriverTest()
+	_ = os.MkdirAll("./tmp", os.ModePerm)
+}
+
+func cleanupFileDriverTest() {
+	_ = os.RemoveAll("./tmp")
+}
+
+func TestFileDriver_WriteLine(t *testing.T) {
+	setupFileDriverTest()
+	t.Cleanup(cleanupFileDriverTest)
+
+	d, err := newFileLogDriver(nil)
+	require.Nil(t, err)
+	defer d.Close()
+
+	id := primitive.NewObjectID()
+
+	err = d.WriteLine(id.Hex(), "it works")
+	require.Nil(t, err)
+
+	logFilePath := fmt.Sprintf("/var/log/crawlab/%s/log.txt", id.Hex())
+	require.FileExists(t, logFilePath)
+	text, err := os.ReadFile(logFilePath)
+	require.Nil(t, err)
+	require.Equal(t, "it works\n", string(text))
+}
+
+func TestFileDriver_WriteLines(t *testing.T) {
+	setupFileDriverTest()
+	t.Cleanup(cleanupFileDriverTest)
+
+	d, err := newFileLogDriver(nil)
+	require.Nil(t, err)
+	defer d.Close()
+
+	id := primitive.NewObjectID()
+
+	for i := 0; i < 100; i++ {
+		err = d.WriteLine(id.Hex(), "it works")
+		require.Nil(t, err)
+	}
+
+	logFilePath := fmt.Sprintf("/var/log/crawlab/%s/log.txt", id.Hex())
+	require.FileExists(t, logFilePath)
+	text, err := os.ReadFile(logFilePath)
+	require.Nil(t, err)
+	require.Contains(t, string(text), "it works\n")
+	lines := strings.Split(string(text), "\n")
+	require.Equal(t, 101, len(lines))
+}
+
+func TestFileDriver_Find(t *testing.T) {
+	setupFileDriverTest()
+	t.Cleanup(cleanupFileDriverTest)
+
+	d, err := newFileLogDriver(nil)
+	require.Nil(t, err)
+	defer d.Close()
+
+	id := primitive.NewObjectID()
+
+	batch := 1000
+	var lines []string
+	for i := 0; i < 10; i++ {
+		for j := 0; j < batch; j++ {
+			line := fmt.Sprintf("line: %d", i*batch+j+1)
+			lines = append(lines, line)
+		}
+		err = d.WriteLines(id.Hex(), lines)
+		require.Nil(t, err)
+		lines = []string{}
+	}
+
+	driver := d
+
+	lines, err = driver.Find(id.Hex(), "", 0, 10)
+	require.Nil(t, err)
+	require.Equal(t, 10, len(lines))
+	require.Equal(t, "line: 1", lines[0])
+	require.Equal(t, "line: 10", lines[len(lines)-1])
+
+	lines, err = driver.Find(id.Hex(), "", 0, 1)
+	require.Nil(t, err)
+	require.Equal(t, 1, len(lines))
+	require.Equal(t, "line: 1", lines[0])
+	require.Equal(t, "line: 1", lines[len(lines)-1])
+
+	lines, err = driver.Find(id.Hex(), "", 0, 1000)
+	require.Nil(t, err)
+	require.Equal(t, 1000, len(lines))
+	require.Equal(t, "line: 1", lines[0])
+	require.Equal(t, "line: 1000", lines[len(lines)-1])
+
+	lines, err = driver.Find(id.Hex(), "", 1000, 1000)
+	require.Nil(t, err)
+	require.Equal(t, 1000, len(lines))
+	require.Equal(t, "line: 1001", lines[0])
+	require.Equal(t, "line: 2000", lines[len(lines)-1])
+
+	lines, err = driver.Find(id.Hex(), "", 1001, 1000)
+	require.Nil(t, err)
+	require.Equal(t, 1000, len(lines))
+	require.Equal(t, "line: 1002", lines[0])
+	require.Equal(t, "line: 2001", lines[len(lines)-1])
+
+	lines, err = driver.Find(id.Hex(), "", 1001, 999)
+	require.Nil(t, err)
+	require.Equal(t, 999, len(lines))
+	require.Equal(t, "line: 1002", lines[0])
+	require.Equal(t, "line: 2000", lines[len(lines)-1])
+
+	lines, err = driver.Find(id.Hex(), "", 999, 2001)
+	require.Nil(t, err)
+	require.Equal(t, 2001, len(lines))
+	require.Equal(t, "line: 1000", lines[0])
+	require.Equal(t, "line: 3000", lines[len(lines)-1])
+
+	cleanupFileDriverTest()
+}

core/task/log/interface.go

@@ -0,0 +1,10 @@
+package log
+
+type Driver interface {
+	Init() (err error)
+	Close() (err error)
+	WriteLine(id string, line string) (err error)
+	WriteLines(id string, lines []string) (err error)
+	Find(id string, pattern string, skip int, limit int) (lines []string, err error)
+	Count(id string, pattern string) (n int, err error)
+}