test: updated test cases

core/config/config_test.go

@@ -60,6 +60,9 @@ server:
 	err = os.WriteFile(configPath, configContent, 0644)
 	require.NoError(t, err, "Failed to write config file")
 
+	// Remove the environment variable before testing with config file
+	os.Unsetenv("CRAWLAB_MONGO_HOST")
+
 	// Create a new Config instance with the config file
 	cWithFile := Config{Name: configPath}
 	err = cWithFile.Init()

core/event/options.go

@@ -1 +0,0 @@
-package event

core/grpc/payload/model_service_v2_payload.go

@@ -1,17 +0,0 @@
-package payload
-
-import (
-	"github.com/crawlab-team/crawlab/db/mongo"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-type ModelServiceV2Payload struct {
-	Type        string             `json:"type,omitempty"`
-	Id          primitive.ObjectID `json:"_id,omitempty"`
-	Query       bson.M             `json:"query,omitempty"`
-	FindOptions *mongo.FindOptions `json:"find_options,omitempty"`
-	Model       any                `json:"model,omitempty"`
-	Update      bson.M             `json:"update,omitempty"`
-	Models      []any              `json:"models,omitempty"`
-}

core/grpc/server/model_base_service_v2_server.go

@@ -3,13 +3,14 @@ package server
 import (
 	"context"
 	"encoding/json"
+	"reflect"
+
 	models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
 	"github.com/crawlab-team/crawlab/core/models/service"
 	"github.com/crawlab-team/crawlab/db/mongo"
 	"github.com/crawlab-team/crawlab/grpc"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
-	"reflect"
 )
 
 var (
@@ -253,7 +254,11 @@ func (svr ModelBaseServiceServerV2) InsertMany(_ context.Context, req *grpc.Mode
 	modelsSlice := reflect.ValueOf(modelsSlicePtr).Elem()
 	modelsInterface := make([]any, modelsSlice.Len())
 	for i := 0; i < modelsSlice.Len(); i++ {
-		modelsInterface[i] = modelsSlice.Index(i).Interface()
+		modelValue := modelsSlice.Index(i)
+		if modelValue.FieldByName("Id").Interface().(primitive.ObjectID).IsZero() {
+			modelValue.FieldByName("Id").Set(reflect.ValueOf(primitive.NewObjectID()))
+		}
+		modelsInterface[i] = modelValue.Interface()
 	}
 	modelSvc := GetModelService[bson.M](req.ModelType)
 	r, err := modelSvc.GetCol().GetCollection().InsertMany(modelSvc.GetCol().GetContext(), modelsInterface)

core/grpc/server/task_server_v2.go

@@ -39,7 +39,6 @@ type TaskServerV2 struct {
 func (svr TaskServerV2) Subscribe(stream grpc.TaskService_SubscribeServer) (err error) {
 	for {
 		msg, err := stream.Recv()
-		utils.LogDebug(msg.String())
 		if err == io.EOF {
 			return nil
 		}

core/grpc/server/utils_handle.go

@@ -38,16 +38,3 @@ func HandleSuccessWithData(data interface{}) (res *grpc.Response, err error) {
 		Data:    bytes,
 	}, nil
 }
-
-func HandleSuccessWithListData(data interface{}, total int) (res *grpc.Response, err error) {
-	bytes, err := json.Marshal(data)
-	if err != nil {
-		return HandleError(err)
-	}
-	return &grpc.Response{
-		Code:    grpc.ResponseCode_OK,
-		Message: "success",
-		Data:    bytes,
-		Total:   int64(total),
-	}, nil
-}

core/interfaces/color.go

@@ -1,7 +0,0 @@
-package interfaces
-
-type Color interface {
-	Entity
-	GetHex() string
-	GetName() string
-}

core/interfaces/color_service.go

@@ -1,7 +0,0 @@
-package interfaces
-
-type ColorService interface {
-	Injectable
-	GetByName(name string) (res Color, err error)
-	GetRandom() (res Color, err error)
-}

core/models/client/model_service_v2.go

@@ -6,7 +6,7 @@ import (
 	"github.com/crawlab-team/crawlab/core/interfaces"
 	nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
 	"github.com/crawlab-team/crawlab/db/mongo"
-	grpc "github.com/crawlab-team/crawlab/grpc"
+	"github.com/crawlab-team/crawlab/grpc"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"reflect"

core/models/client/model_service_v2_test.go

@@ -2,10 +2,14 @@ package client_test
 
 import (
 	"context"
+	"testing"
+	"time"
+
+	"github.com/crawlab-team/crawlab/core/models/models/v2"
+
 	"github.com/apex/log"
 	"github.com/crawlab-team/crawlab/core/grpc/server"
 	"github.com/crawlab-team/crawlab/core/models/client"
-	"github.com/crawlab-team/crawlab/core/models/models/v2"
 	"github.com/crawlab-team/crawlab/core/models/service"
 	"github.com/crawlab-team/crawlab/db/mongo"
 	"github.com/spf13/viper"
@@ -14,12 +18,8 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
-	"testing"
-	"time"
 )
 
-type TestModel models.TestModelV2
-
 func setupTestDB() {
 	viper.Set("mongo.db", "testdb")
 }
@@ -51,10 +51,10 @@ func TestModelServiceV2_GetById(t *testing.T) {
 	go startSvr(svr)
 	defer stopSvr(svr)
 
-	m := TestModel{
+	m := models.TestModelV2{
 		Name: "Test Name",
 	}
-	modelSvc := service.NewModelServiceV2[TestModel]()
+	modelSvc := service.NewModelServiceV2[models.TestModelV2]()
 	id, err := modelSvc.InsertOne(m)
 	require.Nil(t, err)
 	m.SetId(id)
@@ -64,7 +64,7 @@ func TestModelServiceV2_GetById(t *testing.T) {
 	require.Nil(t, err)
 	c.Connect()
 
-	clientSvc := client.NewModelServiceV2[TestModel]()
+	clientSvc := client.NewModelServiceV2[models.TestModelV2]()
 	res, err := clientSvc.GetById(m.Id)
 	require.Nil(t, err)
 	assert.Equal(t, res.Id, m.Id)
@@ -79,10 +79,10 @@ func TestModelServiceV2_GetOne(t *testing.T) {
 	go startSvr(svr)
 	defer stopSvr(svr)
 
-	m := TestModel{
+	m := models.TestModelV2{
 		Name: "Test Name",
 	}
-	modelSvc := service.NewModelServiceV2[TestModel]()
+	modelSvc := service.NewModelServiceV2[models.TestModelV2]()
 	id, err := modelSvc.InsertOne(m)
 	require.Nil(t, err)
 	m.SetId(id)
@@ -92,7 +92,7 @@ func TestModelServiceV2_GetOne(t *testing.T) {
 	require.Nil(t, err)
 	c.Connect()
 
-	clientSvc := client.NewModelServiceV2[TestModel]()
+	clientSvc := client.NewModelServiceV2[models.TestModelV2]()
 	res, err := clientSvc.GetOne(bson.M{"name": m.Name}, nil)
 	require.Nil(t, err)
 	assert.Equal(t, res.Id, m.Id)
@@ -107,10 +107,10 @@ func TestModelServiceV2_GetMany(t *testing.T) {
 	go startSvr(svr)
 	defer stopSvr(svr)
 
-	m := TestModel{
+	m := models.TestModelV2{
 		Name: "Test Name",
 	}
-	modelSvc := service.NewModelServiceV2[TestModel]()
+	modelSvc := service.NewModelServiceV2[models.TestModelV2]()
 	id, err := modelSvc.InsertOne(m)
 	require.Nil(t, err)
 	m.SetId(id)
@@ -120,7 +120,7 @@ func TestModelServiceV2_GetMany(t *testing.T) {
 	require.Nil(t, err)
 	c.Connect()
 
-	clientSvc := client.NewModelServiceV2[TestModel]()
+	clientSvc := client.NewModelServiceV2[models.TestModelV2]()
 	res, err := clientSvc.GetMany(bson.M{"name": m.Name}, nil)
 	require.Nil(t, err)
 	assert.Equal(t, len(res), 1)
@@ -136,10 +136,10 @@ func TestModelServiceV2_DeleteById(t *testing.T) {
 	go startSvr(svr)
 	defer stopSvr(svr)
 
-	m := TestModel{
+	m := models.TestModelV2{
 		Name: "Test Name",
 	}
-	modelSvc := service.NewModelServiceV2[TestModel]()
+	modelSvc := service.NewModelServiceV2[models.TestModelV2]()
 	id, err := modelSvc.InsertOne(m)
 	require.Nil(t, err)
 	m.SetId(id)
@@ -149,7 +149,7 @@ func TestModelServiceV2_DeleteById(t *testing.T) {
 	require.Nil(t, err)
 	c.Connect()
 
-	clientSvc := client.NewModelServiceV2[TestModel]()
+	clientSvc := client.NewModelServiceV2[models.TestModelV2]()
 	err = clientSvc.DeleteById(m.Id)
 	require.Nil(t, err)
 
@@ -166,10 +166,10 @@ func TestModelServiceV2_DeleteOne(t *testing.T) {
 	go startSvr(svr)
 	defer stopSvr(svr)
 
-	m := TestModel{
+	m := models.TestModelV2{
 		Name: "Test Name",
 	}
-	modelSvc := service.NewModelServiceV2[TestModel]()
+	modelSvc := service.NewModelServiceV2[models.TestModelV2]()
 	id, err := modelSvc.InsertOne(m)
 	require.Nil(t, err)
 	m.SetId(id)
@@ -179,7 +179,7 @@ func TestModelServiceV2_DeleteOne(t *testing.T) {
 	require.Nil(t, err)
 	c.Connect()
 
-	clientSvc := client.NewModelServiceV2[TestModel]()
+	clientSvc := client.NewModelServiceV2[models.TestModelV2]()
 	err = clientSvc.DeleteOne(bson.M{"name": m.Name})
 	require.Nil(t, err)
 
@@ -196,10 +196,10 @@ func TestModelServiceV2_DeleteMany(t *testing.T) {
 	go startSvr(svr)
 	defer stopSvr(svr)
 
-	m := TestModel{
+	m := models.TestModelV2{
 		Name: "Test Name",
 	}
-	modelSvc := service.NewModelServiceV2[TestModel]()
+	modelSvc := service.NewModelServiceV2[models.TestModelV2]()
 	id, err := modelSvc.InsertOne(m)
 	require.Nil(t, err)
 	m.SetId(id)
@@ -209,7 +209,7 @@ func TestModelServiceV2_DeleteMany(t *testing.T) {
 	require.Nil(t, err)
 	c.Connect()
 
-	clientSvc := client.NewModelServiceV2[TestModel]()
+	clientSvc := client.NewModelServiceV2[models.TestModelV2]()
 	err = clientSvc.DeleteMany(bson.M{"name": m.Name})
 	require.Nil(t, err)
 
@@ -226,10 +226,10 @@ func TestModelServiceV2_UpdateById(t *testing.T) {
 	go startSvr(svr)
 	defer stopSvr(svr)
 
-	m := TestModel{
+	m := models.TestModelV2{
 		Name: "Test Name",
 	}
-	modelSvc := service.NewModelServiceV2[TestModel]()
+	modelSvc := service.NewModelServiceV2[models.TestModelV2]()
 	id, err := modelSvc.InsertOne(m)
 	require.Nil(t, err)
 	m.SetId(id)
@@ -239,7 +239,7 @@ func TestModelServiceV2_UpdateById(t *testing.T) {
 	require.Nil(t, err)
 	c.Connect()
 
-	clientSvc := client.NewModelServiceV2[TestModel]()
+	clientSvc := client.NewModelServiceV2[models.TestModelV2]()
 	err = clientSvc.UpdateById(m.Id, bson.M{"$set": bson.M{"name": "New Name"}})
 	require.Nil(t, err)
 
@@ -256,10 +256,10 @@ func TestModelServiceV2_UpdateOne(t *testing.T) {
 	go startSvr(svr)
 	defer stopSvr(svr)
 
-	m := TestModel{
+	m := models.TestModelV2{
 		Name: "Test Name",
 	}
-	modelSvc := service.NewModelServiceV2[TestModel]()
+	modelSvc := service.NewModelServiceV2[models.TestModelV2]()
 	id, err := modelSvc.InsertOne(m)
 	require.Nil(t, err)
 	m.SetId(id)
@@ -269,7 +269,7 @@ func TestModelServiceV2_UpdateOne(t *testing.T) {
 	require.Nil(t, err)
 	c.Connect()
 
-	clientSvc := client.NewModelServiceV2[TestModel]()
+	clientSvc := client.NewModelServiceV2[models.TestModelV2]()
 	err = clientSvc.UpdateOne(bson.M{"name": m.Name}, bson.M{"$set": bson.M{"name": "New Name"}})
 	require.Nil(t, err)
 
@@ -286,13 +286,13 @@ func TestModelServiceV2_UpdateMany(t *testing.T) {
 	go startSvr(svr)
 	defer stopSvr(svr)
 
-	m1 := TestModel{
+	m1 := models.TestModelV2{
 		Name: "Test Name",
 	}
-	m2 := TestModel{
+	m2 := models.TestModelV2{
 		Name: "Test Name",
 	}
-	modelSvc := service.NewModelServiceV2[TestModel]()
+	modelSvc := service.NewModelServiceV2[models.TestModelV2]()
 	_, err = modelSvc.InsertOne(m1)
 	require.Nil(t, err)
 	_, err = modelSvc.InsertOne(m2)
@@ -303,7 +303,7 @@ func TestModelServiceV2_UpdateMany(t *testing.T) {
 	require.Nil(t, err)
 	c.Connect()
 
-	clientSvc := client.NewModelServiceV2[TestModel]()
+	clientSvc := client.NewModelServiceV2[models.TestModelV2]()
 	err = clientSvc.UpdateMany(bson.M{"name": "Test Name"}, bson.M{"$set": bson.M{"name": "New Name"}})
 	require.Nil(t, err)
 
@@ -320,10 +320,10 @@ func TestModelServiceV2_ReplaceById(t *testing.T) {
 	go startSvr(svr)
 	defer stopSvr(svr)
 
-	m := TestModel{
+	m := models.TestModelV2{
 		Name: "Test Name",
 	}
-	modelSvc := service.NewModelServiceV2[TestModel]()
+	modelSvc := service.NewModelServiceV2[models.TestModelV2]()
 	id, err := modelSvc.InsertOne(m)
 	require.Nil(t, err)
 	m.SetId(id)
@@ -333,7 +333,7 @@ func TestModelServiceV2_ReplaceById(t *testing.T) {
 	require.Nil(t, err)
 	c.Connect()
 
-	clientSvc := client.NewModelServiceV2[TestModel]()
+	clientSvc := client.NewModelServiceV2[models.TestModelV2]()
 	m.Name = "New Name"
 	err = clientSvc.ReplaceById(m.Id, m)
 	require.Nil(t, err)
@@ -351,10 +351,10 @@ func TestModelServiceV2_ReplaceOne(t *testing.T) {
 	go startSvr(svr)
 	defer stopSvr(svr)
 
-	m := TestModel{
+	m := models.TestModelV2{
 		Name: "Test Name",
 	}
-	modelSvc := service.NewModelServiceV2[TestModel]()
+	modelSvc := service.NewModelServiceV2[models.TestModelV2]()
 	id, err := modelSvc.InsertOne(m)
 	require.Nil(t, err)
 	m.SetId(id)
@@ -364,7 +364,7 @@ func TestModelServiceV2_ReplaceOne(t *testing.T) {
 	require.Nil(t, err)
 	c.Connect()
 
-	clientSvc := client.NewModelServiceV2[TestModel]()
+	clientSvc := client.NewModelServiceV2[models.TestModelV2]()
 	m.Name = "New Name"
 	err = clientSvc.ReplaceOne(bson.M{"name": "Test Name"}, m)
 	require.Nil(t, err)
@@ -386,8 +386,8 @@ func TestModelServiceV2_InsertOne(t *testing.T) {
 	require.Nil(t, err)
 	c.Connect()
 
-	clientSvc := client.NewModelServiceV2[TestModel]()
-	m := TestModel{
+	clientSvc := client.NewModelServiceV2[models.TestModelV2]()
+	m := models.TestModelV2{
 		Name: "Test Name",
 	}
 	id, err := clientSvc.InsertOne(m)
@@ -410,8 +410,8 @@ func TestModelServiceV2_InsertMany(t *testing.T) {
 	require.Nil(t, err)
 	c.Connect()
 
-	clientSvc := client.NewModelServiceV2[TestModel]()
-	testModels := []TestModel{
+	clientSvc := client.NewModelServiceV2[models.TestModelV2]()
+	testModels := []models.TestModelV2{
 		{Name: "Test Name 1"},
 		{Name: "Test Name 2"},
 	}
@@ -433,9 +433,9 @@ func TestModelServiceV2_Count(t *testing.T) {
 	go startSvr(svr)
 	defer stopSvr(svr)
 
-	modelSvc := service.NewModelServiceV2[TestModel]()
+	modelSvc := service.NewModelServiceV2[models.TestModelV2]()
 	for i := 0; i < 5; i++ {
-		_, err = modelSvc.InsertOne(TestModel{
+		_, err = modelSvc.InsertOne(models.TestModelV2{
 			Name: "Test Name",
 		})
 		require.Nil(t, err)
@@ -446,7 +446,7 @@ func TestModelServiceV2_Count(t *testing.T) {
 	require.Nil(t, err)
 	c.Connect()
 
-	clientSvc := client.NewModelServiceV2[TestModel]()
+	clientSvc := client.NewModelServiceV2[models.TestModelV2]()
 	count, err := clientSvc.Count(bson.M{})
 	require.Nil(t, err)
 

core/node/config/options.go

@@ -1,13 +0,0 @@
-package config
-
-import (
-	"github.com/crawlab-team/crawlab/core/interfaces"
-)
-
-type Option func(svc interfaces.NodeConfigService)
-
-func WithConfigPath(path string) Option {
-	return func(svc interfaces.NodeConfigService) {
-		svc.SetConfigPath(path)
-	}
-}

core/task/log/file_driver_test.go

@@ -2,27 +2,39 @@ package log
 
 import (
 	"fmt"
-	"github.com/stretchr/testify/require"
-	"go.mongodb.org/mongo-driver/bson/primitive"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
+var testLogDir string
+
 func setupFileDriverTest() {
-	cleanupFileDriverTest()
-	_ = os.MkdirAll("./tmp", os.ModePerm)
+	var err error
+	testLogDir, err = os.MkdirTemp("", "crawlab-test-logs")
+	if err != nil {
+		panic(err)
+	}
+	// Set the log path in viper configuration
+	viper.Set("log.path", testLogDir)
 }
 
 func cleanupFileDriverTest() {
-	_ = os.RemoveAll("./tmp")
+	_ = os.RemoveAll(testLogDir)
+	// Reset the log path in viper configuration
+	viper.Set("log.path", "")
 }
 
 func TestFileDriver_WriteLine(t *testing.T) {
 	setupFileDriverTest()
 	t.Cleanup(cleanupFileDriverTest)
 
-	d, err := newFileLogDriver(nil)
+	d, err := newFileLogDriver()
 	require.Nil(t, err)
 	defer d.Close()
 
@@ -31,7 +43,7 @@ func TestFileDriver_WriteLine(t *testing.T) {
 	err = d.WriteLine(id.Hex(), "it works")
 	require.Nil(t, err)
 
-	logFilePath := fmt.Sprintf("/var/log/crawlab/%s/log.txt", id.Hex())
+	logFilePath := filepath.Join(testLogDir, id.Hex(), "log.txt")
 	require.FileExists(t, logFilePath)
 	text, err := os.ReadFile(logFilePath)
 	require.Nil(t, err)
@@ -42,7 +54,7 @@ func TestFileDriver_WriteLines(t *testing.T) {
 	setupFileDriverTest()
 	t.Cleanup(cleanupFileDriverTest)
 
-	d, err := newFileLogDriver(nil)
+	d, err := newFileLogDriver()
 	require.Nil(t, err)
 	defer d.Close()
 
@@ -53,7 +65,7 @@ func TestFileDriver_WriteLines(t *testing.T) {
 		require.Nil(t, err)
 	}
 
-	logFilePath := fmt.Sprintf("/var/log/crawlab/%s/log.txt", id.Hex())
+	logFilePath := filepath.Join(testLogDir, id.Hex(), "log.txt")
 	require.FileExists(t, logFilePath)
 	text, err := os.ReadFile(logFilePath)
 	require.Nil(t, err)
@@ -66,7 +78,7 @@ func TestFileDriver_Find(t *testing.T) {
 	setupFileDriverTest()
 	t.Cleanup(cleanupFileDriverTest)
 
-	d, err := newFileLogDriver(nil)
+	d, err := newFileLogDriver()
 	require.Nil(t, err)
 	defer d.Close()
 

core/utils/bson.go

@@ -1,89 +1,10 @@
 package utils
 
 import (
-	"github.com/emirpasic/gods/sets/hashset"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
-	"reflect"
 )
 
-func BsonMEqual(v1, v2 bson.M) (ok bool) {
-	//ok = reflect.DeepEqual(v1, v2)
-	ok = bsonMEqual(v1, v2)
-	return ok
-}
-
-func bsonMEqual(v1, v2 bson.M) (ok bool) {
-	// all keys
-	allKeys := hashset.New()
-	for key := range v1 {
-		allKeys.Add(key)
-	}
-	for key := range v2 {
-		allKeys.Add(key)
-	}
-
-	for _, keyRes := range allKeys.Values() {
-		key := keyRes.(string)
-		v1Value, ok := v1[key]
-		if !ok {
-			return false
-		}
-		v2Value, ok := v2[key]
-		if !ok {
-			return false
-		}
-
-		mode := 0
-
-		var v1ValueBsonM bson.M
-		var v1ValueBsonA bson.A
-		switch v1Value.(type) {
-		case bson.M:
-			mode = 1
-			v1ValueBsonM = v1Value.(bson.M)
-		case bson.A:
-			mode = 2
-			v1ValueBsonA = v1Value.(bson.A)
-		}
-
-		var v2ValueBsonM bson.M
-		var v2ValueBsonA bson.A
-		switch v2Value.(type) {
-		case bson.M:
-			if mode != 1 {
-				return false
-			}
-			v2ValueBsonM = v2Value.(bson.M)
-		case bson.A:
-			if mode != 2 {
-				return false
-			}
-			v2ValueBsonA = v2Value.(bson.A)
-		}
-
-		switch mode {
-		case 0:
-			if v1Value != v2Value {
-				return false
-			}
-		case 1:
-			if !bsonMEqual(v1ValueBsonM, v2ValueBsonM) {
-				return false
-			}
-		case 2:
-			if !reflect.DeepEqual(v1ValueBsonA, v2ValueBsonA) {
-				return false
-			}
-		default:
-			// not reachable
-			return false
-		}
-	}
-
-	return true
-}
-
 func NormalizeBsonMObjectId(m bson.M) (res bson.M) {
 	for k, v := range m {
 		switch v.(type) {
@@ -99,18 +20,6 @@ func NormalizeBsonMObjectId(m bson.M) (res bson.M) {
 	return m
 }
 
-func DenormalizeBsonMObjectId(m bson.M) (res bson.M) {
-	for k, v := range m {
-		switch v.(type) {
-		case primitive.ObjectID:
-			m[k] = v.(primitive.ObjectID).Hex()
-		case bson.M:
-			m[k] = NormalizeBsonMObjectId(v.(bson.M))
-		}
-	}
-	return m
-}
-
 func NormalizeObjectId(v interface{}) (res interface{}) {
 	switch v.(type) {
 	case string:

core/utils/debug.go

@@ -1,18 +0,0 @@
-package utils
-
-import (
-	"fmt"
-	"github.com/spf13/viper"
-	"time"
-)
-
-func IsDebug() bool {
-	return viper.GetBool("debug")
-}
-
-func LogDebug(msg string) {
-	if !IsDebug() {
-		return
-	}
-	fmt.Println(fmt.Sprintf("[DEBUG] %s: %s", time.Now().Format("2006-01-02 15:04:05"), msg))
-}

core/utils/init.go

@@ -1,30 +0,0 @@
-package utils
-
-import (
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"sync"
-)
-
-var moduleInitializedMap = sync.Map{}
-
-func InitModule(id interfaces.ModuleId, fn func() error) (err error) {
-	res, ok := moduleInitializedMap.Load(id)
-	if ok {
-		initialized, _ := res.(bool)
-		if initialized {
-			return nil
-		}
-	}
-
-	if err := fn(); err != nil {
-		return err
-	}
-
-	moduleInitializedMap.Store(id, true)
-
-	return nil
-}
-
-func ForceInitModule(fn func() error) (err error) {
-	return fn()
-}

core/utils/json.go

@@ -1,12 +0,0 @@
-package utils
-
-import "encoding/json"
-
-func JsonToBytes(d interface{}) (bytes []byte, err error) {
-	switch d.(type) {
-	case []byte:
-		return d.([]byte), nil
-	default:
-		return json.Marshal(d)
-	}
-}

core/utils/process.go

@@ -1,23 +1,3 @@
-/*
- * Copyright (c) 2024. Core Digital Limited
- * 版权所有 (c) 2024. 重庆科锐数研科技有限公司 (Core Digital Limited)
- * All rights reserved. 保留所有权利。
- *
- * 该软件由 重庆科锐数研科技有限公司 (Core Digital Limited) 开发，未经明确书面许可，任何人不得使用、复制、修改或分发该软件的任何部分。
- * This software is developed by Core Digital Limited. No one is permitted to use, copy, modify, or distribute this software without explicit written permission.
- *
- * 许可证：
- * 该软件仅供授权使用。授权用户有权在授权范围内使用、复制、修改和分发该软件。
- * License:
- * This software is for authorized use only. Authorized users are permitted to use, copy, modify, and distribute this software within the scope of their authorization.
- *
- * 免责声明：
- * 该软件按“原样”提供，不附带任何明示或暗示的担保，包括但不限于对适销性和适用于特定目的的担保。在任何情况下，版权持有者或其许可方对因使用该软件而产生的任何损害或其他责任概不负责。
- * Disclaimer:
- * This software is provided "as is," without any express or implied warranties, including but not limited to warranties of merchantability and fitness for a particular purpose. In no event shall the copyright holder or its licensors be liable for any damages or other liability arising from the use of this software.
- *
- */
-
 package utils
 
 import (

core/utils/result.go

@@ -1,23 +0,0 @@
-package utils
-
-import (
-	"encoding/json"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-)
-
-func GetResultHash(value interface{}, keys []string) (res string, err error) {
-	m := make(map[string]interface{})
-	for _, k := range keys {
-		_value, ok := value.(interfaces.Result)
-		if !ok {
-			continue
-		}
-		v := _value.GetValue(k)
-		m[k] = v
-	}
-	data, err := json.Marshal(m)
-	if err != nil {
-		return "", err
-	}
-	return EncryptMd5(string(data)), nil
-}

core/utils/spider.go

@@ -1,8 +0,0 @@
-package utils
-
-func GetSpiderCol(col string, name string) string {
-	if col == "" {
-		return "results_" + name
-	}
-	return col
-}

core/utils/stats.go

@@ -1 +0,0 @@
-package utils