From cecf4dfd1d4de87e78e4f2617c857ebdf6ab5f22 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 20 Oct 2024 17:55:57 +0800 Subject: [PATCH] test: updated test cases --- core/config/config_test.go | 3 + core/event/options.go | 1 - core/grpc/payload/model_service_v2_payload.go | 17 ---- .../server/model_base_service_v2_server.go | 9 +- core/grpc/server/task_server_v2.go | 1 - core/grpc/server/utils_handle.go | 13 --- core/interfaces/color.go | 7 -- core/interfaces/color_service.go | 7 -- core/models/client/model_service_v2.go | 2 +- core/models/client/model_service_v2_test.go | 92 +++++++++---------- core/node/config/options.go | 13 --- core/task/log/file_driver_test.go | 32 +++++-- core/utils/bson.go | 91 ------------------ core/utils/debug.go | 18 ---- core/utils/init.go | 30 ------ core/utils/json.go | 12 --- core/utils/process.go | 20 ---- core/utils/result.go | 23 ----- core/utils/spider.go | 8 -- core/utils/stats.go | 1 - 20 files changed, 79 insertions(+), 321 deletions(-) delete mode 100644 core/event/options.go delete mode 100644 core/grpc/payload/model_service_v2_payload.go delete mode 100644 core/interfaces/color.go delete mode 100644 core/interfaces/color_service.go delete mode 100644 core/node/config/options.go delete mode 100644 core/utils/debug.go delete mode 100644 core/utils/init.go delete mode 100644 core/utils/json.go delete mode 100644 core/utils/result.go delete mode 100644 core/utils/spider.go delete mode 100644 core/utils/stats.go diff --git a/core/config/config_test.go b/core/config/config_test.go index 0a3c6b57..ab68cac2 100644 --- a/core/config/config_test.go +++ b/core/config/config_test.go @@ -60,6 +60,9 @@ server: err = os.WriteFile(configPath, configContent, 0644) require.NoError(t, err, "Failed to write config file") + // Remove the environment variable before testing with config file + os.Unsetenv("CRAWLAB_MONGO_HOST") + // Create a new Config instance with the config file cWithFile := Config{Name: configPath} err = cWithFile.Init() diff --git a/core/event/options.go b/core/event/options.go deleted file mode 100644 index 0e4b82e8..00000000 --- a/core/event/options.go +++ /dev/null @@ -1 +0,0 @@ -package event diff --git a/core/grpc/payload/model_service_v2_payload.go b/core/grpc/payload/model_service_v2_payload.go deleted file mode 100644 index 926386cf..00000000 --- a/core/grpc/payload/model_service_v2_payload.go +++ /dev/null @@ -1,17 +0,0 @@ -package payload - -import ( - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type ModelServiceV2Payload struct { - Type string `json:"type,omitempty"` - Id primitive.ObjectID `json:"_id,omitempty"` - Query bson.M `json:"query,omitempty"` - FindOptions *mongo.FindOptions `json:"find_options,omitempty"` - Model any `json:"model,omitempty"` - Update bson.M `json:"update,omitempty"` - Models []any `json:"models,omitempty"` -} diff --git a/core/grpc/server/model_base_service_v2_server.go b/core/grpc/server/model_base_service_v2_server.go index 26b6dd0b..dbadbaf8 100644 --- a/core/grpc/server/model_base_service_v2_server.go +++ b/core/grpc/server/model_base_service_v2_server.go @@ -3,13 +3,14 @@ package server import ( "context" "encoding/json" + "reflect" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/grpc" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - "reflect" ) var ( @@ -253,7 +254,11 @@ func (svr ModelBaseServiceServerV2) InsertMany(_ context.Context, req *grpc.Mode modelsSlice := reflect.ValueOf(modelsSlicePtr).Elem() modelsInterface := make([]any, modelsSlice.Len()) for i := 0; i < modelsSlice.Len(); i++ { - modelsInterface[i] = modelsSlice.Index(i).Interface() + modelValue := modelsSlice.Index(i) + if modelValue.FieldByName("Id").Interface().(primitive.ObjectID).IsZero() { + modelValue.FieldByName("Id").Set(reflect.ValueOf(primitive.NewObjectID())) + } + modelsInterface[i] = modelValue.Interface() } modelSvc := GetModelService[bson.M](req.ModelType) r, err := modelSvc.GetCol().GetCollection().InsertMany(modelSvc.GetCol().GetContext(), modelsInterface) diff --git a/core/grpc/server/task_server_v2.go b/core/grpc/server/task_server_v2.go index 8703296d..3f53455b 100644 --- a/core/grpc/server/task_server_v2.go +++ b/core/grpc/server/task_server_v2.go @@ -39,7 +39,6 @@ type TaskServerV2 struct { func (svr TaskServerV2) Subscribe(stream grpc.TaskService_SubscribeServer) (err error) { for { msg, err := stream.Recv() - utils.LogDebug(msg.String()) if err == io.EOF { return nil } diff --git a/core/grpc/server/utils_handle.go b/core/grpc/server/utils_handle.go index d860360a..756a740f 100644 --- a/core/grpc/server/utils_handle.go +++ b/core/grpc/server/utils_handle.go @@ -38,16 +38,3 @@ func HandleSuccessWithData(data interface{}) (res *grpc.Response, err error) { Data: bytes, }, nil } - -func HandleSuccessWithListData(data interface{}, total int) (res *grpc.Response, err error) { - bytes, err := json.Marshal(data) - if err != nil { - return HandleError(err) - } - return &grpc.Response{ - Code: grpc.ResponseCode_OK, - Message: "success", - Data: bytes, - Total: int64(total), - }, nil -} diff --git a/core/interfaces/color.go b/core/interfaces/color.go deleted file mode 100644 index 7f10fbc5..00000000 --- a/core/interfaces/color.go +++ /dev/null @@ -1,7 +0,0 @@ -package interfaces - -type Color interface { - Entity - GetHex() string - GetName() string -} diff --git a/core/interfaces/color_service.go b/core/interfaces/color_service.go deleted file mode 100644 index 68bba285..00000000 --- a/core/interfaces/color_service.go +++ /dev/null @@ -1,7 +0,0 @@ -package interfaces - -type ColorService interface { - Injectable - GetByName(name string) (res Color, err error) - GetRandom() (res Color, err error) -} diff --git a/core/models/client/model_service_v2.go b/core/models/client/model_service_v2.go index 8ad827d8..2077e3e9 100644 --- a/core/models/client/model_service_v2.go +++ b/core/models/client/model_service_v2.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/db/mongo" - grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/crawlab/grpc" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "reflect" diff --git a/core/models/client/model_service_v2_test.go b/core/models/client/model_service_v2_test.go index 6738af4a..9ed1b05d 100644 --- a/core/models/client/model_service_v2_test.go +++ b/core/models/client/model_service_v2_test.go @@ -2,10 +2,14 @@ package client_test import ( "context" + "testing" + "time" + + "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/apex/log" "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "github.com/spf13/viper" @@ -14,12 +18,8 @@ import ( "go.mongodb.org/mongo-driver/bson" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "testing" - "time" ) -type TestModel models.TestModelV2 - func setupTestDB() { viper.Set("mongo.db", "testdb") } @@ -51,10 +51,10 @@ func TestModelServiceV2_GetById(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := TestModel{ + m := models.TestModelV2{ Name: "Test Name", } - modelSvc := service.NewModelServiceV2[TestModel]() + modelSvc := service.NewModelServiceV2[models.TestModelV2]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -64,7 +64,7 @@ func TestModelServiceV2_GetById(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelServiceV2[TestModel]() + clientSvc := client.NewModelServiceV2[models.TestModelV2]() res, err := clientSvc.GetById(m.Id) require.Nil(t, err) assert.Equal(t, res.Id, m.Id) @@ -79,10 +79,10 @@ func TestModelServiceV2_GetOne(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := TestModel{ + m := models.TestModelV2{ Name: "Test Name", } - modelSvc := service.NewModelServiceV2[TestModel]() + modelSvc := service.NewModelServiceV2[models.TestModelV2]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -92,7 +92,7 @@ func TestModelServiceV2_GetOne(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelServiceV2[TestModel]() + clientSvc := client.NewModelServiceV2[models.TestModelV2]() res, err := clientSvc.GetOne(bson.M{"name": m.Name}, nil) require.Nil(t, err) assert.Equal(t, res.Id, m.Id) @@ -107,10 +107,10 @@ func TestModelServiceV2_GetMany(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := TestModel{ + m := models.TestModelV2{ Name: "Test Name", } - modelSvc := service.NewModelServiceV2[TestModel]() + modelSvc := service.NewModelServiceV2[models.TestModelV2]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -120,7 +120,7 @@ func TestModelServiceV2_GetMany(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelServiceV2[TestModel]() + clientSvc := client.NewModelServiceV2[models.TestModelV2]() res, err := clientSvc.GetMany(bson.M{"name": m.Name}, nil) require.Nil(t, err) assert.Equal(t, len(res), 1) @@ -136,10 +136,10 @@ func TestModelServiceV2_DeleteById(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := TestModel{ + m := models.TestModelV2{ Name: "Test Name", } - modelSvc := service.NewModelServiceV2[TestModel]() + modelSvc := service.NewModelServiceV2[models.TestModelV2]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -149,7 +149,7 @@ func TestModelServiceV2_DeleteById(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelServiceV2[TestModel]() + clientSvc := client.NewModelServiceV2[models.TestModelV2]() err = clientSvc.DeleteById(m.Id) require.Nil(t, err) @@ -166,10 +166,10 @@ func TestModelServiceV2_DeleteOne(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := TestModel{ + m := models.TestModelV2{ Name: "Test Name", } - modelSvc := service.NewModelServiceV2[TestModel]() + modelSvc := service.NewModelServiceV2[models.TestModelV2]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -179,7 +179,7 @@ func TestModelServiceV2_DeleteOne(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelServiceV2[TestModel]() + clientSvc := client.NewModelServiceV2[models.TestModelV2]() err = clientSvc.DeleteOne(bson.M{"name": m.Name}) require.Nil(t, err) @@ -196,10 +196,10 @@ func TestModelServiceV2_DeleteMany(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := TestModel{ + m := models.TestModelV2{ Name: "Test Name", } - modelSvc := service.NewModelServiceV2[TestModel]() + modelSvc := service.NewModelServiceV2[models.TestModelV2]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -209,7 +209,7 @@ func TestModelServiceV2_DeleteMany(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelServiceV2[TestModel]() + clientSvc := client.NewModelServiceV2[models.TestModelV2]() err = clientSvc.DeleteMany(bson.M{"name": m.Name}) require.Nil(t, err) @@ -226,10 +226,10 @@ func TestModelServiceV2_UpdateById(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := TestModel{ + m := models.TestModelV2{ Name: "Test Name", } - modelSvc := service.NewModelServiceV2[TestModel]() + modelSvc := service.NewModelServiceV2[models.TestModelV2]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -239,7 +239,7 @@ func TestModelServiceV2_UpdateById(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelServiceV2[TestModel]() + clientSvc := client.NewModelServiceV2[models.TestModelV2]() err = clientSvc.UpdateById(m.Id, bson.M{"$set": bson.M{"name": "New Name"}}) require.Nil(t, err) @@ -256,10 +256,10 @@ func TestModelServiceV2_UpdateOne(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := TestModel{ + m := models.TestModelV2{ Name: "Test Name", } - modelSvc := service.NewModelServiceV2[TestModel]() + modelSvc := service.NewModelServiceV2[models.TestModelV2]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -269,7 +269,7 @@ func TestModelServiceV2_UpdateOne(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelServiceV2[TestModel]() + clientSvc := client.NewModelServiceV2[models.TestModelV2]() err = clientSvc.UpdateOne(bson.M{"name": m.Name}, bson.M{"$set": bson.M{"name": "New Name"}}) require.Nil(t, err) @@ -286,13 +286,13 @@ func TestModelServiceV2_UpdateMany(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m1 := TestModel{ + m1 := models.TestModelV2{ Name: "Test Name", } - m2 := TestModel{ + m2 := models.TestModelV2{ Name: "Test Name", } - modelSvc := service.NewModelServiceV2[TestModel]() + modelSvc := service.NewModelServiceV2[models.TestModelV2]() _, err = modelSvc.InsertOne(m1) require.Nil(t, err) _, err = modelSvc.InsertOne(m2) @@ -303,7 +303,7 @@ func TestModelServiceV2_UpdateMany(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelServiceV2[TestModel]() + clientSvc := client.NewModelServiceV2[models.TestModelV2]() err = clientSvc.UpdateMany(bson.M{"name": "Test Name"}, bson.M{"$set": bson.M{"name": "New Name"}}) require.Nil(t, err) @@ -320,10 +320,10 @@ func TestModelServiceV2_ReplaceById(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := TestModel{ + m := models.TestModelV2{ Name: "Test Name", } - modelSvc := service.NewModelServiceV2[TestModel]() + modelSvc := service.NewModelServiceV2[models.TestModelV2]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -333,7 +333,7 @@ func TestModelServiceV2_ReplaceById(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelServiceV2[TestModel]() + clientSvc := client.NewModelServiceV2[models.TestModelV2]() m.Name = "New Name" err = clientSvc.ReplaceById(m.Id, m) require.Nil(t, err) @@ -351,10 +351,10 @@ func TestModelServiceV2_ReplaceOne(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := TestModel{ + m := models.TestModelV2{ Name: "Test Name", } - modelSvc := service.NewModelServiceV2[TestModel]() + modelSvc := service.NewModelServiceV2[models.TestModelV2]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -364,7 +364,7 @@ func TestModelServiceV2_ReplaceOne(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelServiceV2[TestModel]() + clientSvc := client.NewModelServiceV2[models.TestModelV2]() m.Name = "New Name" err = clientSvc.ReplaceOne(bson.M{"name": "Test Name"}, m) require.Nil(t, err) @@ -386,8 +386,8 @@ func TestModelServiceV2_InsertOne(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelServiceV2[TestModel]() - m := TestModel{ + clientSvc := client.NewModelServiceV2[models.TestModelV2]() + m := models.TestModelV2{ Name: "Test Name", } id, err := clientSvc.InsertOne(m) @@ -410,8 +410,8 @@ func TestModelServiceV2_InsertMany(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelServiceV2[TestModel]() - testModels := []TestModel{ + clientSvc := client.NewModelServiceV2[models.TestModelV2]() + testModels := []models.TestModelV2{ {Name: "Test Name 1"}, {Name: "Test Name 2"}, } @@ -433,9 +433,9 @@ func TestModelServiceV2_Count(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - modelSvc := service.NewModelServiceV2[TestModel]() + modelSvc := service.NewModelServiceV2[models.TestModelV2]() for i := 0; i < 5; i++ { - _, err = modelSvc.InsertOne(TestModel{ + _, err = modelSvc.InsertOne(models.TestModelV2{ Name: "Test Name", }) require.Nil(t, err) @@ -446,7 +446,7 @@ func TestModelServiceV2_Count(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelServiceV2[TestModel]() + clientSvc := client.NewModelServiceV2[models.TestModelV2]() count, err := clientSvc.Count(bson.M{}) require.Nil(t, err) diff --git a/core/node/config/options.go b/core/node/config/options.go deleted file mode 100644 index 3bc29719..00000000 --- a/core/node/config/options.go +++ /dev/null @@ -1,13 +0,0 @@ -package config - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" -) - -type Option func(svc interfaces.NodeConfigService) - -func WithConfigPath(path string) Option { - return func(svc interfaces.NodeConfigService) { - svc.SetConfigPath(path) - } -} diff --git a/core/task/log/file_driver_test.go b/core/task/log/file_driver_test.go index 260edf71..f0b9714d 100644 --- a/core/task/log/file_driver_test.go +++ b/core/task/log/file_driver_test.go @@ -2,27 +2,39 @@ package log import ( "fmt" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson/primitive" "os" + "path/filepath" "strings" "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson/primitive" ) +var testLogDir string + func setupFileDriverTest() { - cleanupFileDriverTest() - _ = os.MkdirAll("./tmp", os.ModePerm) + var err error + testLogDir, err = os.MkdirTemp("", "crawlab-test-logs") + if err != nil { + panic(err) + } + // Set the log path in viper configuration + viper.Set("log.path", testLogDir) } func cleanupFileDriverTest() { - _ = os.RemoveAll("./tmp") + _ = os.RemoveAll(testLogDir) + // Reset the log path in viper configuration + viper.Set("log.path", "") } func TestFileDriver_WriteLine(t *testing.T) { setupFileDriverTest() t.Cleanup(cleanupFileDriverTest) - d, err := newFileLogDriver(nil) + d, err := newFileLogDriver() require.Nil(t, err) defer d.Close() @@ -31,7 +43,7 @@ func TestFileDriver_WriteLine(t *testing.T) { err = d.WriteLine(id.Hex(), "it works") require.Nil(t, err) - logFilePath := fmt.Sprintf("/var/log/crawlab/%s/log.txt", id.Hex()) + logFilePath := filepath.Join(testLogDir, id.Hex(), "log.txt") require.FileExists(t, logFilePath) text, err := os.ReadFile(logFilePath) require.Nil(t, err) @@ -42,7 +54,7 @@ func TestFileDriver_WriteLines(t *testing.T) { setupFileDriverTest() t.Cleanup(cleanupFileDriverTest) - d, err := newFileLogDriver(nil) + d, err := newFileLogDriver() require.Nil(t, err) defer d.Close() @@ -53,7 +65,7 @@ func TestFileDriver_WriteLines(t *testing.T) { require.Nil(t, err) } - logFilePath := fmt.Sprintf("/var/log/crawlab/%s/log.txt", id.Hex()) + logFilePath := filepath.Join(testLogDir, id.Hex(), "log.txt") require.FileExists(t, logFilePath) text, err := os.ReadFile(logFilePath) require.Nil(t, err) @@ -66,7 +78,7 @@ func TestFileDriver_Find(t *testing.T) { setupFileDriverTest() t.Cleanup(cleanupFileDriverTest) - d, err := newFileLogDriver(nil) + d, err := newFileLogDriver() require.Nil(t, err) defer d.Close() diff --git a/core/utils/bson.go b/core/utils/bson.go index e6443e88..ce743ebb 100644 --- a/core/utils/bson.go +++ b/core/utils/bson.go @@ -1,89 +1,10 @@ package utils import ( - "github.com/emirpasic/gods/sets/hashset" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - "reflect" ) -func BsonMEqual(v1, v2 bson.M) (ok bool) { - //ok = reflect.DeepEqual(v1, v2) - ok = bsonMEqual(v1, v2) - return ok -} - -func bsonMEqual(v1, v2 bson.M) (ok bool) { - // all keys - allKeys := hashset.New() - for key := range v1 { - allKeys.Add(key) - } - for key := range v2 { - allKeys.Add(key) - } - - for _, keyRes := range allKeys.Values() { - key := keyRes.(string) - v1Value, ok := v1[key] - if !ok { - return false - } - v2Value, ok := v2[key] - if !ok { - return false - } - - mode := 0 - - var v1ValueBsonM bson.M - var v1ValueBsonA bson.A - switch v1Value.(type) { - case bson.M: - mode = 1 - v1ValueBsonM = v1Value.(bson.M) - case bson.A: - mode = 2 - v1ValueBsonA = v1Value.(bson.A) - } - - var v2ValueBsonM bson.M - var v2ValueBsonA bson.A - switch v2Value.(type) { - case bson.M: - if mode != 1 { - return false - } - v2ValueBsonM = v2Value.(bson.M) - case bson.A: - if mode != 2 { - return false - } - v2ValueBsonA = v2Value.(bson.A) - } - - switch mode { - case 0: - if v1Value != v2Value { - return false - } - case 1: - if !bsonMEqual(v1ValueBsonM, v2ValueBsonM) { - return false - } - case 2: - if !reflect.DeepEqual(v1ValueBsonA, v2ValueBsonA) { - return false - } - default: - // not reachable - return false - } - } - - return true -} - func NormalizeBsonMObjectId(m bson.M) (res bson.M) { for k, v := range m { switch v.(type) { @@ -99,18 +20,6 @@ func NormalizeBsonMObjectId(m bson.M) (res bson.M) { return m } -func DenormalizeBsonMObjectId(m bson.M) (res bson.M) { - for k, v := range m { - switch v.(type) { - case primitive.ObjectID: - m[k] = v.(primitive.ObjectID).Hex() - case bson.M: - m[k] = NormalizeBsonMObjectId(v.(bson.M)) - } - } - return m -} - func NormalizeObjectId(v interface{}) (res interface{}) { switch v.(type) { case string: diff --git a/core/utils/debug.go b/core/utils/debug.go deleted file mode 100644 index 09ef3098..00000000 --- a/core/utils/debug.go +++ /dev/null @@ -1,18 +0,0 @@ -package utils - -import ( - "fmt" - "github.com/spf13/viper" - "time" -) - -func IsDebug() bool { - return viper.GetBool("debug") -} - -func LogDebug(msg string) { - if !IsDebug() { - return - } - fmt.Println(fmt.Sprintf("[DEBUG] %s: %s", time.Now().Format("2006-01-02 15:04:05"), msg)) -} diff --git a/core/utils/init.go b/core/utils/init.go deleted file mode 100644 index 21a12bd8..00000000 --- a/core/utils/init.go +++ /dev/null @@ -1,30 +0,0 @@ -package utils - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "sync" -) - -var moduleInitializedMap = sync.Map{} - -func InitModule(id interfaces.ModuleId, fn func() error) (err error) { - res, ok := moduleInitializedMap.Load(id) - if ok { - initialized, _ := res.(bool) - if initialized { - return nil - } - } - - if err := fn(); err != nil { - return err - } - - moduleInitializedMap.Store(id, true) - - return nil -} - -func ForceInitModule(fn func() error) (err error) { - return fn() -} diff --git a/core/utils/json.go b/core/utils/json.go deleted file mode 100644 index 564a67eb..00000000 --- a/core/utils/json.go +++ /dev/null @@ -1,12 +0,0 @@ -package utils - -import "encoding/json" - -func JsonToBytes(d interface{}) (bytes []byte, err error) { - switch d.(type) { - case []byte: - return d.([]byte), nil - default: - return json.Marshal(d) - } -} diff --git a/core/utils/process.go b/core/utils/process.go index 3ebb3a42..72575f7f 100644 --- a/core/utils/process.go +++ b/core/utils/process.go @@ -1,23 +1,3 @@ -/* - * Copyright (c) 2024. Core Digital Limited - * 版权所有 (c) 2024. 重庆科锐数研科技有限公司 (Core Digital Limited) - * All rights reserved. 保留所有权利。 - * - * 该软件由 重庆科锐数研科技有限公司 (Core Digital Limited) 开发，未经明确书面许可，任何人不得使用、复制、修改或分发该软件的任何部分。 - * This software is developed by Core Digital Limited. No one is permitted to use, copy, modify, or distribute this software without explicit written permission. - * - * 许可证： - * 该软件仅供授权使用。授权用户有权在授权范围内使用、复制、修改和分发该软件。 - * License: - * This software is for authorized use only. Authorized users are permitted to use, copy, modify, and distribute this software within the scope of their authorization. - * - * 免责声明： - * 该软件按“原样”提供，不附带任何明示或暗示的担保，包括但不限于对适销性和适用于特定目的的担保。在任何情况下，版权持有者或其许可方对因使用该软件而产生的任何损害或其他责任概不负责。 - * Disclaimer: - * This software is provided "as is," without any express or implied warranties, including but not limited to warranties of merchantability and fitness for a particular purpose. In no event shall the copyright holder or its licensors be liable for any damages or other liability arising from the use of this software. - * - */ - package utils import ( diff --git a/core/utils/result.go b/core/utils/result.go deleted file mode 100644 index 62450309..00000000 --- a/core/utils/result.go +++ /dev/null @@ -1,23 +0,0 @@ -package utils - -import ( - "encoding/json" - "github.com/crawlab-team/crawlab/core/interfaces" -) - -func GetResultHash(value interface{}, keys []string) (res string, err error) { - m := make(map[string]interface{}) - for _, k := range keys { - _value, ok := value.(interfaces.Result) - if !ok { - continue - } - v := _value.GetValue(k) - m[k] = v - } - data, err := json.Marshal(m) - if err != nil { - return "", err - } - return EncryptMd5(string(data)), nil -} diff --git a/core/utils/spider.go b/core/utils/spider.go deleted file mode 100644 index 4484ccf0..00000000 --- a/core/utils/spider.go +++ /dev/null @@ -1,8 +0,0 @@ -package utils - -func GetSpiderCol(col string, name string) string { - if col == "" { - return "results_" + name - } - return col -} diff --git a/core/utils/stats.go b/core/utils/stats.go deleted file mode 100644 index d4b585bf..00000000 --- a/core/utils/stats.go +++ /dev/null @@ -1 +0,0 @@ -package utils