test: updated test cases

core/utils/bson.go

@@ -1,89 +1,10 @@
 package utils
 
 import (
-	"github.com/emirpasic/gods/sets/hashset"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
-	"reflect"
 )
 
-func BsonMEqual(v1, v2 bson.M) (ok bool) {
-	//ok = reflect.DeepEqual(v1, v2)
-	ok = bsonMEqual(v1, v2)
-	return ok
-}
-
-func bsonMEqual(v1, v2 bson.M) (ok bool) {
-	// all keys
-	allKeys := hashset.New()
-	for key := range v1 {
-		allKeys.Add(key)
-	}
-	for key := range v2 {
-		allKeys.Add(key)
-	}
-
-	for _, keyRes := range allKeys.Values() {
-		key := keyRes.(string)
-		v1Value, ok := v1[key]
-		if !ok {
-			return false
-		}
-		v2Value, ok := v2[key]
-		if !ok {
-			return false
-		}
-
-		mode := 0
-
-		var v1ValueBsonM bson.M
-		var v1ValueBsonA bson.A
-		switch v1Value.(type) {
-		case bson.M:
-			mode = 1
-			v1ValueBsonM = v1Value.(bson.M)
-		case bson.A:
-			mode = 2
-			v1ValueBsonA = v1Value.(bson.A)
-		}
-
-		var v2ValueBsonM bson.M
-		var v2ValueBsonA bson.A
-		switch v2Value.(type) {
-		case bson.M:
-			if mode != 1 {
-				return false
-			}
-			v2ValueBsonM = v2Value.(bson.M)
-		case bson.A:
-			if mode != 2 {
-				return false
-			}
-			v2ValueBsonA = v2Value.(bson.A)
-		}
-
-		switch mode {
-		case 0:
-			if v1Value != v2Value {
-				return false
-			}
-		case 1:
-			if !bsonMEqual(v1ValueBsonM, v2ValueBsonM) {
-				return false
-			}
-		case 2:
-			if !reflect.DeepEqual(v1ValueBsonA, v2ValueBsonA) {
-				return false
-			}
-		default:
-			// not reachable
-			return false
-		}
-	}
-
-	return true
-}
-
 func NormalizeBsonMObjectId(m bson.M) (res bson.M) {
 	for k, v := range m {
 		switch v.(type) {
@@ -99,18 +20,6 @@ func NormalizeBsonMObjectId(m bson.M) (res bson.M) {
 	return m
 }
 
-func DenormalizeBsonMObjectId(m bson.M) (res bson.M) {
-	for k, v := range m {
-		switch v.(type) {
-		case primitive.ObjectID:
-			m[k] = v.(primitive.ObjectID).Hex()
-		case bson.M:
-			m[k] = NormalizeBsonMObjectId(v.(bson.M))
-		}
-	}
-	return m
-}
-
 func NormalizeObjectId(v interface{}) (res interface{}) {
 	switch v.(type) {
 	case string:

core/utils/debug.go

@@ -1,18 +0,0 @@
-package utils
-
-import (
-	"fmt"
-	"github.com/spf13/viper"
-	"time"
-)
-
-func IsDebug() bool {
-	return viper.GetBool("debug")
-}
-
-func LogDebug(msg string) {
-	if !IsDebug() {
-		return
-	}
-	fmt.Println(fmt.Sprintf("[DEBUG] %s: %s", time.Now().Format("2006-01-02 15:04:05"), msg))
-}

core/utils/init.go

@@ -1,30 +0,0 @@
-package utils
-
-import (
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"sync"
-)
-
-var moduleInitializedMap = sync.Map{}
-
-func InitModule(id interfaces.ModuleId, fn func() error) (err error) {
-	res, ok := moduleInitializedMap.Load(id)
-	if ok {
-		initialized, _ := res.(bool)
-		if initialized {
-			return nil
-		}
-	}
-
-	if err := fn(); err != nil {
-		return err
-	}
-
-	moduleInitializedMap.Store(id, true)
-
-	return nil
-}
-
-func ForceInitModule(fn func() error) (err error) {
-	return fn()
-}

core/utils/json.go

@@ -1,12 +0,0 @@
-package utils
-
-import "encoding/json"
-
-func JsonToBytes(d interface{}) (bytes []byte, err error) {
-	switch d.(type) {
-	case []byte:
-		return d.([]byte), nil
-	default:
-		return json.Marshal(d)
-	}
-}

core/utils/process.go

@@ -1,23 +1,3 @@
-/*
- * Copyright (c) 2024. Core Digital Limited
- * 版权所有 (c) 2024. 重庆科锐数研科技有限公司 (Core Digital Limited)
- * All rights reserved. 保留所有权利。
- *
- * 该软件由 重庆科锐数研科技有限公司 (Core Digital Limited) 开发，未经明确书面许可，任何人不得使用、复制、修改或分发该软件的任何部分。
- * This software is developed by Core Digital Limited. No one is permitted to use, copy, modify, or distribute this software without explicit written permission.
- *
- * 许可证：
- * 该软件仅供授权使用。授权用户有权在授权范围内使用、复制、修改和分发该软件。
- * License:
- * This software is for authorized use only. Authorized users are permitted to use, copy, modify, and distribute this software within the scope of their authorization.
- *
- * 免责声明：
- * 该软件按“原样”提供，不附带任何明示或暗示的担保，包括但不限于对适销性和适用于特定目的的担保。在任何情况下，版权持有者或其许可方对因使用该软件而产生的任何损害或其他责任概不负责。
- * Disclaimer:
- * This software is provided "as is," without any express or implied warranties, including but not limited to warranties of merchantability and fitness for a particular purpose. In no event shall the copyright holder or its licensors be liable for any damages or other liability arising from the use of this software.
- *
- */
-
 package utils
 
 import (

core/utils/result.go

@@ -1,23 +0,0 @@
-package utils
-
-import (
-	"encoding/json"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-)
-
-func GetResultHash(value interface{}, keys []string) (res string, err error) {
-	m := make(map[string]interface{})
-	for _, k := range keys {
-		_value, ok := value.(interfaces.Result)
-		if !ok {
-			continue
-		}
-		v := _value.GetValue(k)
-		m[k] = v
-	}
-	data, err := json.Marshal(m)
-	if err != nil {
-		return "", err
-	}
-	return EncryptMd5(string(data)), nil
-}

core/utils/spider.go

@@ -1,8 +0,0 @@
-package utils
-
-func GetSpiderCol(col string, name string) string {
-	if col == "" {
-		return "results_" + name
-	}
-	return col
-}

core/utils/stats.go

@@ -1 +0,0 @@
-package utils