diff --git a/backend/go.mod b/backend/go.mod index d9f52b42..5ec77932 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -4,7 +4,6 @@ go 1.22.9 replace ( github.com/crawlab-team/crawlab/core => ../core - github.com/crawlab-team/crawlab/db => ../db github.com/crawlab-team/crawlab/grpc => ../grpc github.com/crawlab-team/crawlab/trace => ../trace github.com/crawlab-team/crawlab/vcs => ../vcs @@ -29,7 +28,6 @@ require ( github.com/cloudflare/circl v1.3.7 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect - github.com/crawlab-team/crawlab/db v0.0.0 // indirect github.com/crawlab-team/crawlab/grpc v0.0.0 // indirect github.com/crawlab-team/crawlab/trace v0.0.0 // indirect github.com/crawlab-team/crawlab/vcs v0.0.0 // indirect diff --git a/core/controllers/base.go b/core/controllers/base.go index aa1dbeed..d8c4dd05 100644 --- a/core/controllers/base.go +++ b/core/controllers/base.go @@ -4,7 +4,7 @@ import ( "errors" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/db/mongo" + "github.com/crawlab-team/crawlab/core/mongo" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/controllers/base_test.go b/core/controllers/base_test.go index 05197026..85cb4b34 100644 --- a/core/controllers/base_test.go +++ b/core/controllers/base_test.go @@ -8,13 +8,13 @@ import ( "github.com/crawlab-team/crawlab/core/middlewares" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/user" "github.com/spf13/viper" "net/http" "net/http/httptest" "testing" - "github.com/crawlab-team/crawlab/db/mongo" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" ) diff --git a/core/controllers/filter.go b/core/controllers/filter.go index 9455804d..c7d753d9 100644 --- a/core/controllers/filter.go +++ b/core/controllers/filter.go @@ -2,7 +2,7 @@ package controllers import ( "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/db/mongo" + "github.com/crawlab-team/crawlab/core/mongo" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" mongo2 "go.mongodb.org/mongo-driver/mongo" diff --git a/core/controllers/project.go b/core/controllers/project.go index 37b34056..0bd21595 100644 --- a/core/controllers/project.go +++ b/core/controllers/project.go @@ -4,7 +4,7 @@ import ( "errors" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/db/mongo" + "github.com/crawlab-team/crawlab/core/mongo" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/controllers/spider.go b/core/controllers/spider.go index edc05507..bb924dca 100644 --- a/core/controllers/spider.go +++ b/core/controllers/spider.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + mongo2 "github.com/crawlab-team/crawlab/core/mongo" "math" "os" "path/filepath" @@ -14,12 +15,10 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/crawlab/db/mongo" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo" ) func GetSpiderById(c *gin.Context) { @@ -29,7 +28,7 @@ func GetSpiderById(c *gin.Context) { return } s, err := service.NewModelService[models.Spider]().GetById(id) - if errors.Is(err, mongo2.ErrNoDocuments) { + if errors.Is(err, mongo.ErrNoDocuments) { HandleErrorNotFound(c, err) return } @@ -41,7 +40,7 @@ func GetSpiderById(c *gin.Context) { // stat s.Stat, err = service.NewModelService[models.SpiderStat]().GetById(s.Id) if err != nil { - if !errors.Is(err, mongo2.ErrNoDocuments) { + if !errors.Is(err, mongo.ErrNoDocuments) { HandleErrorInternalServerError(c, err) return } @@ -51,7 +50,7 @@ func GetSpiderById(c *gin.Context) { if s.ColName == "" && !s.ColId.IsZero() { col, err := service.NewModelService[models.DataCollection]().GetById(s.ColId) if err != nil { - if !errors.Is(err, mongo2.ErrNoDocuments) { + if !errors.Is(err, mongo.ErrNoDocuments) { HandleErrorInternalServerError(c, err) return } @@ -64,7 +63,7 @@ func GetSpiderById(c *gin.Context) { if utils.IsPro() && !s.GitId.IsZero() { s.Git, err = service.NewModelService[models.Git]().GetById(s.GitId) if err != nil { - if !errors.Is(err, mongo2.ErrNoDocuments) { + if !errors.Is(err, mongo.ErrNoDocuments) { HandleErrorInternalServerError(c, err) return } @@ -100,13 +99,13 @@ func getSpiderListWithStats(c *gin.Context) { sort := MustGetSortOption(c) // get list - spiders, err := service.NewModelService[models.Spider]().GetMany(query, &mongo.FindOptions{ + spiders, err := service.NewModelService[models.Spider]().GetMany(query, &mongo2.FindOptions{ Sort: sort, Skip: pagination.Size * (pagination.Page - 1), Limit: pagination.Size, }) if err != nil { - if err.Error() != mongo2.ErrNoDocuments.Error() { + if err.Error() != mongo.ErrNoDocuments.Error() { HandleErrorInternalServerError(c, err) } return @@ -347,7 +346,7 @@ func DeleteSpiderById(c *gin.Context) { return } - if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { + if err := mongo2.RunTransaction(func(context mongo.SessionContext) (err error) { // delete spider err = service.NewModelService[models.Spider]().DeleteById(id) if err != nil { @@ -448,7 +447,7 @@ func DeleteSpiderList(c *gin.Context) { return } - if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { + if err := mongo2.RunTransaction(func(context mongo.SessionContext) (err error) { // delete spiders if err := service.NewModelService[models.Spider]().DeleteMany(bson.M{ "_id": bson.M{ @@ -683,11 +682,11 @@ func GetSpiderResults(c *gin.Context) { pagination := MustGetPagination(c) query := getResultListQuery(c) - col := mongo.GetMongoCol(s.ColName) + col := mongo2.GetMongoCol(s.ColName) var results []bson.M - err = col.Find(utils.GetMongoQuery(query), utils.GetMongoOpts(&generic.ListOptions{ - Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}}, + err = col.Find(mongo2.GetMongoQuery(query), mongo2.GetMongoOpts(&mongo2.ListOptions{ + Sort: []mongo2.ListSort{{"_id", mongo2.SortDirectionDesc}}, Skip: pagination.Size * (pagination.Page - 1), Limit: pagination.Size, })).All(&results) @@ -696,7 +695,7 @@ func GetSpiderResults(c *gin.Context) { return } - total, err := mongo.GetMongoCol(s.ColName).Count(utils.GetMongoQuery(query)) + total, err := mongo2.GetMongoCol(s.ColName).Count(mongo2.GetMongoQuery(query)) if err != nil { HandleErrorInternalServerError(c, err) return diff --git a/core/controllers/task.go b/core/controllers/task.go index 868ef107..af110dbe 100644 --- a/core/controllers/task.go +++ b/core/controllers/task.go @@ -6,11 +6,11 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" + mongo3 "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/task/log" "github.com/crawlab-team/crawlab/core/task/scheduler" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -80,7 +80,7 @@ func GetTaskList(c *gin.Context) { sort := MustGetSortOption(c) // get tasks - tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo.FindOptions{ + tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo3.FindOptions{ Sort: sort, Skip: pagination.Size * (pagination.Page - 1), Limit: pagination.Size, @@ -176,7 +176,7 @@ func DeleteTaskById(c *gin.Context) { } // delete in db - if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { + if err := mongo3.RunTransaction(func(context mongo2.SessionContext) (err error) { // delete task _, err = service.NewModelService[models.Task]().GetById(id) if err != nil { @@ -223,7 +223,7 @@ func DeleteList(c *gin.Context) { return } - if err := mongo.RunTransaction(func(context mongo2.SessionContext) error { + if err := mongo3.RunTransaction(func(context mongo2.SessionContext) error { // delete tasks if err := service.NewModelService[models.Task]().DeleteMany(bson.M{ "_id": bson.M{ diff --git a/core/controllers/user.go b/core/controllers/user.go index 70982840..663886e0 100644 --- a/core/controllers/user.go +++ b/core/controllers/user.go @@ -3,12 +3,12 @@ package controllers import ( "errors" "fmt" + "github.com/crawlab-team/crawlab/core/mongo" "regexp" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/controllers/utils_filter.go b/core/controllers/utils_filter.go index 08e00971..7bc5e168 100644 --- a/core/controllers/utils_filter.go +++ b/core/controllers/utils_filter.go @@ -5,8 +5,8 @@ import ( errors2 "errors" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -125,13 +125,13 @@ func MustGetFilterAll(c *gin.Context) (res bool) { return res } -func getResultListQuery(c *gin.Context) (q generic.ListQuery) { +func getResultListQuery(c *gin.Context) (q mongo.ListQuery) { f, err := GetFilter(c) if err != nil { return q } for _, cond := range f.Conditions { - q = append(q, generic.ListQueryCondition{ + q = append(q, mongo.ListQueryCondition{ Key: cond.Key, Op: cond.Op, Value: utils.NormalizeObjectId(cond.Value), diff --git a/core/export/csv_service.go b/core/export/csv_service.go index 46175774..aac21876 100644 --- a/core/export/csv_service.go +++ b/core/export/csv_service.go @@ -9,8 +9,8 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" "github.com/hashicorp/go-uuid" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/export/csv_service_test.go b/core/export/csv_service_test.go index 8e4dffaa..ccb74592 100644 --- a/core/export/csv_service_test.go +++ b/core/export/csv_service_test.go @@ -4,7 +4,7 @@ import ( "encoding/csv" "fmt" "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/db/mongo" + "github.com/crawlab-team/crawlab/core/mongo" "github.com/stretchr/testify/require" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/export/json_service.go b/core/export/json_service.go index 1b0dbe37..0929d9e2 100644 --- a/core/export/json_service.go +++ b/core/export/json_service.go @@ -8,8 +8,8 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" "github.com/hashicorp/go-uuid" mongo2 "go.mongodb.org/mongo-driver/mongo" "os" diff --git a/core/go.mod b/core/go.mod index 68751cd6..3992b840 100644 --- a/core/go.mod +++ b/core/go.mod @@ -3,7 +3,6 @@ module github.com/crawlab-team/crawlab/core go 1.22.9 replace ( - github.com/crawlab-team/crawlab/db => ../db github.com/crawlab-team/crawlab/grpc => ../grpc github.com/crawlab-team/crawlab/trace => ../trace github.com/crawlab-team/crawlab/vcs => ../vcs @@ -14,7 +13,6 @@ require ( github.com/ReneKroon/ttlcache v1.7.0 github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.3.0 - github.com/crawlab-team/crawlab/db v0.0.0 github.com/crawlab-team/crawlab/grpc v0.0.0 github.com/crawlab-team/crawlab/trace v0.0.0 github.com/crawlab-team/crawlab/vcs v0.0.0 diff --git a/core/grpc/server/dependency_service_server.go b/core/grpc/server/dependency_service_server.go index c164911b..243eedeb 100644 --- a/core/grpc/server/dependency_service_server.go +++ b/core/grpc/server/dependency_service_server.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/cenkalti/backoff/v4" "github.com/crawlab-team/crawlab/core/interfaces" + mongo2 "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/utils" "io" "sync" @@ -14,7 +15,6 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" - mongo2 "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/grpc" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/grpc/server/model_base_service_server.go b/core/grpc/server/model_base_service_server.go index e82b4786..2f9d0657 100644 --- a/core/grpc/server/model_base_service_server.go +++ b/core/grpc/server/model_base_service_server.go @@ -3,6 +3,7 @@ package server import ( "context" "encoding/json" + "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/utils" "go.mongodb.org/mongo-driver/mongo/options" "reflect" @@ -10,7 +11,6 @@ import ( "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/grpc" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/grpc/server/task_service_server.go b/core/grpc/server/task_service_server.go index 29ffb43f..27d57673 100644 --- a/core/grpc/server/task_service_server.go +++ b/core/grpc/server/task_service_server.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + mongo3 "github.com/crawlab-team/crawlab/core/mongo" "io" "strings" "sync" @@ -17,7 +18,6 @@ import ( "github.com/crawlab-team/crawlab/core/notification" "github.com/crawlab-team/crawlab/core/task/stats" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/grpc" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -147,14 +147,14 @@ func (svr TaskServiceServer) FetchTask(ctx context.Context, request *grpc.TaskSe return nil, err } var tid primitive.ObjectID - opts := &mongo.FindOptions{ + opts := &mongo3.FindOptions{ Sort: bson.D{ {"priority", 1}, {"_id", 1}, }, Limit: 1, } - if err := mongo.RunTransactionWithContext(ctx, func(sc mongo2.SessionContext) (err error) { + if err := mongo3.RunTransactionWithContext(ctx, func(sc mongo2.SessionContext) (err error) { // fetch task for the given node t, err := service.NewModelService[models.Task]().GetOne(bson.M{ "node_id": n.Id, diff --git a/core/interfaces/result_service.go b/core/interfaces/result_service.go deleted file mode 100644 index 739bc6df..00000000 --- a/core/interfaces/result_service.go +++ /dev/null @@ -1,15 +0,0 @@ -package interfaces - -import ( - "github.com/crawlab-team/crawlab/db/generic" - "time" -) - -type ResultService interface { - Insert(records ...interface{}) (err error) - List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) - Count(query generic.ListQuery) (n int, err error) - Index(fields []string) - SetTime(t time.Time) - GetTime() (t time.Time) -} diff --git a/core/interfaces/result_service_registry.go b/core/interfaces/result_service_registry.go deleted file mode 100644 index e989ffe8..00000000 --- a/core/interfaces/result_service_registry.go +++ /dev/null @@ -1,11 +0,0 @@ -package interfaces - -import "go.mongodb.org/mongo-driver/bson/primitive" - -type ResultServiceRegistry interface { - Register(key string, fn ResultServiceRegistryFn) - Unregister(key string) - Get(key string) (fn ResultServiceRegistryFn) -} - -type ResultServiceRegistryFn func(colId primitive.ObjectID, dsId primitive.ObjectID) (ResultService, error) diff --git a/core/models/client/model_service.go b/core/models/client/model_service.go index acd57877..df82de19 100644 --- a/core/models/client/model_service.go +++ b/core/models/client/model_service.go @@ -4,8 +4,8 @@ import ( "encoding/json" "github.com/crawlab-team/crawlab/core/grpc/client" "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/mongo" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" - "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/grpc" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/models/client/model_service_test.go b/core/models/client/model_service_test.go index 0c7f15ff..004d2ad2 100644 --- a/core/models/client/model_service_test.go +++ b/core/models/client/model_service_test.go @@ -4,6 +4,7 @@ import ( "context" client2 "github.com/crawlab-team/crawlab/core/grpc/client" "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/mongo" "testing" "time" @@ -11,7 +12,6 @@ import ( "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/models/client" "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/db/mongo" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/core/models/common/index_utils.go b/core/models/common/index_utils.go index f1a96e65..92086552 100644 --- a/core/models/common/index_utils.go +++ b/core/models/common/index_utils.go @@ -3,8 +3,8 @@ package common import ( "encoding/json" "fmt" + "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" "go.mongodb.org/mongo-driver/bson" mongo2 "go.mongodb.org/mongo-driver/mongo" ) diff --git a/core/models/common/index_utils_test.go b/core/models/common/index_utils_test.go index db0f9109..13e45ad7 100644 --- a/core/models/common/index_utils_test.go +++ b/core/models/common/index_utils_test.go @@ -2,9 +2,9 @@ package common import ( "fmt" + "github.com/crawlab-team/crawlab/core/mongo" "testing" - "github.com/crawlab-team/crawlab/db/mongo" "github.com/stretchr/testify/assert" "go.mongodb.org/mongo-driver/bson" mongo2 "go.mongodb.org/mongo-driver/mongo" diff --git a/core/models/common/init_index.go b/core/models/common/init_index.go index 4eee5e41..ac4ea63d 100644 --- a/core/models/common/init_index.go +++ b/core/models/common/init_index.go @@ -3,7 +3,7 @@ package common import ( "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/db/mongo" + "github.com/crawlab-team/crawlab/core/mongo" "go.mongodb.org/mongo-driver/bson" mongo2 "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" diff --git a/core/models/service/base_service.go b/core/models/service/base_service.go index 00482e01..dee6e5e2 100644 --- a/core/models/service/base_service.go +++ b/core/models/service/base_service.go @@ -3,13 +3,13 @@ package service import ( "context" "fmt" + "github.com/crawlab-team/crawlab/core/mongo" "reflect" "sync" "go.mongodb.org/mongo-driver/mongo/options" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/db/mongo" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" ) diff --git a/core/models/service/base_service_test.go b/core/models/service/base_service_test.go index d354f6ab..b9743370 100644 --- a/core/models/service/base_service_test.go +++ b/core/models/service/base_service_test.go @@ -4,11 +4,11 @@ import ( "context" "github.com/apex/log" "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/mongo" "testing" "time" "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/db/mongo" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/db/mongo/client.go b/core/mongo/client.go similarity index 79% rename from db/mongo/client.go rename to core/mongo/client.go index f49a443a..07fbc302 100644 --- a/db/mongo/client.go +++ b/core/mongo/client.go @@ -4,13 +4,12 @@ import ( "context" "encoding/json" "fmt" - "github.com/apex/log" "github.com/cenkalti/backoff/v4" - "github.com/crawlab-team/crawlab/trace" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "sync" + "time" ) var AppName = "crawlab-db" @@ -70,7 +69,7 @@ func GetMongoClient(opts ...ClientOption) (c *mongo.Client, err error) { // client options key json string _optsKeyBytes, err := json.Marshal(_opts) if err != nil { - return nil, trace.TraceError(err) + return nil, err } _optsKey := string(_optsKeyBytes) @@ -81,8 +80,9 @@ func GetMongoClient(opts ...ClientOption) (c *mongo.Client, err error) { } // create new mongo client - c, err = newMongoClient(_opts.Context, _opts) + c, err = newMongoClient(_opts) if err != nil { + logger.Errorf("create mongo client error: %v", err) return nil, err } @@ -94,7 +94,7 @@ func GetMongoClient(opts ...ClientOption) (c *mongo.Client, err error) { return c, nil } -func newMongoClient(ctx context.Context, _opts *ClientOptions) (c *mongo.Client, err error) { +func newMongoClient(_opts *ClientOptions) (c *mongo.Client, err error) { // mongo client options mongoOpts := &options.ClientOptions{ AppName: &AppName, @@ -105,9 +105,8 @@ func newMongoClient(ctx context.Context, _opts *ClientOptions) (c *mongo.Client, mongoOpts.ApplyURI(_opts.Uri) } else { // uri is unset - - // username and password are set if _opts.Username != "" && _opts.Password != "" { + // username and password are set mongoOpts.SetAuth(options.Credential{ AuthMechanism: _opts.AuthMechanism, AuthMechanismProperties: _opts.AuthMechanismProperties, @@ -128,20 +127,27 @@ func newMongoClient(ctx context.Context, _opts *ClientOptions) (c *mongo.Client, } // attempt to connect with retry - bp := backoff.NewExponentialBackOff() - err = backoff.Retry(func() error { - errMsg := fmt.Sprintf("waiting for connect mongo database, after %f seconds try again.", bp.NextBackOff().Seconds()) - c, err = mongo.NewClient(mongoOpts) + op := func() error { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + logger.Infof("connecting to mongo") + c, err = mongo.Connect(ctx, mongoOpts) if err != nil { - log.WithError(err).Warnf(errMsg) - return err - } - if err := c.Connect(ctx); err != nil { - log.WithError(err).Warnf(errMsg) return err } + logger.Infof("connected to mongo") return nil - }, bp) + } + b := backoff.NewExponentialBackOff() + n := func(err error, duration time.Duration) { + logger.Errorf("connect to mongo error: %v. retrying in %s", err, duration) + } + err = backoff.RetryNotify(op, b, n) + if err != nil { + logger.Errorf("connect to mongo error: %v", err) + return nil, err + } return c, nil } diff --git a/db/mongo/client_options.go b/core/mongo/client_options.go similarity index 100% rename from db/mongo/client_options.go rename to core/mongo/client_options.go diff --git a/db/mongo/col.go b/core/mongo/col.go similarity index 91% rename from db/mongo/col.go rename to core/mongo/col.go index d5288595..4d3cddb9 100644 --- a/db/mongo/col.go +++ b/core/mongo/col.go @@ -3,7 +3,6 @@ package mongo import ( "context" "errors" - "github.com/apex/log" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" @@ -53,14 +52,14 @@ type Col struct { func (col *Col) Insert(doc interface{}) (id primitive.ObjectID, err error) { res, err := col.c.InsertOne(col.ctx, doc) if err != nil { - log.Errorf("error inserting document: %v", err) + logger.Errorf("error inserting document: %v", err) return primitive.NilObjectID, err } if id, ok := res.InsertedID.(primitive.ObjectID); ok { return id, nil } err = errors.New("InsertedID is not ObjectID") - log.Errorf("error inserting document: %v", err) + logger.Errorf("error inserting document: %v", err) return primitive.NilObjectID, err } @@ -76,7 +75,7 @@ func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err er ids = append(ids, id) default: err = errors.New("InsertedID is not ObjectID") - log.Errorf("error inserting document: %v", err) + logger.Errorf("error inserting document: %v", err) return nil, err } } @@ -86,7 +85,7 @@ func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err er func (col *Col) UpdateId(id primitive.ObjectID, update interface{}) (err error) { _, err = col.c.UpdateOne(col.ctx, bson.M{"_id": id}, update) if err != nil { - log.Errorf("error updating document: %v", err) + logger.Errorf("error updating document: %v", err) return err } return nil @@ -103,7 +102,7 @@ func (col *Col) UpdateWithOptions(query bson.M, update interface{}, opts *option _, err = col.c.UpdateMany(col.ctx, query, update, opts) } if err != nil { - log.Errorf("error updating document: %v", err) + logger.Errorf("error updating document: %v", err) return err } return nil @@ -124,7 +123,7 @@ func (col *Col) ReplaceWithOptions(query bson.M, doc interface{}, opts *options. _, err = col.c.ReplaceOne(col.ctx, query, doc, opts) } if err != nil { - log.Errorf("error replacing document: %v", err) + logger.Errorf("error replacing document: %v", err) return err } return nil @@ -133,7 +132,7 @@ func (col *Col) ReplaceWithOptions(query bson.M, doc interface{}, opts *options. func (col *Col) DeleteId(id primitive.ObjectID) (err error) { _, err = col.c.DeleteOne(col.ctx, bson.M{"_id": id}) if err != nil { - log.Errorf("error deleting document: %v", err) + logger.Errorf("error deleting document: %v", err) return err } return nil @@ -150,7 +149,7 @@ func (col *Col) DeleteWithOptions(query bson.M, opts *options.DeleteOptions) (er _, err = col.c.DeleteMany(col.ctx, query, opts) } if err != nil { - log.Errorf("error deleting document: %v", err) + logger.Errorf("error deleting document: %v", err) return err } return nil @@ -233,7 +232,7 @@ func (col *Col) Aggregate(pipeline mongo.Pipeline, opts *options.AggregateOption func (col *Col) CreateIndex(indexModel mongo.IndexModel) (err error) { _, err = col.c.Indexes().CreateOne(col.ctx, indexModel) if err != nil { - log.Errorf("error creating index: %v", err) + logger.Errorf("error creating index: %v", err) return err } return nil @@ -242,7 +241,7 @@ func (col *Col) CreateIndex(indexModel mongo.IndexModel) (err error) { func (col *Col) CreateIndexes(indexModels []mongo.IndexModel) (err error) { _, err = col.c.Indexes().CreateMany(col.ctx, indexModels) if err != nil { - log.Errorf("error creating indexes: %v", err) + logger.Errorf("error creating indexes: %v", err) return err } return nil @@ -263,7 +262,7 @@ func (col *Col) DeleteIndex(name string) (err error) { if errors.As(err, &e) && e.HasErrorCode(26) { return nil } - log.Errorf("error deleting index: %v", err) + logger.Errorf("error deleting index: %v", err) return err } return nil @@ -276,7 +275,7 @@ func (col *Col) DeleteAllIndexes() (err error) { if errors.As(err, &e) && e.HasErrorCode(26) { return nil } - log.Errorf("error deleting all indexes: %v", err) + logger.Errorf("error deleting all indexes: %v", err) return err } return nil @@ -285,11 +284,11 @@ func (col *Col) DeleteAllIndexes() (err error) { func (col *Col) ListIndexes() (indexes []map[string]interface{}, err error) { cur, err := col.c.Indexes().List(col.ctx) if err != nil { - log.Errorf("error listing indexes: %v", err) + logger.Errorf("error listing indexes: %v", err) return nil, err } if err := cur.All(col.ctx, &indexes); err != nil { - log.Errorf("error listing indexes: %v", err) + logger.Errorf("error listing indexes: %v", err) return nil, err } return indexes, nil diff --git a/db/mongo/col_test.go b/core/mongo/col_test.go similarity index 100% rename from db/mongo/col_test.go rename to core/mongo/col_test.go diff --git a/db/mongo/db.go b/core/mongo/db.go similarity index 83% rename from db/mongo/db.go rename to core/mongo/db.go index 3f30d620..3cbcd084 100644 --- a/db/mongo/db.go +++ b/core/mongo/db.go @@ -1,7 +1,6 @@ package mongo import ( - "github.com/apex/log" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/mongo" ) @@ -16,7 +15,7 @@ func GetMongoDb(dbName string) *mongo.Database { c, err := GetMongoClient() if err != nil { - log.Errorf("error getting mongo client: %v", err) + logger.Errorf("error getting mongo client: %v", err) return nil } diff --git a/db/generic/list.go b/core/mongo/list.go similarity index 92% rename from db/generic/list.go rename to core/mongo/list.go index 99e30fea..09b45da1 100644 --- a/db/generic/list.go +++ b/core/mongo/list.go @@ -1,4 +1,4 @@ -package generic +package mongo type ListQueryCondition struct { Key string diff --git a/core/mongo/logger.go b/core/mongo/logger.go new file mode 100644 index 00000000..1b8c9b3c --- /dev/null +++ b/core/mongo/logger.go @@ -0,0 +1,5 @@ +package mongo + +import "github.com/crawlab-team/crawlab/core/utils" + +var logger = utils.NewLogger("Mongo") diff --git a/db/generic/op.go b/core/mongo/op.go similarity index 72% rename from db/generic/op.go rename to core/mongo/op.go index 0051d39b..42d2a890 100644 --- a/db/generic/op.go +++ b/core/mongo/op.go @@ -1,4 +1,4 @@ -package generic +package mongo type Op string diff --git a/db/mongo/result.go b/core/mongo/result.go similarity index 90% rename from db/mongo/result.go rename to core/mongo/result.go index 8cdf11b9..a34e9bd3 100644 --- a/db/mongo/result.go +++ b/core/mongo/result.go @@ -12,7 +12,6 @@ type FindResultInterface interface { GetCol() (col *Col) GetSingleResult() (res *mongo.SingleResult) GetCursor() (cur *mongo.Cursor) - GetError() (err error) } type FindResult struct { @@ -22,11 +21,6 @@ type FindResult struct { err error } -func (fr *FindResult) GetError() (err error) { - //TODO implement me - panic("implement me") -} - func (fr *FindResult) One(val interface{}) (err error) { if fr.err != nil { return fr.err diff --git a/db/generic/sort.go b/core/mongo/sort.go similarity index 92% rename from db/generic/sort.go rename to core/mongo/sort.go index 388909c2..46db82f7 100644 --- a/db/generic/sort.go +++ b/core/mongo/sort.go @@ -1,4 +1,4 @@ -package generic +package mongo type SortDirection string diff --git a/db/mongo/transaction.go b/core/mongo/transaction.go similarity index 100% rename from db/mongo/transaction.go rename to core/mongo/transaction.go diff --git a/core/mongo/utils.go b/core/mongo/utils.go new file mode 100644 index 00000000..2932c007 --- /dev/null +++ b/core/mongo/utils.go @@ -0,0 +1,36 @@ +package mongo + +import "go.mongodb.org/mongo-driver/bson" + +func GetMongoQuery(query ListQuery) (res bson.M) { + res = bson.M{} + for _, c := range query { + switch c.Op { + case OpEqual: + res[c.Key] = c.Value + default: + res[c.Key] = bson.M{ + c.Op: c.Value, + } + } + } + return res +} + +func GetMongoOpts(opts *ListOptions) (res *FindOptions) { + var sort bson.D + for _, s := range opts.Sort { + direction := 1 + if s.Direction == SortDirectionAsc { + direction = 1 + } else if s.Direction == SortDirectionDesc { + direction = -1 + } + sort = append(sort, bson.E{Key: s.Key, Value: direction}) + } + return &FindOptions{ + Skip: opts.Skip, + Limit: opts.Limit, + Sort: sort, + } +} diff --git a/core/schedule/logger.go b/core/schedule/logger.go index 0556b3c8..0eabb42d 100644 --- a/core/schedule/logger.go +++ b/core/schedule/logger.go @@ -31,6 +31,6 @@ func (l *CronLogger) getPlaceholder(n int) (s string) { func NewCronLogger() cron.Logger { return &CronLogger{ - Logger: utils.NewLogger("CronLogger"), + Logger: utils.NewLogger("Cron"), } } diff --git a/core/stats/service.go b/core/stats/service.go index 92a967cb..6f1d26ea 100644 --- a/core/stats/service.go +++ b/core/stats/service.go @@ -5,7 +5,7 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" + "github.com/crawlab-team/crawlab/core/mongo" "go.mongodb.org/mongo-driver/bson" mongo2 "go.mongodb.org/mongo-driver/mongo" ) diff --git a/core/task/stats/service.go b/core/task/stats/service.go index 5dfb87f1..d06e1c5f 100644 --- a/core/task/stats/service.go +++ b/core/task/stats/service.go @@ -7,10 +7,10 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/mongo" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/log" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "sync" diff --git a/core/utils/mongo.go b/core/utils/mongo.go deleted file mode 100644 index c223d3ef..00000000 --- a/core/utils/mongo.go +++ /dev/null @@ -1,40 +0,0 @@ -package utils - -import ( - "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" -) - -func GetMongoQuery(query generic.ListQuery) (res bson.M) { - res = bson.M{} - for _, c := range query { - switch c.Op { - case generic.OpEqual: - res[c.Key] = c.Value - default: - res[c.Key] = bson.M{ - c.Op: c.Value, - } - } - } - return res -} - -func GetMongoOpts(opts *generic.ListOptions) (res *mongo.FindOptions) { - var sort bson.D - for _, s := range opts.Sort { - direction := 1 - if s.Direction == generic.SortDirectionAsc { - direction = 1 - } else if s.Direction == generic.SortDirectionDesc { - direction = -1 - } - sort = append(sort, bson.E{Key: s.Key, Value: direction}) - } - return &mongo.FindOptions{ - Skip: opts.Skip, - Limit: opts.Limit, - Sort: sort, - } -} diff --git a/db/.editorconfig b/db/.editorconfig deleted file mode 100644 index ef3de0a1..00000000 --- a/db/.editorconfig +++ /dev/null @@ -1,9 +0,0 @@ -root = true - -[*] -charset = utf-8 -end_of_line = lf -indent_size = 4 -indent_style = tab -insert_final_newline = true -trim_trailing_whitespace = true diff --git a/db/.gitignore b/db/.gitignore deleted file mode 100644 index 4089fa24..00000000 --- a/db/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.idea -.DS_Store -tmp/ -vendor/ -go.sum \ No newline at end of file diff --git a/db/LICENSE b/db/LICENSE deleted file mode 100644 index e41311d2..00000000 --- a/db/LICENSE +++ /dev/null @@ -1,29 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2020, Crawlab Team -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/db/README.md b/db/README.md deleted file mode 100644 index 04b2346f..00000000 --- a/db/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# crawlab-db -Backend database module for Crawlab diff --git a/db/go.mod b/db/go.mod deleted file mode 100644 index 617b176e..00000000 --- a/db/go.mod +++ /dev/null @@ -1,50 +0,0 @@ -module github.com/crawlab-team/crawlab/db - -go 1.22.9 - -replace github.com/crawlab-team/crawlab/trace => ../trace - -require ( - github.com/apex/log v1.9.0 - github.com/cenkalti/backoff/v4 v4.3.0 - github.com/crawlab-team/crawlab/trace v0.0.0 - github.com/spf13/viper v1.19.0 - github.com/stretchr/testify v1.9.0 - go.mongodb.org/mongo-driver v1.15.1 -) - -require ( - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/klauspost/compress v1.17.7 // indirect - github.com/magiconair/properties v1.8.7 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect - github.com/sagikazarmark/locafero v0.4.0 // indirect - github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.6.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/subosito/gotenv v1.6.0 // indirect - github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.1.2 // indirect - github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect - github.com/ztrue/tracerr v0.4.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) diff --git a/go.work b/go.work index 0cc75df3..91441d84 100644 --- a/go.work +++ b/go.work @@ -3,7 +3,6 @@ go 1.22.9 use ( backend core - db grpc trace vcs