git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: remove db module and update imports to core/mongo

Browse Source 
- Deleted the db module, consolidating database-related functionality into the core/mongo package for better organization and maintainability.
- Updated all import paths across the codebase to replace references to the removed db module with core/mongo.
- Cleaned up unused code and dependencies, enhancing overall project clarity and reducing complexity.
- This refactor improves the structure of the codebase by centralizing database operations and simplifying module management.
This commit is contained in:
Marvin Zhang
2024-12-25 10:28:21 +08:00
parent a28ffbf66c
commit dc59599509
47 changed files with 126 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/go.mod
View File

@@ -4,7 +4,6 @@ go 1.22.9
replace (
github.com/crawlab-team/crawlab/core => ../core
github.com/crawlab-team/crawlab/db => ../db
github.com/crawlab-team/crawlab/grpc => ../grpc
github.com/crawlab-team/crawlab/trace => ../trace
github.com/crawlab-team/crawlab/vcs => ../vcs
@@ -29,7 +28,6 @@ require (
github.com/cloudflare/circl v1.3.7 // indirect
github.com/cloudwego/base64x v0.1.4 // indirect
github.com/cloudwego/iasm v0.2.0 // indirect
github.com/crawlab-team/crawlab/db v0.0.0 // indirect
github.com/crawlab-team/crawlab/grpc v0.0.0 // indirect
github.com/crawlab-team/crawlab/trace v0.0.0 // indirect
github.com/crawlab-team/crawlab/vcs v0.0.0 // indirect

2
core/controllers/base.go
View File

@@ -4,7 +4,7 @@ import (
"errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/core/mongo"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"

2
core/controllers/base_test.go
View File

@@ -8,13 +8,13 @@ import (
"github.com/crawlab-team/crawlab/core/middlewares"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/mongo"
"github.com/crawlab-team/crawlab/core/user"
"github.com/spf13/viper"
"net/http"
"net/http/httptest"
"testing"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
)

2
core/controllers/filter.go
View File

@@ -2,7 +2,7 @@ package controllers
import (
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/core/mongo"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
mongo2 "go.mongodb.org/mongo-driver/mongo"

2
core/controllers/project.go
View File

@@ -4,7 +4,7 @@ import (
"errors"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/core/mongo"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"

29
core/controllers/spider.go
View File

@@ -4,6 +4,7 @@ import (
"errors"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
mongo2 "github.com/crawlab-team/crawlab/core/mongo"
"math"
"os"
"path/filepath"
@@ -14,12 +15,10 @@ import (
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/spider/admin"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
mongo2 "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo"
)
func GetSpiderById(c *gin.Context) {
@@ -29,7 +28,7 @@ func GetSpiderById(c *gin.Context) {
return
}
s, err := service.NewModelService[models.Spider]().GetById(id)
if errors.Is(err, mongo2.ErrNoDocuments) {
if errors.Is(err, mongo.ErrNoDocuments) {
HandleErrorNotFound(c, err)
return
}
@@ -41,7 +40,7 @@ func GetSpiderById(c *gin.Context) {
// stat
s.Stat, err = service.NewModelService[models.SpiderStat]().GetById(s.Id)
if err != nil {
if !errors.Is(err, mongo2.ErrNoDocuments) {
if !errors.Is(err, mongo.ErrNoDocuments) {
HandleErrorInternalServerError(c, err)
return
}
@@ -51,7 +50,7 @@ func GetSpiderById(c *gin.Context) {
if s.ColName == "" && !s.ColId.IsZero() {
col, err := service.NewModelService[models.DataCollection]().GetById(s.ColId)
if err != nil {
if !errors.Is(err, mongo2.ErrNoDocuments) {
if !errors.Is(err, mongo.ErrNoDocuments) {
HandleErrorInternalServerError(c, err)
return
}
@@ -64,7 +63,7 @@ func GetSpiderById(c *gin.Context) {
if utils.IsPro() && !s.GitId.IsZero() {
s.Git, err = service.NewModelService[models.Git]().GetById(s.GitId)
if err != nil {
if !errors.Is(err, mongo2.ErrNoDocuments) {
if !errors.Is(err, mongo.ErrNoDocuments) {
HandleErrorInternalServerError(c, err)
return
}
@@ -100,13 +99,13 @@ func getSpiderListWithStats(c *gin.Context) {
sort := MustGetSortOption(c)
// get list
spiders, err := service.NewModelService[models.Spider]().GetMany(query, &mongo.FindOptions{
spiders, err := service.NewModelService[models.Spider]().GetMany(query, &mongo2.FindOptions{
Sort: sort,
Skip: pagination.Size * (pagination.Page - 1),
Limit: pagination.Size,
})
if err != nil {
if err.Error() != mongo2.ErrNoDocuments.Error() {
if err.Error() != mongo.ErrNoDocuments.Error() {
HandleErrorInternalServerError(c, err)
}
return
@@ -347,7 +346,7 @@ func DeleteSpiderById(c *gin.Context) {
return
}
if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) {
if err := mongo2.RunTransaction(func(context mongo.SessionContext) (err error) {
// delete spider
err = service.NewModelService[models.Spider]().DeleteById(id)
if err != nil {
@@ -448,7 +447,7 @@ func DeleteSpiderList(c *gin.Context) {
return
}
if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) {
if err := mongo2.RunTransaction(func(context mongo.SessionContext) (err error) {
// delete spiders
if err := service.NewModelService[models.Spider]().DeleteMany(bson.M{
"_id": bson.M{
@@ -683,11 +682,11 @@ func GetSpiderResults(c *gin.Context) {
pagination := MustGetPagination(c)
query := getResultListQuery(c)
col := mongo.GetMongoCol(s.ColName)
col := mongo2.GetMongoCol(s.ColName)
var results []bson.M
err = col.Find(utils.GetMongoQuery(query), utils.GetMongoOpts(&generic.ListOptions{
Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}},
err = col.Find(mongo2.GetMongoQuery(query), mongo2.GetMongoOpts(&mongo2.ListOptions{
Sort: []mongo2.ListSort{{"_id", mongo2.SortDirectionDesc}},
Skip: pagination.Size * (pagination.Page - 1),
Limit: pagination.Size,
})).All(&results)
@@ -696,7 +695,7 @@ func GetSpiderResults(c *gin.Context) {
return
}
total, err := mongo.GetMongoCol(s.ColName).Count(utils.GetMongoQuery(query))
total, err := mongo2.GetMongoCol(s.ColName).Count(mongo2.GetMongoQuery(query))
if err != nil {
HandleErrorInternalServerError(c, err)
return

8
core/controllers/task.go
View File

@@ -6,11 +6,11 @@ import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
mongo3 "github.com/crawlab-team/crawlab/core/mongo"
"github.com/crawlab-team/crawlab/core/spider/admin"
"github.com/crawlab-team/crawlab/core/task/log"
"github.com/crawlab-team/crawlab/core/task/scheduler"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
@@ -80,7 +80,7 @@ func GetTaskList(c *gin.Context) {
sort := MustGetSortOption(c)
// get tasks
tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo.FindOptions{
tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo3.FindOptions{
Sort: sort,
Skip: pagination.Size * (pagination.Page - 1),
Limit: pagination.Size,
@@ -176,7 +176,7 @@ func DeleteTaskById(c *gin.Context) {
}
// delete in db
if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) {
if err := mongo3.RunTransaction(func(context mongo2.SessionContext) (err error) {
// delete task
_, err = service.NewModelService[models.Task]().GetById(id)
if err != nil {
@@ -223,7 +223,7 @@ func DeleteList(c *gin.Context) {
return
}
if err := mongo.RunTransaction(func(context mongo2.SessionContext) error {
if err := mongo3.RunTransaction(func(context mongo2.SessionContext) error {
// delete tasks
if err := service.NewModelService[models.Task]().DeleteMany(bson.M{
"_id": bson.M{

2
core/controllers/user.go
View File

@@ -3,12 +3,12 @@ package controllers
import (
"errors"
"fmt"
"github.com/crawlab-team/crawlab/core/mongo"
"regexp"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"

6
core/controllers/utils_filter.go
View File

@@ -5,8 +5,8 @@ import (
errors2 "errors"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/mongo"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
@@ -125,13 +125,13 @@ func MustGetFilterAll(c *gin.Context) (res bool) {
return res
}
func getResultListQuery(c *gin.Context) (q generic.ListQuery) {
func getResultListQuery(c *gin.Context) (q mongo.ListQuery) {
f, err := GetFilter(c)
if err != nil {
return q
}
for _, cond := range f.Conditions {
q = append(q, generic.ListQueryCondition{
q = append(q, mongo.ListQueryCondition{
Key: cond.Key,
Op: cond.Op,
Value: utils.NormalizeObjectId(cond.Value),

2
core/export/csv_service.go
View File

@@ -9,8 +9,8 @@ import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/mongo"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/hashicorp/go-uuid"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"

2
core/export/csv_service_test.go
View File

@@ -4,7 +4,7 @@ import (
"encoding/csv"
"fmt"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/core/mongo"
"github.com/stretchr/testify/require"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"

2
core/export/json_service.go
View File

@@ -8,8 +8,8 @@ import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/mongo"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/hashicorp/go-uuid"
mongo2 "go.mongodb.org/mongo-driver/mongo"
"os"

2
core/go.mod
View File

@@ -3,7 +3,6 @@ module github.com/crawlab-team/crawlab/core
go 1.22.9
replace (
github.com/crawlab-team/crawlab/db => ../db
github.com/crawlab-team/crawlab/grpc => ../grpc
github.com/crawlab-team/crawlab/trace => ../trace
github.com/crawlab-team/crawlab/vcs => ../vcs
@@ -14,7 +13,6 @@ require (
github.com/ReneKroon/ttlcache v1.7.0
github.com/apex/log v1.9.0
github.com/cenkalti/backoff/v4 v4.3.0
github.com/crawlab-team/crawlab/db v0.0.0
github.com/crawlab-team/crawlab/grpc v0.0.0
github.com/crawlab-team/crawlab/trace v0.0.0
github.com/crawlab-team/crawlab/vcs v0.0.0

2
core/grpc/server/dependency_service_server.go
View File

@@ -6,6 +6,7 @@ import (
"fmt"
"github.com/cenkalti/backoff/v4"
"github.com/crawlab-team/crawlab/core/interfaces"
mongo2 "github.com/crawlab-team/crawlab/core/mongo"
"github.com/crawlab-team/crawlab/core/utils"
"io"
"sync"
@@ -14,7 +15,6 @@ import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
mongo2 "github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/grpc"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"

2
core/grpc/server/model_base_service_server.go
View File

@@ -3,6 +3,7 @@ package server
import (
"context"
"encoding/json"
"github.com/crawlab-team/crawlab/core/mongo"
"github.com/crawlab-team/crawlab/core/utils"
"go.mongodb.org/mongo-driver/mongo/options"
"reflect"
@@ -10,7 +11,6 @@ import (
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/grpc"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"

6
core/grpc/server/task_service_server.go
View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
mongo3 "github.com/crawlab-team/crawlab/core/mongo"
"io"
"strings"
"sync"
@@ -17,7 +18,6 @@ import (
"github.com/crawlab-team/crawlab/core/notification"
"github.com/crawlab-team/crawlab/core/task/stats"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/grpc"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
@@ -147,14 +147,14 @@ func (svr TaskServiceServer) FetchTask(ctx context.Context, request *grpc.TaskSe
return nil, err
}
var tid primitive.ObjectID
opts := &mongo.FindOptions{
opts := &mongo3.FindOptions{
Sort: bson.D{
{"priority", 1},
{"_id", 1},
},
Limit: 1,
}
if err := mongo.RunTransactionWithContext(ctx, func(sc mongo2.SessionContext) (err error) {
if err := mongo3.RunTransactionWithContext(ctx, func(sc mongo2.SessionContext) (err error) {
// fetch task for the given node
t, err := service.NewModelService[models.Task]().GetOne(bson.M{
"node_id": n.Id,

15
core/interfaces/result_service.go
View File

@@ -1,15 +0,0 @@
package interfaces
import (
"github.com/crawlab-team/crawlab/db/generic"
"time"
)
type ResultService interface {
Insert(records ...interface{}) (err error)
List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error)
Count(query generic.ListQuery) (n int, err error)
Index(fields []string)
SetTime(t time.Time)
GetTime() (t time.Time)
}

11
core/interfaces/result_service_registry.go
View File

@@ -1,11 +0,0 @@
package interfaces
import "go.mongodb.org/mongo-driver/bson/primitive"
type ResultServiceRegistry interface {
Register(key string, fn ResultServiceRegistryFn)
Unregister(key string)
Get(key string) (fn ResultServiceRegistryFn)
}
type ResultServiceRegistryFn func(colId primitive.ObjectID, dsId primitive.ObjectID) (ResultService, error)

2
core/models/client/model_service.go
View File

@@ -4,8 +4,8 @@ import (
"encoding/json"
"github.com/crawlab-team/crawlab/core/grpc/client"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/mongo"
nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/grpc"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"

2
core/models/client/model_service_test.go
View File

@@ -4,6 +4,7 @@ import (
"context"
client2 "github.com/crawlab-team/crawlab/core/grpc/client"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/mongo"
"testing"
"time"
@@ -11,7 +12,6 @@ import (
"github.com/crawlab-team/crawlab/core/grpc/server"
"github.com/crawlab-team/crawlab/core/models/client"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

2
core/models/common/index_utils.go
View File

@@ -3,8 +3,8 @@ package common
import (
"encoding/json"
"fmt"
"github.com/crawlab-team/crawlab/core/mongo"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
mongo2 "go.mongodb.org/mongo-driver/mongo"
)

2
core/models/common/index_utils_test.go
View File

@@ -2,9 +2,9 @@ package common
import (
"fmt"
"github.com/crawlab-team/crawlab/core/mongo"
"testing"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/stretchr/testify/assert"
"go.mongodb.org/mongo-driver/bson"
mongo2 "go.mongodb.org/mongo-driver/mongo"

2
core/models/common/init_index.go
View File

@@ -3,7 +3,7 @@ package common
import (
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/core/mongo"
"go.mongodb.org/mongo-driver/bson"
mongo2 "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"

2
core/models/service/base_service.go
View File

@@ -3,13 +3,13 @@ package service
import (
"context"
"fmt"
"github.com/crawlab-team/crawlab/core/mongo"
"reflect"
"sync"
"go.mongodb.org/mongo-driver/mongo/options"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)

2
core/models/service/base_service_test.go
View File

@@ -4,11 +4,11 @@ import (
"context"
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/mongo"
"testing"
"time"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

40
db/mongo/client.go → core/mongo/client.go
View File

@@ -4,13 +4,12 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/apex/log"
"github.com/cenkalti/backoff/v4"
"github.com/crawlab-team/crawlab/trace"
"github.com/spf13/viper"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"sync"
"time"
)
var AppName = "crawlab-db"
@@ -70,7 +69,7 @@ func GetMongoClient(opts ...ClientOption) (c *mongo.Client, err error) {
// client options key json string
_optsKeyBytes, err := json.Marshal(_opts)
if err != nil {
return nil, trace.TraceError(err)
return nil, err
}
_optsKey := string(_optsKeyBytes)
@@ -81,8 +80,9 @@ func GetMongoClient(opts ...ClientOption) (c *mongo.Client, err error) {
}
// create new mongo client
c, err = newMongoClient(_opts.Context, _opts)
c, err = newMongoClient(_opts)
if err != nil {
logger.Errorf("create mongo client error: %v", err)
return nil, err
}
@@ -94,7 +94,7 @@ func GetMongoClient(opts ...ClientOption) (c *mongo.Client, err error) {
return c, nil
}
func newMongoClient(ctx context.Context, _opts *ClientOptions) (c *mongo.Client, err error) {
func newMongoClient(_opts *ClientOptions) (c *mongo.Client, err error) {
// mongo client options
mongoOpts := &options.ClientOptions{
AppName: &AppName,
@@ -105,9 +105,8 @@ func newMongoClient(ctx context.Context, _opts *ClientOptions) (c *mongo.Client,
mongoOpts.ApplyURI(_opts.Uri)
} else {
// uri is unset
// username and password are set
if _opts.Username != "" && _opts.Password != "" {
// username and password are set
mongoOpts.SetAuth(options.Credential{
AuthMechanism: _opts.AuthMechanism,
AuthMechanismProperties: _opts.AuthMechanismProperties,
@@ -128,20 +127,27 @@ func newMongoClient(ctx context.Context, _opts *ClientOptions) (c *mongo.Client,
}
// attempt to connect with retry
bp := backoff.NewExponentialBackOff()
err = backoff.Retry(func() error {
errMsg := fmt.Sprintf("waiting for connect mongo database, after %f seconds try again.", bp.NextBackOff().Seconds())
c, err = mongo.NewClient(mongoOpts)
op := func() error {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
logger.Infof("connecting to mongo")
c, err = mongo.Connect(ctx, mongoOpts)
if err != nil {
log.WithError(err).Warnf(errMsg)
return err
}
if err := c.Connect(ctx); err != nil {
log.WithError(err).Warnf(errMsg)
return err
}
logger.Infof("connected to mongo")
return nil
}, bp)
}
b := backoff.NewExponentialBackOff()
n := func(err error, duration time.Duration) {
logger.Errorf("connect to mongo error: %v. retrying in %s", err, duration)
}
err = backoff.RetryNotify(op, b, n)
if err != nil {
logger.Errorf("connect to mongo error: %v", err)
return nil, err
}
return c, nil
}

0
db/mongo/client_options.go → core/mongo/client_options.go
View File

29
db/mongo/col.go → core/mongo/col.go
View File

@@ -3,7 +3,6 @@ package mongo
import (
"context"
"errors"
"github.com/apex/log"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
@@ -53,14 +52,14 @@ type Col struct {
func (col *Col) Insert(doc interface{}) (id primitive.ObjectID, err error) {
res, err := col.c.InsertOne(col.ctx, doc)
if err != nil {
log.Errorf("error inserting document: %v", err)
logger.Errorf("error inserting document: %v", err)
return primitive.NilObjectID, err
}
if id, ok := res.InsertedID.(primitive.ObjectID); ok {
return id, nil
}
err = errors.New("InsertedID is not ObjectID")
log.Errorf("error inserting document: %v", err)
logger.Errorf("error inserting document: %v", err)
return primitive.NilObjectID, err
}
@@ -76,7 +75,7 @@ func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err er
ids = append(ids, id)
default:
err = errors.New("InsertedID is not ObjectID")
log.Errorf("error inserting document: %v", err)
logger.Errorf("error inserting document: %v", err)
return nil, err
}
}
@@ -86,7 +85,7 @@ func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err er
func (col *Col) UpdateId(id primitive.ObjectID, update interface{}) (err error) {
_, err = col.c.UpdateOne(col.ctx, bson.M{"_id": id}, update)
if err != nil {
log.Errorf("error updating document: %v", err)
logger.Errorf("error updating document: %v", err)
return err
}
return nil
@@ -103,7 +102,7 @@ func (col *Col) UpdateWithOptions(query bson.M, update interface{}, opts *option
_, err = col.c.UpdateMany(col.ctx, query, update, opts)
}
if err != nil {
log.Errorf("error updating document: %v", err)
logger.Errorf("error updating document: %v", err)
return err
}
return nil
@@ -124,7 +123,7 @@ func (col *Col) ReplaceWithOptions(query bson.M, doc interface{}, opts *options.
_, err = col.c.ReplaceOne(col.ctx, query, doc, opts)
}
if err != nil {
log.Errorf("error replacing document: %v", err)
logger.Errorf("error replacing document: %v", err)
return err
}
return nil
@@ -133,7 +132,7 @@ func (col *Col) ReplaceWithOptions(query bson.M, doc interface{}, opts *options.
func (col *Col) DeleteId(id primitive.ObjectID) (err error) {
_, err = col.c.DeleteOne(col.ctx, bson.M{"_id": id})
if err != nil {
log.Errorf("error deleting document: %v", err)
logger.Errorf("error deleting document: %v", err)
return err
}
return nil
@@ -150,7 +149,7 @@ func (col *Col) DeleteWithOptions(query bson.M, opts *options.DeleteOptions) (er
_, err = col.c.DeleteMany(col.ctx, query, opts)
}
if err != nil {
log.Errorf("error deleting document: %v", err)
logger.Errorf("error deleting document: %v", err)
return err
}
return nil
@@ -233,7 +232,7 @@ func (col *Col) Aggregate(pipeline mongo.Pipeline, opts *options.AggregateOption
func (col *Col) CreateIndex(indexModel mongo.IndexModel) (err error) {
_, err = col.c.Indexes().CreateOne(col.ctx, indexModel)
if err != nil {
log.Errorf("error creating index: %v", err)
logger.Errorf("error creating index: %v", err)
return err
}
return nil
@@ -242,7 +241,7 @@ func (col *Col) CreateIndex(indexModel mongo.IndexModel) (err error) {
func (col *Col) CreateIndexes(indexModels []mongo.IndexModel) (err error) {
_, err = col.c.Indexes().CreateMany(col.ctx, indexModels)
if err != nil {
log.Errorf("error creating indexes: %v", err)
logger.Errorf("error creating indexes: %v", err)
return err
}
return nil
@@ -263,7 +262,7 @@ func (col *Col) DeleteIndex(name string) (err error) {
if errors.As(err, &e) && e.HasErrorCode(26) {
return nil
}
log.Errorf("error deleting index: %v", err)
logger.Errorf("error deleting index: %v", err)
return err
}
return nil
@@ -276,7 +275,7 @@ func (col *Col) DeleteAllIndexes() (err error) {
if errors.As(err, &e) && e.HasErrorCode(26) {
return nil
}
log.Errorf("error deleting all indexes: %v", err)
logger.Errorf("error deleting all indexes: %v", err)
return err
}
return nil
@@ -285,11 +284,11 @@ func (col *Col) DeleteAllIndexes() (err error) {
func (col *Col) ListIndexes() (indexes []map[string]interface{}, err error) {
cur, err := col.c.Indexes().List(col.ctx)
if err != nil {
log.Errorf("error listing indexes: %v", err)
logger.Errorf("error listing indexes: %v", err)
return nil, err
}
if err := cur.All(col.ctx, &indexes); err != nil {
log.Errorf("error listing indexes: %v", err)
logger.Errorf("error listing indexes: %v", err)
return nil, err
}
return indexes, nil

0
db/mongo/col_test.go → core/mongo/col_test.go
View File

3
db/mongo/db.go → core/mongo/db.go
View File

@@ -1,7 +1,6 @@
package mongo
import (
"github.com/apex/log"
"github.com/spf13/viper"
"go.mongodb.org/mongo-driver/mongo"
)
@@ -16,7 +15,7 @@ func GetMongoDb(dbName string) *mongo.Database {
c, err := GetMongoClient()
if err != nil {
log.Errorf("error getting mongo client: %v", err)
logger.Errorf("error getting mongo client: %v", err)
return nil
}

2
db/generic/list.go → core/mongo/list.go
View File

@@ -1,4 +1,4 @@
package generic
package mongo
type ListQueryCondition struct {
Key string

5
core/mongo/logger.go Normal file
View File

@@ -0,0 +1,5 @@
package mongo
import "github.com/crawlab-team/crawlab/core/utils"
var logger = utils.NewLogger("Mongo")

2
db/generic/op.go → core/mongo/op.go
View File

@@ -1,4 +1,4 @@
package generic
package mongo
type Op string

6
db/mongo/result.go → core/mongo/result.go
View File

@@ -12,7 +12,6 @@ type FindResultInterface interface {
GetCol() (col *Col)
GetSingleResult() (res *mongo.SingleResult)
GetCursor() (cur *mongo.Cursor)
GetError() (err error)
}
type FindResult struct {
@@ -22,11 +21,6 @@ type FindResult struct {
err error
}
func (fr *FindResult) GetError() (err error) {
//TODO implement me
panic("implement me")
}
func (fr *FindResult) One(val interface{}) (err error) {
if fr.err != nil {
return fr.err

2
db/generic/sort.go → core/mongo/sort.go
View File

@@ -1,4 +1,4 @@
package generic
package mongo
type SortDirection string

0
db/mongo/transaction.go → core/mongo/transaction.go
View File

36
core/mongo/utils.go Normal file
View File

@@ -0,0 +1,36 @@
package mongo
import "go.mongodb.org/mongo-driver/bson"
func GetMongoQuery(query ListQuery) (res bson.M) {
res = bson.M{}
for _, c := range query {
switch c.Op {
case OpEqual:
res[c.Key] = c.Value
default:
res[c.Key] = bson.M{
c.Op: c.Value,
}
}
}
return res
}
func GetMongoOpts(opts *ListOptions) (res *FindOptions) {
var sort bson.D
for _, s := range opts.Sort {
direction := 1
if s.Direction == SortDirectionAsc {
direction = 1
} else if s.Direction == SortDirectionDesc {
direction = -1
}
sort = append(sort, bson.E{Key: s.Key, Value: direction})
}
return &FindOptions{
Skip: opts.Skip,
Limit: opts.Limit,
Sort: sort,
}
}

2
core/schedule/logger.go
View File

@@ -31,6 +31,6 @@ func (l *CronLogger) getPlaceholder(n int) (s string) {
func NewCronLogger() cron.Logger {
return &CronLogger{
Logger: utils.NewLogger("CronLogger"),
Logger: utils.NewLogger("Cron"),
}
}

2
core/stats/service.go
View File

@@ -5,7 +5,7 @@ import (
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/core/mongo"
"go.mongodb.org/mongo-driver/bson"
mongo2 "go.mongodb.org/mongo-driver/mongo"
)

2
core/task/stats/service.go
View File

@@ -7,10 +7,10 @@ import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/mongo"
nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
"github.com/crawlab-team/crawlab/core/task/log"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"sync"

40
core/utils/mongo.go
View File

@@ -1,40 +0,0 @@
package utils
import (
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
)
func GetMongoQuery(query generic.ListQuery) (res bson.M) {
res = bson.M{}
for _, c := range query {
switch c.Op {
case generic.OpEqual:
res[c.Key] = c.Value
default:
res[c.Key] = bson.M{
c.Op: c.Value,
}
}
}
return res
}
func GetMongoOpts(opts *generic.ListOptions) (res *mongo.FindOptions) {
var sort bson.D
for _, s := range opts.Sort {
direction := 1
if s.Direction == generic.SortDirectionAsc {
direction = 1
} else if s.Direction == generic.SortDirectionDesc {
direction = -1
}
sort = append(sort, bson.E{Key: s.Key, Value: direction})
}
return &mongo.FindOptions{
Skip: opts.Skip,
Limit: opts.Limit,
Sort: sort,
}
}

9
db/.editorconfig
View File

@@ -1,9 +0,0 @@
root = true
[*]
charset = utf-8
end_of_line = lf
indent_size = 4
indent_style = tab
insert_final_newline = true
trim_trailing_whitespace = true

5
db/.gitignore vendored
View File

@@ -1,5 +0,0 @@
.idea
.DS_Store
tmp/
vendor/
go.sum

29
db/LICENSE
View File

@@ -1,29 +0,0 @@
BSD 3-Clause License
Copyright (c) 2020, Crawlab Team
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

2
db/README.md
View File

@@ -1,2 +0,0 @@
# crawlab-db
Backend database module for Crawlab

50
db/go.mod
View File

@@ -1,50 +0,0 @@
module github.com/crawlab-team/crawlab/db
go 1.22.9
replace github.com/crawlab-team/crawlab/trace => ../trace
require (
github.com/apex/log v1.9.0
github.com/cenkalti/backoff/v4 v4.3.0
github.com/crawlab-team/crawlab/trace v0.0.0
github.com/spf13/viper v1.19.0
github.com/stretchr/testify v1.9.0
go.mongodb.org/mongo-driver v1.15.1
)
require (
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/klauspost/compress v1.17.7 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/rogpeppe/go-internal v1.11.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
github.com/ztrue/tracerr v0.4.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/text v0.21.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

1
go.work
View File

@@ -3,7 +3,6 @@ go 1.22.9
use (
backend
core
db
grpc
trace
vcs