diff --git a/core/controllers/base_v2.go b/core/controllers/base_v2.go index be6eebff..82130a3c 100644 --- a/core/controllers/base_v2.go +++ b/core/controllers/base_v2.go @@ -11,6 +11,12 @@ import ( mongo2 "go.mongodb.org/mongo-driver/mongo" ) +type Action struct { + Method string + Path string + HandlerFunc gin.HandlerFunc +} + type BaseControllerV2[T any] struct { modelSvc *service.ModelServiceV2[T] actions []Action diff --git a/core/controllers/base_v2_test.go b/core/controllers/base_v2_test.go index 3e0351ef..5259ca9a 100644 --- a/core/controllers/base_v2_test.go +++ b/core/controllers/base_v2_test.go @@ -6,7 +6,7 @@ import ( "encoding/json" "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/user" "github.com/spf13/viper" @@ -24,7 +24,7 @@ func init() { } // TestModel is a simple struct to be used as a model in tests -type TestModel models.TestModel +type TestModel models.TestModelV2 var TestToken string diff --git a/core/controllers/spider_v2_test.go b/core/controllers/spider_v2_test.go index 69c50f28..a744ba85 100644 --- a/core/controllers/spider_v2_test.go +++ b/core/controllers/spider_v2_test.go @@ -5,7 +5,7 @@ import ( "encoding/json" "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" diff --git a/core/ds/service_v2.go b/core/ds/service_v2.go index 12093949..e96a9645 100644 --- a/core/ds/service_v2.go +++ b/core/ds/service_v2.go @@ -4,7 +4,7 @@ import ( "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" constants2 "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/result" "github.com/crawlab-team/crawlab/core/utils" diff --git a/core/grpc/server/dependencies_server_v2.go b/core/grpc/server/dependencies_server_v2.go index f7ede5c0..32c39760 100644 --- a/core/grpc/server/dependencies_server_v2.go +++ b/core/grpc/server/dependencies_server_v2.go @@ -4,7 +4,7 @@ import ( "context" "errors" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" mongo2 "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/grpc" @@ -40,12 +40,12 @@ func (svr DependenciesServerV2) Connect(req *grpc.DependenciesServiceV2ConnectRe } func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.DependenciesServiceV2SyncRequest) (response *grpc.Response, err error) { - n, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": request.NodeKey}, nil) + n, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": request.NodeKey}, nil) if err != nil { return nil, err } - depsDb, err := service.NewModelServiceV2[models.DependencyV2]().GetMany(bson.M{ + depsDb, err := service.NewModelServiceV2[models2.DependencyV2]().GetMany(bson.M{ "node_id": n.Id, "type": request.Lang, }, nil) @@ -56,15 +56,15 @@ func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.Dependen } } - depsDbMap := make(map[string]*models.DependencyV2) + depsDbMap := make(map[string]*models2.DependencyV2) for _, d := range depsDb { depsDbMap[d.Name] = &d } - var depsToInsert []models.DependencyV2 - depsMap := make(map[string]*models.DependencyV2) + var depsToInsert []models2.DependencyV2 + depsMap := make(map[string]*models2.DependencyV2) for _, dep := range request.Dependencies { - d := models.DependencyV2{ + d := models2.DependencyV2{ Name: dep.Name, NodeId: n.Id, Type: request.Lang, @@ -90,7 +90,7 @@ func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.Dependen err = mongo2.RunTransaction(func(ctx mongo.SessionContext) (err error) { if len(depIdsToDelete) > 0 { - err = service.NewModelServiceV2[models.DependencyV2]().DeleteMany(bson.M{ + err = service.NewModelServiceV2[models2.DependencyV2]().DeleteMany(bson.M{ "_id": bson.M{"$in": depIdsToDelete}, }) if err != nil { @@ -101,7 +101,7 @@ func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.Dependen } if len(depsToInsert) > 0 { - _, err = service.NewModelServiceV2[models.DependencyV2]().InsertMany(depsToInsert) + _, err = service.NewModelServiceV2[models2.DependencyV2]().InsertMany(depsToInsert) if err != nil { log.Errorf("[DependenciesServerV2] insert dependencies in db error: %v", err) trace.PrintError(err) @@ -116,7 +116,7 @@ func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.Dependen } func (svr DependenciesServerV2) UpdateTaskLog(stream grpc.DependenciesServiceV2_UpdateTaskLogServer) (err error) { - var t *models.DependencyTaskV2 + var t *models2.DependencyTaskV2 for { req, err := stream.Recv() if err == io.EOF { @@ -131,21 +131,21 @@ func (svr DependenciesServerV2) UpdateTaskLog(stream grpc.DependenciesServiceV2_ return err } if t == nil { - t, err = service.NewModelServiceV2[models.DependencyTaskV2]().GetById(taskId) + t, err = service.NewModelServiceV2[models2.DependencyTaskV2]().GetById(taskId) if err != nil { return err } } - var logs []models.DependencyLogV2 + var logs []models2.DependencyLogV2 for _, line := range req.LogLines { - l := models.DependencyLogV2{ + l := models2.DependencyLogV2{ TaskId: taskId, Content: line, } l.SetCreated(t.CreatedBy) logs = append(logs, l) } - _, err = service.NewModelServiceV2[models.DependencyLogV2]().InsertMany(logs) + _, err = service.NewModelServiceV2[models2.DependencyLogV2]().InsertMany(logs) if err != nil { return err } diff --git a/core/grpc/server/metrics_server_v2.go b/core/grpc/server/metrics_server_v2.go index fc51ad01..60a92569 100644 --- a/core/grpc/server/metrics_server_v2.go +++ b/core/grpc/server/metrics_server_v2.go @@ -3,7 +3,7 @@ package server import ( "context" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/grpc" "go.mongodb.org/mongo-driver/bson" @@ -17,12 +17,12 @@ type MetricsServerV2 struct { func (svr MetricsServerV2) Send(_ context.Context, req *grpc.MetricsServiceV2SendRequest) (res *grpc.Response, err error) { log.Info("[MetricsServerV2] received metric from node: " + req.NodeKey) - n, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil) + n, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil) if err != nil { log.Errorf("[MetricsServerV2] error getting node: %v", err) return HandleError(err) } - metric := models.MetricV2{ + metric := models2.MetricV2{ Type: req.Type, NodeId: n.Id, CpuUsagePercent: req.CpuUsagePercent, @@ -40,7 +40,7 @@ func (svr MetricsServerV2) Send(_ context.Context, req *grpc.MetricsServiceV2Sen NetworkBytesRecvRate: req.NetworkBytesRecvRate, } metric.CreatedAt = time.Unix(req.Timestamp, 0) - _, err = service.NewModelServiceV2[models.MetricV2]().InsertOne(metric) + _, err = service.NewModelServiceV2[models2.MetricV2]().InsertOne(metric) if err != nil { log.Errorf("[MetricsServerV2] error inserting metric: %v", err) return HandleError(err) diff --git a/core/grpc/server/model_base_service_v2_server.go b/core/grpc/server/model_base_service_v2_server.go index f777da61..eefd389a 100644 --- a/core/grpc/server/model_base_service_v2_server.go +++ b/core/grpc/server/model_base_service_v2_server.go @@ -3,10 +3,10 @@ package server import ( "context" "encoding/json" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" - grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/crawlab/grpc" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "reflect" @@ -16,32 +16,32 @@ var ( typeNameColNameMap = make(map[string]string) typeOneNameModelMap = make(map[string]any) typeOneInstances = []any{ - *new(models.TestModel), - *new(models.DataCollectionV2), - *new(models.DataSourceV2), - *new(models.DependencyV2), - *new(models.DependencyLogV2), - *new(models.DependencySettingV2), - *new(models.DependencyTaskV2), - *new(models.EnvironmentV2), - *new(models.GitV2), - *new(models.MetricV2), - *new(models.NodeV2), - *new(models.NotificationSettingV2), - *new(models.PermissionV2), - *new(models.ProjectV2), - *new(models.RolePermissionV2), - *new(models.RoleV2), - *new(models.ScheduleV2), - *new(models.SettingV2), - *new(models.SpiderV2), - *new(models.SpiderStatV2), - *new(models.TaskQueueItemV2), - *new(models.TaskStatV2), - *new(models.TaskV2), - *new(models.TokenV2), - *new(models.UserRoleV2), - *new(models.UserV2), + *new(models2.TestModelV2), + *new(models2.DataCollectionV2), + *new(models2.DataSourceV2), + *new(models2.DependencyV2), + *new(models2.DependencyLogV2), + *new(models2.DependencySettingV2), + *new(models2.DependencyTaskV2), + *new(models2.EnvironmentV2), + *new(models2.GitV2), + *new(models2.MetricV2), + *new(models2.NodeV2), + *new(models2.NotificationSettingV2), + *new(models2.PermissionV2), + *new(models2.ProjectV2), + *new(models2.RolePermissionV2), + *new(models2.RoleV2), + *new(models2.ScheduleV2), + *new(models2.SettingV2), + *new(models2.SpiderV2), + *new(models2.SpiderStatV2), + *new(models2.TaskQueueItemV2), + *new(models2.TaskStatV2), + *new(models2.TaskV2), + *new(models2.TokenV2), + *new(models2.UserRoleV2), + *new(models2.UserV2), } ) diff --git a/core/grpc/server/node_server_v2.go b/core/grpc/server/node_server_v2.go index a45e222e..4ef7de74 100644 --- a/core/grpc/server/node_server_v2.go +++ b/core/grpc/server/node_server_v2.go @@ -7,7 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/grpc" diff --git a/core/grpc/server/task_server_v2.go b/core/grpc/server/task_server_v2.go index 62b6ab32..80691492 100644 --- a/core/grpc/server/task_server_v2.go +++ b/core/grpc/server/task_server_v2.go @@ -8,7 +8,7 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/notification" @@ -72,7 +72,7 @@ func (svr TaskServerV2) Fetch(ctx context.Context, request *grpc.Request) (respo if nodeKey == "" { return nil, errors.New("invalid node key") } - n, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + n, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) if err != nil { return nil, trace.TraceError(err) } @@ -111,11 +111,11 @@ func (svr TaskServerV2) Fetch(ctx context.Context, request *grpc.Request) (respo func (svr TaskServerV2) SendNotification(ctx context.Context, request *grpc.Request) (response *grpc.Response, err error) { svc := notification.GetNotificationServiceV2() - var t = new(models.TaskV2) + var t = new(models2.TaskV2) if err := json.Unmarshal(request.Data, t); err != nil { return nil, trace.TraceError(err) } - t, err = service.NewModelServiceV2[models.TaskV2]().GetById(t.Id) + t, err = service.NewModelServiceV2[models2.TaskV2]().GetById(t.Id) if err != nil { return nil, trace.TraceError(err) } @@ -127,7 +127,7 @@ func (svr TaskServerV2) SendNotification(ctx context.Context, request *grpc.Requ if err := json.Unmarshal(td, &e); err != nil { return nil, trace.TraceError(err) } - ts, err := service.NewModelServiceV2[models.TaskStatV2]().GetById(t.Id) + ts, err := service.NewModelServiceV2[models2.TaskStatV2]().GetById(t.Id) if err != nil { return nil, trace.TraceError(err) } @@ -190,22 +190,22 @@ func (svr TaskServerV2) handleInsertLogs(msg *grpc.StreamMessage) (err error) { } func (svr TaskServerV2) getTaskQueueItemIdAndDequeue(query bson.M, opts *mongo.FindOptions, nid primitive.ObjectID) (tid primitive.ObjectID, err error) { - tq, err := service.NewModelServiceV2[models.TaskQueueItemV2]().GetOne(query, opts) + tq, err := service.NewModelServiceV2[models2.TaskQueueItemV2]().GetOne(query, opts) if err != nil { if errors.Is(err, mongo2.ErrNoDocuments) { return tid, nil } return tid, trace.TraceError(err) } - t, err := service.NewModelServiceV2[models.TaskV2]().GetById(tq.Id) + t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(tq.Id) if err == nil { t.NodeId = nid - err = service.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t) + err = service.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t) if err != nil { return tid, trace.TraceError(err) } } - err = service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteById(tq.Id) + err = service.NewModelServiceV2[models2.TaskQueueItemV2]().DeleteById(tq.Id) if err != nil { return tid, trace.TraceError(err) } diff --git a/core/models/client/model_service_v2_test.go b/core/models/client/model_service_v2_test.go index c4053bb1..7bd5ac5f 100644 --- a/core/models/client/model_service_v2_test.go +++ b/core/models/client/model_service_v2_test.go @@ -4,7 +4,7 @@ import ( "context" "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "github.com/spf13/viper" @@ -17,7 +17,7 @@ import ( "time" ) -type TestModel models.TestModel +type TestModel models.TestModelV2 func setupTestDB() { viper.Set("mongo.db", "testdb") diff --git a/core/models/common/index_service_v2.go b/core/models/common/index_service_v2.go index e87ec7f3..9db29c88 100644 --- a/core/models/common/index_service_v2.go +++ b/core/models/common/index_service_v2.go @@ -1,7 +1,7 @@ package common import ( - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "go.mongodb.org/mongo-driver/bson" @@ -11,7 +11,7 @@ import ( func CreateIndexesV2() { // nodes - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.NodeV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NodeV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"key": 1}}, // key {Keys: bson.M{"name": 1}}, // name {Keys: bson.M{"is_master": 1}}, // is_master @@ -21,12 +21,12 @@ func CreateIndexesV2() { }) // projects - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.ProjectV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ProjectV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, }) // spiders - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.SpiderV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SpiderV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, {Keys: bson.M{"type": 1}}, {Keys: bson.M{"col_id": 1}}, @@ -34,7 +34,7 @@ func CreateIndexesV2() { }) // tasks - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TaskV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"spider_id": 1}}, {Keys: bson.M{"status": 1}}, {Keys: bson.M{"node_id": 1}}, @@ -48,73 +48,73 @@ func CreateIndexesV2() { }) // task stats - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TaskStatV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskStatV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"create_ts": 1}}, }) // schedules - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.ScheduleV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ScheduleV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, {Keys: bson.M{"spider_id": 1}}, {Keys: bson.M{"enabled": 1}}, }) // users - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.UserV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"username": 1}}, {Keys: bson.M{"role": 1}}, {Keys: bson.M{"email": 1}}, }) // settings - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"key": 1}}, }) // tokens - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TokenV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TokenV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, }) // variables - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.VariableV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.VariableV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"key": 1}}, }) // data sources - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DataSourceV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataSourceV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, }) // data collections - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DataCollectionV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataCollectionV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, }) // roles - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.RoleV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RoleV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, }) // user role relations - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.UserRoleV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserRoleV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.D{{"user_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)}, {Keys: bson.D{{"role_id", 1}, {"user_id", 1}}, Options: options.Index().SetUnique(true)}, }) // permissions - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.PermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.PermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, }) // role permission relations - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.RolePermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RolePermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.D{{"role_id", 1}, {"permission_id", 1}}, Options: options.Index().SetUnique(true)}, {Keys: bson.D{{"permission_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)}, }) // dependencies - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyV2{})).MustCreateIndexes([]mongo2.IndexModel{ { Keys: bson.D{ {"type", 1}, @@ -126,7 +126,7 @@ func CreateIndexesV2() { }) // dependency settings - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencySettingV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencySettingV2{})).MustCreateIndexes([]mongo2.IndexModel{ { Keys: bson.D{ {"type", 1}, @@ -138,7 +138,7 @@ func CreateIndexesV2() { }) // dependency logs - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyLogV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyLogV2{})).MustCreateIndexes([]mongo2.IndexModel{ { Keys: bson.D{{"task_id", 1}}, }, @@ -149,7 +149,7 @@ func CreateIndexesV2() { }) // dependency tasks - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyTaskV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyTaskV2{})).MustCreateIndexes([]mongo2.IndexModel{ { Keys: bson.D{ {"update_ts", 1}, @@ -159,7 +159,7 @@ func CreateIndexesV2() { }) // metrics - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.MetricV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.MetricV2{})).MustCreateIndexes([]mongo2.IndexModel{ { Keys: bson.D{ {"created_ts", -1}, diff --git a/core/models/models/v2/data_collection_v2.go b/core/models/models/v2/data_collection_v2.go index 1a3c724e..8ec5e8cb 100644 --- a/core/models/models/v2/data_collection_v2.go +++ b/core/models/models/v2/data_collection_v2.go @@ -5,11 +5,11 @@ import ( ) type DataCollectionV2 struct { - any `collection:"data_collections"` - BaseModelV2[DataCollection] `bson:",inline"` - Name string `json:"name" bson:"name"` - Fields []entity.DataField `json:"fields" bson:"fields"` - Dedup struct { + any `collection:"data_collections"` + BaseModelV2[DataCollectionV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Fields []entity.DataField `json:"fields" bson:"fields"` + Dedup struct { Enabled bool `json:"enabled" bson:"enabled"` Keys []string `json:"keys" bson:"keys"` Type string `json:"type" bson:"type"` diff --git a/core/models/models/v2/data_source_v2.go b/core/models/models/v2/data_source_v2.go index 16415510..7e9201cd 100644 --- a/core/models/models/v2/data_source_v2.go +++ b/core/models/models/v2/data_source_v2.go @@ -1,20 +1,20 @@ package models type DataSourceV2 struct { - any `collection:"data_sources"` - BaseModelV2[DataSource] `bson:",inline"` - Name string `json:"name" bson:"name"` - Type string `json:"type" bson:"type"` - Description string `json:"description" bson:"description"` - Host string `json:"host" bson:"host"` - Port string `json:"port" bson:"port"` - Url string `json:"url" bson:"url"` - Hosts []string `json:"hosts" bson:"hosts"` - Database string `json:"database" bson:"database"` - Username string `json:"username" bson:"username"` - Password string `json:"-,omitempty" bson:"password"` - ConnectType string `json:"connect_type" bson:"connect_type"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"` + any `collection:"data_sources"` + BaseModelV2[DataSourceV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Type string `json:"type" bson:"type"` + Description string `json:"description" bson:"description"` + Host string `json:"host" bson:"host"` + Port string `json:"port" bson:"port"` + Url string `json:"url" bson:"url"` + Hosts []string `json:"hosts" bson:"hosts"` + Database string `json:"database" bson:"database"` + Username string `json:"username" bson:"username"` + Password string `json:"-,omitempty" bson:"password"` + ConnectType string `json:"connect_type" bson:"connect_type"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"` } diff --git a/core/models/models/v2/dependency_setting_v2.go b/core/models/models/v2/dependency_setting_v2.go index d4c241e6..528c44d0 100644 --- a/core/models/models/v2/dependency_setting_v2.go +++ b/core/models/models/v2/dependency_setting_v2.go @@ -5,13 +5,13 @@ import ( ) type DependencySettingV2 struct { - any `collection:"dependency_settings"` - BaseModelV2[DependencySetting] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - Cmd string `json:"cmd" bson:"cmd"` - Proxy string `json:"proxy" bson:"proxy"` - LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"` + any `collection:"dependency_settings"` + BaseModelV2[DependencySettingV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + Cmd string `json:"cmd" bson:"cmd"` + Proxy string `json:"proxy" bson:"proxy"` + LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"` } diff --git a/core/models/models/v2/git_v2.go b/core/models/models/v2/git_v2.go index 577be132..730c2c89 100644 --- a/core/models/models/v2/git_v2.go +++ b/core/models/models/v2/git_v2.go @@ -1,26 +1,25 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/vcs" "time" ) type GitV2 struct { - any `collection:"gits"` - models.BaseModelV2[GitV2] `bson:",inline"` - Url string `json:"url" bson:"url"` - Name string `json:"name" bson:"name"` - AuthType string `json:"auth_type" bson:"auth_type"` - Username string `json:"username" bson:"username"` - Password string `json:"password" bson:"password"` - CurrentBranch string `json:"current_branch" bson:"current_branch"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"` - Refs []vcs.GitRef `json:"refs" bson:"refs"` - RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"` - CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"` + any `collection:"gits"` + BaseModelV2[GitV2] `bson:",inline"` + Url string `json:"url" bson:"url"` + Name string `json:"name" bson:"name"` + AuthType string `json:"auth_type" bson:"auth_type"` + Username string `json:"username" bson:"username"` + Password string `json:"password" bson:"password"` + CurrentBranch string `json:"current_branch" bson:"current_branch"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"` + Refs []vcs.GitRef `json:"refs" bson:"refs"` + RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"` + CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"` // settings AutoPull bool `json:"auto_pull" bson:"auto_pull"` diff --git a/core/models/models/v2/metric_v2.go b/core/models/models/v2/metric_v2.go index 08cc7749..8d23dd91 100644 --- a/core/models/models/v2/metric_v2.go +++ b/core/models/models/v2/metric_v2.go @@ -1,26 +1,25 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson/primitive" ) type MetricV2 struct { - any `collection:"metrics"` - models.BaseModelV2[MetricV2] `bson:",inline"` - Type string `json:"type" bson:"type"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` - TotalMemory uint64 `json:"total_memory" bson:"total_memory"` - AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` - UsedMemory uint64 `json:"used_memory" bson:"used_memory"` - UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` - TotalDisk uint64 `json:"total_disk" bson:"total_disk"` - AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` - UsedDisk uint64 `json:"used_disk" bson:"used_disk"` - UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` - DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"` - DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"` - NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"` - NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"` + any `collection:"metrics"` + BaseModelV2[MetricV2] `bson:",inline"` + Type string `json:"type" bson:"type"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` + TotalMemory uint64 `json:"total_memory" bson:"total_memory"` + AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` + UsedMemory uint64 `json:"used_memory" bson:"used_memory"` + UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` + TotalDisk uint64 `json:"total_disk" bson:"total_disk"` + AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` + UsedDisk uint64 `json:"used_disk" bson:"used_disk"` + UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` + DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"` + DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"` + NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"` + NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"` } diff --git a/core/models/models/v2/node_v2.go b/core/models/models/v2/node_v2.go index 4031b97c..6d5a8442 100644 --- a/core/models/models/v2/node_v2.go +++ b/core/models/models/v2/node_v2.go @@ -1,25 +1,24 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "time" ) type NodeV2 struct { - any `collection:"nodes"` - models.BaseModelV2[NodeV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Ip string `json:"ip" bson:"ip"` - Port string `json:"port" bson:"port"` - Mac string `json:"mac" bson:"mac"` - Hostname string `json:"hostname" bson:"hostname"` - Description string `json:"description" bson:"description"` - IsMaster bool `json:"is_master" bson:"is_master"` - Status string `json:"status" bson:"status"` - Enabled bool `json:"enabled" bson:"enabled"` - Active bool `json:"active" bson:"active"` - ActiveAt time.Time `json:"active_at" bson:"active_ts"` - AvailableRunners int `json:"available_runners" bson:"available_runners"` - MaxRunners int `json:"max_runners" bson:"max_runners"` + any `collection:"nodes"` + BaseModelV2[NodeV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Ip string `json:"ip" bson:"ip"` + Port string `json:"port" bson:"port"` + Mac string `json:"mac" bson:"mac"` + Hostname string `json:"hostname" bson:"hostname"` + Description string `json:"description" bson:"description"` + IsMaster bool `json:"is_master" bson:"is_master"` + Status string `json:"status" bson:"status"` + Enabled bool `json:"enabled" bson:"enabled"` + Active bool `json:"active" bson:"active"` + ActiveAt time.Time `json:"active_at" bson:"active_ts"` + AvailableRunners int `json:"available_runners" bson:"available_runners"` + MaxRunners int `json:"max_runners" bson:"max_runners"` } diff --git a/core/models/models/v2/notification_channel_v2.go b/core/models/models/v2/notification_channel_v2.go index 2640e7f9..cd317221 100644 --- a/core/models/models/v2/notification_channel_v2.go +++ b/core/models/models/v2/notification_channel_v2.go @@ -1 +1,4 @@ package models + +type NotificationChannel struct { +} diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/v2/notification_setting_v2.go index d24796b8..b07a0197 100644 --- a/core/models/models/v2/notification_setting_v2.go +++ b/core/models/models/v2/notification_setting_v2.go @@ -1,20 +1,18 @@ package models -import "github.com/crawlab-team/crawlab/core/models/models/v2" - type NotificationSettingV2 struct { - any `collection:"notification_settings"` - models.BaseModelV2[NotificationSettingV2] `bson:",inline"` - Type string `json:"type" bson:"type"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - Global bool `json:"global" bson:"global"` - Title string `json:"title,omitempty" bson:"title,omitempty"` - Template string `json:"template,omitempty" bson:"template,omitempty"` - TaskTrigger string `json:"task_trigger" bson:"task_trigger"` - Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` - Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` + any `collection:"notification_settings"` + BaseModelV2[NotificationSettingV2] `bson:",inline"` + Type string `json:"type" bson:"type"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + Global bool `json:"global" bson:"global"` + Title string `json:"title,omitempty" bson:"title,omitempty"` + Template string `json:"template,omitempty" bson:"template,omitempty"` + TaskTrigger string `json:"task_trigger" bson:"task_trigger"` + Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` + Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` } type NotificationSettingMail struct { diff --git a/core/models/models/v2/permission_v2.go b/core/models/models/v2/permission_v2.go index 55558384..77c830a3 100644 --- a/core/models/models/v2/permission_v2.go +++ b/core/models/models/v2/permission_v2.go @@ -1,15 +1,13 @@ package models -import "github.com/crawlab-team/crawlab/core/models/models/v2" - type PermissionV2 struct { - any `collection:"permissions"` - models.BaseModelV2[PermissionV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Type string `json:"type" bson:"type"` - Target []string `json:"target" bson:"target"` - Allow []string `json:"allow" bson:"allow"` - Deny []string `json:"deny" bson:"deny"` + any `collection:"permissions"` + BaseModelV2[PermissionV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Type string `json:"type" bson:"type"` + Target []string `json:"target" bson:"target"` + Allow []string `json:"allow" bson:"allow"` + Deny []string `json:"deny" bson:"deny"` } diff --git a/core/models/models/v2/project_v2.go b/core/models/models/v2/project_v2.go index fb1152c1..ee07e705 100644 --- a/core/models/models/v2/project_v2.go +++ b/core/models/models/v2/project_v2.go @@ -1,11 +1,9 @@ package models -import "github.com/crawlab-team/crawlab/core/models/models/v2" - type ProjectV2 struct { - any `collection:"projects"` - models.BaseModelV2[ProjectV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Spiders int `json:"spiders" bson:"-"` + any `collection:"projects"` + BaseModelV2[ProjectV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Spiders int `json:"spiders" bson:"-"` } diff --git a/core/models/models/v2/role_permission_v2.go b/core/models/models/v2/role_permission_v2.go index 8a77d2fb..386fa052 100644 --- a/core/models/models/v2/role_permission_v2.go +++ b/core/models/models/v2/role_permission_v2.go @@ -1,13 +1,12 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson/primitive" ) type RolePermissionV2 struct { - any `collection:"role_permissions"` - models.BaseModelV2[RolePermissionV2] `bson:",inline"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"` + any `collection:"role_permissions"` + BaseModelV2[RolePermissionV2] `bson:",inline"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"` } diff --git a/core/models/models/v2/role_v2.go b/core/models/models/v2/role_v2.go index 3deda9eb..de287a61 100644 --- a/core/models/models/v2/role_v2.go +++ b/core/models/models/v2/role_v2.go @@ -1,11 +1,9 @@ package models -import "github.com/crawlab-team/crawlab/core/models/models/v2" - type RoleV2 struct { - any `collection:"roles"` - models.BaseModelV2[RoleV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` + any `collection:"roles"` + BaseModelV2[RoleV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` } diff --git a/core/models/models/v2/schedule_v2.go b/core/models/models/v2/schedule_v2.go index f304942b..a52f1b92 100644 --- a/core/models/models/v2/schedule_v2.go +++ b/core/models/models/v2/schedule_v2.go @@ -1,24 +1,23 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/robfig/cron/v3" "go.mongodb.org/mongo-driver/bson/primitive" ) type ScheduleV2 struct { - any `collection:"schedules"` - models.BaseModelV2[ScheduleV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` - Cron string `json:"cron" bson:"cron"` - EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` - Cmd string `json:"cmd" bson:"cmd"` - Param string `json:"param" bson:"param"` - Mode string `json:"mode" bson:"mode"` - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` - Priority int `json:"priority" bson:"priority"` - Enabled bool `json:"enabled" bson:"enabled"` - UserId primitive.ObjectID `json:"user_id" bson:"user_id"` + any `collection:"schedules"` + BaseModelV2[ScheduleV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` + Cron string `json:"cron" bson:"cron"` + EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` + Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` + Mode string `json:"mode" bson:"mode"` + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` + Priority int `json:"priority" bson:"priority"` + Enabled bool `json:"enabled" bson:"enabled"` + UserId primitive.ObjectID `json:"user_id" bson:"user_id"` } diff --git a/core/models/models/v2/setting_v2.go b/core/models/models/v2/setting_v2.go index bdb35694..86481607 100644 --- a/core/models/models/v2/setting_v2.go +++ b/core/models/models/v2/setting_v2.go @@ -1,13 +1,12 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson" ) type SettingV2 struct { - any `collection:"settings"` - models.BaseModelV2[SettingV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Value bson.M `json:"value" bson:"value"` + any `collection:"settings"` + BaseModelV2[SettingV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Value bson.M `json:"value" bson:"value"` } diff --git a/core/models/models/v2/spider_stat_v2.go b/core/models/models/v2/spider_stat_v2.go index 3cdc8954..ea01c0cc 100644 --- a/core/models/models/v2/spider_stat_v2.go +++ b/core/models/models/v2/spider_stat_v2.go @@ -1,21 +1,20 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson/primitive" ) type SpiderStatV2 struct { - any `collection:"spider_stats"` - models.BaseModelV2[SpiderStatV2] `bson:",inline"` - LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"` - LastTask *TaskV2 `json:"last_task,omitempty" bson:"-"` - Tasks int `json:"tasks" bson:"tasks"` - Results int `json:"results" bson:"results"` - WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second - RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second - TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second - AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second - AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second - AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second + any `collection:"spider_stats"` + BaseModelV2[SpiderStatV2] `bson:",inline"` + LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"` + LastTask *TaskV2 `json:"last_task,omitempty" bson:"-"` + Tasks int `json:"tasks" bson:"tasks"` + Results int `json:"results" bson:"results"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second + AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second + AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second + AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second } diff --git a/core/models/models/v2/spider_v2.go b/core/models/models/v2/spider_v2.go index 19cdf9c5..ac3bf300 100644 --- a/core/models/models/v2/spider_v2.go +++ b/core/models/models/v2/spider_v2.go @@ -1,26 +1,25 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson/primitive" ) type SpiderV2 struct { - any `collection:"spiders"` - models.BaseModelV2[SpiderV2] `bson:",inline"` - Name string `json:"name" bson:"name"` // spider name - Type string `json:"type" bson:"type"` // spider type - ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id - ColName string `json:"col_name,omitempty" bson:"-"` // data collection name - DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id - DataSource *models.DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source - Description string `json:"description" bson:"description"` // description - ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id - Mode string `json:"mode" bson:"mode"` // default Task.Mode - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds - GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id - GitRootPath string `json:"git_root_path" bson:"git_root_path"` - Git *GitV2 `json:"git,omitempty" bson:"-"` + any `collection:"spiders"` + BaseModelV2[SpiderV2] `bson:",inline"` + Name string `json:"name" bson:"name"` // spider name + Type string `json:"type" bson:"type"` // spider type + ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id + ColName string `json:"col_name,omitempty" bson:"-"` // data collection name + DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id + DataSource *DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source + Description string `json:"description" bson:"description"` // description + ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id + Mode string `json:"mode" bson:"mode"` // default Task.Mode + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds + GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id + GitRootPath string `json:"git_root_path" bson:"git_root_path"` + Git *GitV2 `json:"git,omitempty" bson:"-"` // stats Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"` diff --git a/core/models/models/v2/task_queue_item_v2.go b/core/models/models/v2/task_queue_item_v2.go index 28b2f88d..f222aef3 100644 --- a/core/models/models/v2/task_queue_item_v2.go +++ b/core/models/models/v2/task_queue_item_v2.go @@ -1,13 +1,12 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson/primitive" ) type TaskQueueItemV2 struct { - any `collection:"task_queue"` - models.BaseModelV2[TaskQueueItemV2] `bson:",inline"` - Priority int `json:"p" bson:"p"` - NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"` + any `collection:"task_queue"` + BaseModelV2[TaskQueueItemV2] `bson:",inline"` + Priority int `json:"p" bson:"p"` + NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"` } diff --git a/core/models/models/v2/task_stat_v2.go b/core/models/models/v2/task_stat_v2.go index 5cf1fff1..5456946d 100644 --- a/core/models/models/v2/task_stat_v2.go +++ b/core/models/models/v2/task_stat_v2.go @@ -1,19 +1,18 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "time" ) type TaskStatV2 struct { - any `collection:"task_stats"` - models.BaseModelV2[TaskStatV2] `bson:",inline"` - CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"` - StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` - EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` - WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond - RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond - TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond - ResultCount int64 `json:"result_count" bson:"result_count"` - ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"` + any `collection:"task_stats"` + BaseModelV2[TaskStatV2] `bson:",inline"` + CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"` + StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` + EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond + ResultCount int64 `json:"result_count" bson:"result_count"` + ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"` } diff --git a/core/models/models/v2/task_v2.go b/core/models/models/v2/task_v2.go index 31f46706..3473b6db 100644 --- a/core/models/models/v2/task_v2.go +++ b/core/models/models/v2/task_v2.go @@ -1,31 +1,30 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson/primitive" "time" ) type TaskV2 struct { - any `collection:"tasks"` - models.BaseModelV2[TaskV2] `bson:",inline"` - SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` - Status string `json:"status" bson:"status"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - Cmd string `json:"cmd" bson:"cmd"` - Param string `json:"param" bson:"param"` - Error string `json:"error" bson:"error"` - Pid int `json:"pid" bson:"pid"` - ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` - Type string `json:"type" bson:"type"` - Mode string `json:"mode" bson:"mode"` - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` - ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"` - Priority int `json:"priority" bson:"priority"` - Stat *TaskStatV2 `json:"stat,omitempty" bson:"-"` - HasSub bool `json:"has_sub" json:"has_sub"` - SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"` - Spider *SpiderV2 `json:"spider,omitempty" bson:"-"` - UserId primitive.ObjectID `json:"-" bson:"-"` - CreateTs time.Time `json:"create_ts" bson:"create_ts"` + any `collection:"tasks"` + BaseModelV2[TaskV2] `bson:",inline"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` + Status string `json:"status" bson:"status"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` + Error string `json:"error" bson:"error"` + Pid int `json:"pid" bson:"pid"` + ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` + Type string `json:"type" bson:"type"` + Mode string `json:"mode" bson:"mode"` + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` + ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"` + Priority int `json:"priority" bson:"priority"` + Stat *TaskStatV2 `json:"stat,omitempty" bson:"-"` + HasSub bool `json:"has_sub" json:"has_sub"` + SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"` + Spider *SpiderV2 `json:"spider,omitempty" bson:"-"` + UserId primitive.ObjectID `json:"-" bson:"-"` + CreateTs time.Time `json:"create_ts" bson:"create_ts"` } diff --git a/core/models/models/v2/test_v2.go b/core/models/models/v2/test_v2.go index 67027d19..1468abc1 100644 --- a/core/models/models/v2/test_v2.go +++ b/core/models/models/v2/test_v2.go @@ -1,7 +1,7 @@ package models -type TestModel struct { - any `collection:"testmodels"` - BaseModelV2[TestModel] `bson:",inline"` - Name string `json:"name" bson:"name"` +type TestModelV2 struct { + any `collection:"testmodels"` + BaseModelV2[TestModelV2] `bson:",inline"` + Name string `json:"name" bson:"name"` } diff --git a/core/models/models/v2/token_v2.go b/core/models/models/v2/token_v2.go index 1fe20d88..c792c9ee 100644 --- a/core/models/models/v2/token_v2.go +++ b/core/models/models/v2/token_v2.go @@ -1,10 +1,8 @@ package models -import "github.com/crawlab-team/crawlab/core/models/models/v2" - type TokenV2 struct { - any `collection:"tokens"` - models.BaseModelV2[TokenV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Token string `json:"token" bson:"token"` + any `collection:"tokens"` + BaseModelV2[TokenV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Token string `json:"token" bson:"token"` } diff --git a/core/models/models/v2/user_role_v2.go b/core/models/models/v2/user_role_v2.go index e36e2f70..aa0ac34a 100644 --- a/core/models/models/v2/user_role_v2.go +++ b/core/models/models/v2/user_role_v2.go @@ -1,13 +1,12 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson/primitive" ) type UserRoleV2 struct { - any `collection:"user_roles"` - models.BaseModelV2[UserRoleV2] `bson:",inline"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - UserId primitive.ObjectID `json:"user_id" bson:"user_id"` + any `collection:"user_roles"` + BaseModelV2[UserRoleV2] `bson:",inline"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + UserId primitive.ObjectID `json:"user_id" bson:"user_id"` } diff --git a/core/models/models/v2/user_v2.go b/core/models/models/v2/user_v2.go index 131159aa..acfbe2b6 100644 --- a/core/models/models/v2/user_v2.go +++ b/core/models/models/v2/user_v2.go @@ -1,12 +1,10 @@ package models -import "github.com/crawlab-team/crawlab/core/models/models/v2" - type UserV2 struct { - any `collection:"users"` - models.BaseModelV2[UserV2] `bson:",inline"` - Username string `json:"username" bson:"username"` - Password string `json:"-,omitempty" bson:"password"` - Role string `json:"role" bson:"role"` - Email string `json:"email" bson:"email"` + any `collection:"users"` + BaseModelV2[UserV2] `bson:",inline"` + Username string `json:"username" bson:"username"` + Password string `json:"-,omitempty" bson:"password"` + Role string `json:"role" bson:"role"` + Email string `json:"email" bson:"email"` } diff --git a/core/models/models/v2/variable_v2.go b/core/models/models/v2/variable_v2.go index cd8be4b4..49788832 100644 --- a/core/models/models/v2/variable_v2.go +++ b/core/models/models/v2/variable_v2.go @@ -1,11 +1,9 @@ package models -import "github.com/crawlab-team/crawlab/core/models/models/v2" - type VariableV2 struct { - any `collection:"variables"` - models.BaseModelV2[VariableV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Value string `json:"value" bson:"value"` - Remark string `json:"remark" bson:"remark"` + any `collection:"variables"` + BaseModelV2[VariableV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Value string `json:"value" bson:"value"` + Remark string `json:"remark" bson:"remark"` } diff --git a/core/models/service/base_service_v2_test.go b/core/models/service/base_service_v2_test.go index 18eec697..2d23cb47 100644 --- a/core/models/service/base_service_v2_test.go +++ b/core/models/service/base_service_v2_test.go @@ -2,7 +2,7 @@ package service_test import ( "context" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "github.com/spf13/viper" diff --git a/core/node/service/master_service_v2.go b/core/node/service/master_service_v2.go index cbf2249d..73f9cf34 100644 --- a/core/node/service/master_service_v2.go +++ b/core/node/service/master_service_v2.go @@ -9,7 +9,7 @@ import ( "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/common" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/notification" @@ -142,11 +142,11 @@ func (svc *MasterServiceV2) SetMonitorInterval(duration time.Duration) { func (svc *MasterServiceV2) Register() (err error) { nodeKey := svc.GetConfigService().GetNodeKey() nodeName := svc.GetConfigService().GetNodeName() - node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + node, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) if err != nil && err.Error() == mongo2.ErrNoDocuments.Error() { // not exists log.Infof("master[%s] does not exist in db", nodeKey) - node := models.NodeV2{ + node := models2.NodeV2{ Key: nodeKey, Name: nodeName, MaxRunners: config.DefaultConfigOptions.MaxRunners, @@ -158,7 +158,7 @@ func (svc *MasterServiceV2) Register() (err error) { } node.SetCreated(primitive.NilObjectID) node.SetUpdated(primitive.NilObjectID) - id, err := service.NewModelServiceV2[models.NodeV2]().InsertOne(node) + id, err := service.NewModelServiceV2[models2.NodeV2]().InsertOne(node) if err != nil { return err } @@ -170,7 +170,7 @@ func (svc *MasterServiceV2) Register() (err error) { node.Status = constants.NodeStatusOnline node.Active = true node.ActiveAt = time.Now() - err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node) if err != nil { return err } @@ -209,7 +209,7 @@ func (svc *MasterServiceV2) monitor() (err error) { wg := sync.WaitGroup{} wg.Add(len(workerNodes)) for _, n := range workerNodes { - go func(n *models.NodeV2) { + go func(n *models2.NodeV2) { // subscribe ok := svc.subscribeNode(n) if !ok { @@ -243,12 +243,12 @@ func (svc *MasterServiceV2) monitor() (err error) { return nil } -func (svc *MasterServiceV2) getAllWorkerNodes() (nodes []models.NodeV2, err error) { +func (svc *MasterServiceV2) getAllWorkerNodes() (nodes []models2.NodeV2, err error) { query := bson.M{ "key": bson.M{"$ne": svc.cfgSvc.GetNodeKey()}, // not self "active": true, // active } - nodes, err = service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil) + nodes, err = service.NewModelServiceV2[models2.NodeV2]().GetMany(query, nil) if err != nil { if errors.Is(err, mongo2.ErrNoDocuments) { return nil, nil @@ -260,32 +260,32 @@ func (svc *MasterServiceV2) getAllWorkerNodes() (nodes []models.NodeV2, err erro func (svc *MasterServiceV2) updateMasterNodeStatus() (err error) { nodeKey := svc.GetConfigService().GetNodeKey() - node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + node, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) if err != nil { return err } node.Status = constants.NodeStatusOnline node.Active = true node.ActiveAt = time.Now() - err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node) if err != nil { return err } return nil } -func (svc *MasterServiceV2) setWorkerNodeOffline(node *models.NodeV2) { +func (svc *MasterServiceV2) setWorkerNodeOffline(node *models2.NodeV2) { node.Status = constants.NodeStatusOffline node.Active = false err := backoff.Retry(func() error { - return service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + return service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node) }, backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 3)) if err != nil { trace.PrintError(err) } } -func (svc *MasterServiceV2) subscribeNode(n *models.NodeV2) (ok bool) { +func (svc *MasterServiceV2) subscribeNode(n *models2.NodeV2) (ok bool) { _, err := svc.server.GetSubscribe("node:" + n.Key) if err != nil { log.Errorf("cannot subscribe worker node[%s]: %v", n.Key, err) @@ -294,7 +294,7 @@ func (svc *MasterServiceV2) subscribeNode(n *models.NodeV2) (ok bool) { return true } -func (svc *MasterServiceV2) pingNodeClient(n *models.NodeV2) (ok bool) { +func (svc *MasterServiceV2) pingNodeClient(n *models2.NodeV2) (ok bool) { if err := svc.server.SendStreamMessage("node:"+n.Key, grpc.StreamMessageCode_PING); err != nil { log.Errorf("cannot ping worker node client[%s]: %v", n.Key, err) return false @@ -302,17 +302,17 @@ func (svc *MasterServiceV2) pingNodeClient(n *models.NodeV2) (ok bool) { return true } -func (svc *MasterServiceV2) updateNodeAvailableRunners(node *models.NodeV2) (err error) { +func (svc *MasterServiceV2) updateNodeAvailableRunners(node *models2.NodeV2) (err error) { query := bson.M{ "node_id": node.Id, "status": constants.TaskStatusRunning, } - runningTasksCount, err := service.NewModelServiceV2[models.TaskV2]().Count(query) + runningTasksCount, err := service.NewModelServiceV2[models2.TaskV2]().Count(query) if err != nil { return trace.TraceError(err) } node.AvailableRunners = node.MaxRunners - runningTasksCount - err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node) if err != nil { return err } diff --git a/core/node/service/worker_service_v2.go b/core/node/service/worker_service_v2.go index da404758..a041877c 100644 --- a/core/node/service/worker_service_v2.go +++ b/core/node/service/worker_service_v2.go @@ -9,6 +9,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" client2 "github.com/crawlab-team/crawlab/core/models/client" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/handler" "github.com/crawlab-team/crawlab/core/utils" @@ -31,7 +32,7 @@ type WorkerServiceV2 struct { heartbeatInterval time.Duration // internals - n *models.NodeV2 + n *models2.NodeV2 s grpc.NodeService_SubscribeClient } @@ -87,7 +88,7 @@ func (svc *WorkerServiceV2) Register() { if err != nil { panic(err) } - svc.n, err = client2.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": svc.GetConfigService().GetNodeKey()}, nil) + svc.n, err = client2.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": svc.GetConfigService().GetNodeKey()}, nil) if err != nil { panic(err) } diff --git a/core/notification/mail.go b/core/notification/mail.go index 905a5446..c7f71a0e 100644 --- a/core/notification/mail.go +++ b/core/notification/mail.go @@ -3,7 +3,7 @@ package notification import ( "errors" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/matcornic/hermes/v2" "gopkg.in/gomail.v2" "net/mail" diff --git a/core/notification/service.go b/core/notification/service.go index 37a7f93d..0f3bf7bf 100644 --- a/core/notification/service.go +++ b/core/notification/service.go @@ -4,7 +4,7 @@ import ( "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" mongo2 "github.com/crawlab-team/crawlab/db/mongo" diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 0d8a4c88..a2978431 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -4,7 +4,7 @@ import ( "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" mongo2 "github.com/crawlab-team/crawlab/db/mongo" parser "github.com/crawlab-team/crawlab/template-parser" diff --git a/core/schedule/service_v2.go b/core/schedule/service_v2.go index 1d2d09da..a1a244ac 100644 --- a/core/schedule/service_v2.go +++ b/core/schedule/service_v2.go @@ -4,7 +4,7 @@ import ( "github.com/apex/log" "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/utils" @@ -19,7 +19,7 @@ import ( type ServiceV2 struct { // dependencies interfaces.WithConfigPath - modelSvc *service.ModelServiceV2[models.ScheduleV2] + modelSvc *service.ModelServiceV2[models2.ScheduleV2] adminSvc *admin.ServiceV2 // settings variables @@ -31,7 +31,7 @@ type ServiceV2 struct { // internals cron *cron.Cron logger cron.Logger - schedules []models.ScheduleV2 + schedules []models2.ScheduleV2 stopped bool mu sync.Mutex } @@ -87,7 +87,7 @@ func (svc *ServiceV2) Stop() { svc.cron.Stop() } -func (svc *ServiceV2) Enable(s models.ScheduleV2, by primitive.ObjectID) (err error) { +func (svc *ServiceV2) Enable(s models2.ScheduleV2, by primitive.ObjectID) (err error) { svc.mu.Lock() defer svc.mu.Unlock() @@ -101,7 +101,7 @@ func (svc *ServiceV2) Enable(s models.ScheduleV2, by primitive.ObjectID) (err er return svc.modelSvc.ReplaceById(s.Id, s) } -func (svc *ServiceV2) Disable(s models.ScheduleV2, by primitive.ObjectID) (err error) { +func (svc *ServiceV2) Disable(s models2.ScheduleV2, by primitive.ObjectID) (err error) { svc.mu.Lock() defer svc.mu.Unlock() @@ -191,7 +191,7 @@ func (svc *ServiceV2) schedule(id primitive.ObjectID) (fn func()) { } // spider - spider, err := service.NewModelServiceV2[models.SpiderV2]().GetById(s.SpiderId) + spider, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(s.SpiderId) if err != nil { trace.PrintError(err) return @@ -250,7 +250,7 @@ func NewScheduleServiceV2() (svc2 *ServiceV2, err error) { if err != nil { return nil, err } - svc.modelSvc = service.NewModelServiceV2[models.ScheduleV2]() + svc.modelSvc = service.NewModelServiceV2[models2.ScheduleV2]() // logger svc.logger = NewLogger() diff --git a/core/spider/admin/service_v2.go b/core/spider/admin/service_v2.go index 4ead1ec8..cf821201 100644 --- a/core/spider/admin/service_v2.go +++ b/core/spider/admin/service_v2.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/scheduler" @@ -35,7 +35,7 @@ func (svc *ServiceV2) Start() (err error) { func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { // spider - s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if err != nil { return nil, err } @@ -44,9 +44,9 @@ func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRun return svc.scheduleTasks(s, opts) } -func (svc *ServiceV2) scheduleTasks(s *models.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { +func (svc *ServiceV2) scheduleTasks(s *models2.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { // main task - mainTask := &models.TaskV2{ + mainTask := &models2.TaskV2{ SpiderId: s.Id, Mode: opts.Mode, NodeIds: opts.NodeIds, @@ -83,7 +83,7 @@ func (svc *ServiceV2) scheduleTasks(s *models.SpiderV2, opts *interfaces.SpiderR return nil, err } for _, nodeId := range nodeIds { - t := &models.TaskV2{ + t := &models2.TaskV2{ SpiderId: s.Id, Mode: opts.Mode, Cmd: opts.Cmd, @@ -127,7 +127,7 @@ func (svc *ServiceV2) getNodeIds(opts *interfaces.SpiderRunOptions) (nodeIds []p "enabled": true, "status": constants.NodeStatusOnline, } - nodes, err := service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil) + nodes, err := service.NewModelServiceV2[models2.NodeV2]().GetMany(query, nil) if err != nil { return nil, err } @@ -147,7 +147,7 @@ func (svc *ServiceV2) isMultiTask(opts *interfaces.SpiderRunOptions) (res bool) "enabled": true, "status": constants.NodeStatusOnline, } - nodes, err := service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil) + nodes, err := service.NewModelServiceV2[models2.NodeV2]().GetMany(query, nil) if err != nil { trace.PrintError(err) return false diff --git a/core/system/service_v2.go b/core/system/service_v2.go index a893c9d9..d82a6048 100644 --- a/core/system/service_v2.go +++ b/core/system/service_v2.go @@ -1,7 +1,7 @@ package system import ( - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "go.mongodb.org/mongo-driver/bson" "sync" diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go index 885b1c01..7fec5e18 100644 --- a/core/task/handler/runner_v2.go +++ b/core/task/handler/runner_v2.go @@ -15,6 +15,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/client" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" service2 "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/sys_exec" "github.com/crawlab-team/crawlab/core/utils" @@ -47,8 +48,8 @@ type RunnerV2 struct { cmd *exec.Cmd // process command instance pid int // process id tid primitive.ObjectID // task id - t *models.TaskV2 // task model.Task - s *models.SpiderV2 // spider model.Spider + t *models2.TaskV2 // task model.Task + s *models2.SpiderV2 // spider model.Spider ch chan constants.TaskSignal // channel to communicate between Service and RunnerV2 err error // standard process error envs []models.Env // environment variables @@ -315,7 +316,7 @@ func (r *RunnerV2) configureEnv() { } // global environment variables - envs, err := client.NewModelServiceV2[models.EnvironmentV2]().GetMany(nil, nil) + envs, err := client.NewModelServiceV2[models2.EnvironmentV2]().GetMany(nil, nil) if err != nil { trace.PrintError(err) return @@ -510,12 +511,12 @@ func (r *RunnerV2) updateTask(status string, e error) (err error) { r.t.Error = e.Error() } if r.svc.GetNodeConfigService().IsMaster() { - err = service2.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t) + err = service2.NewModelServiceV2[models2.TaskV2]().ReplaceById(r.t.Id, *r.t) if err != nil { return err } } else { - err = client.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t) + err = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(r.t.Id, *r.t) if err != nil { return err } @@ -568,7 +569,7 @@ func (r *RunnerV2) writeLogLines(lines []string) { } func (r *RunnerV2) _updateTaskStat(status string) { - ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid) + ts, err := client.NewModelServiceV2[models2.TaskStatV2]().GetById(r.tid) if err != nil { trace.PrintError(err) return @@ -589,13 +590,13 @@ func (r *RunnerV2) _updateTaskStat(status string) { ts.TotalDuration = ts.EndTs.Sub(ts.CreateTs).Milliseconds() } if r.svc.GetNodeConfigService().IsMaster() { - err = service2.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts) + err = service2.NewModelServiceV2[models2.TaskStatV2]().ReplaceById(ts.Id, *ts) if err != nil { trace.PrintError(err) return } } else { - err = client.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts) + err = client.NewModelServiceV2[models2.TaskStatV2]().ReplaceById(ts.Id, *ts) if err != nil { trace.PrintError(err) return @@ -622,7 +623,7 @@ func (r *RunnerV2) sendNotification() { func (r *RunnerV2) _updateSpiderStat(status string) { // task stat - ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid) + ts, err := client.NewModelServiceV2[models2.TaskStatV2]().GetById(r.tid) if err != nil { trace.PrintError(err) return @@ -660,13 +661,13 @@ func (r *RunnerV2) _updateSpiderStat(status string) { // perform update if r.svc.GetNodeConfigService().IsMaster() { - err = service2.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update) + err = service2.NewModelServiceV2[models2.SpiderStatV2]().UpdateById(r.s.Id, update) if err != nil { trace.PrintError(err) return } } else { - err = client.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update) + err = client.NewModelServiceV2[models2.SpiderStatV2]().UpdateById(r.s.Id, update) if err != nil { trace.PrintError(err) return diff --git a/core/task/handler/service_v2.go b/core/task/handler/service_v2.go index e6edb2b2..be7c2725 100644 --- a/core/task/handler/service_v2.go +++ b/core/task/handler/service_v2.go @@ -10,7 +10,7 @@ import ( grpcclient "github.com/crawlab-team/crawlab/core/grpc/client" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/trace" @@ -121,7 +121,7 @@ func (svc *ServiceV2) Fetch() { t.Error = err.Error() t.Status = constants.TaskStatusError t.SetUpdated(t.CreatedBy) - _ = client.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t) + _ = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t) continue } continue @@ -202,15 +202,15 @@ func (svc *ServiceV2) GetNodeConfigService() (cfgSvc interfaces.NodeConfigServic return svc.cfgSvc } -func (svc *ServiceV2) GetCurrentNode() (n *models.NodeV2, err error) { +func (svc *ServiceV2) GetCurrentNode() (n *models2.NodeV2, err error) { // node key nodeKey := svc.cfgSvc.GetNodeKey() // current node if svc.cfgSvc.IsMaster() { - n, err = service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + n, err = service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) } else { - n, err = client.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + n, err = client.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) } if err != nil { return nil, err @@ -219,11 +219,11 @@ func (svc *ServiceV2) GetCurrentNode() (n *models.NodeV2, err error) { return n, nil } -func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models.TaskV2, err error) { +func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models2.TaskV2, err error) { if svc.cfgSvc.IsMaster() { - t, err = service.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err = service.NewModelServiceV2[models2.TaskV2]().GetById(id) } else { - t, err = client.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err = client.NewModelServiceV2[models2.TaskV2]().GetById(id) } if err != nil { return nil, err @@ -232,11 +232,11 @@ func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models.TaskV2, err return t, nil } -func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models.SpiderV2, err error) { +func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models2.SpiderV2, err error) { if svc.cfgSvc.IsMaster() { - s, err = service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err = service.NewModelServiceV2[models2.SpiderV2]().GetById(id) } else { - s, err = client.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err = client.NewModelServiceV2[models2.SpiderV2]().GetById(id) } if err != nil { return nil, err @@ -267,13 +267,13 @@ func (svc *ServiceV2) getRunnerCount() (count int) { "status": constants.TaskStatusRunning, } if svc.cfgSvc.IsMaster() { - count, err = service.NewModelServiceV2[models.TaskV2]().Count(query) + count, err = service.NewModelServiceV2[models2.TaskV2]().Count(query) if err != nil { trace.PrintError(err) return } } else { - count, err = client.NewModelServiceV2[models.TaskV2]().Count(query) + count, err = client.NewModelServiceV2[models2.TaskV2]().Count(query) if err != nil { trace.PrintError(err) return @@ -323,9 +323,9 @@ func (svc *ServiceV2) reportStatus() (err error) { // save node n.SetUpdated(n.CreatedBy) if svc.cfgSvc.IsMaster() { - err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(n.Id, *n) + err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(n.Id, *n) } else { - err = client.NewModelServiceV2[models.NodeV2]().ReplaceById(n.Id, *n) + err = client.NewModelServiceV2[models2.NodeV2]().ReplaceById(n.Id, *n) } if err != nil { return err diff --git a/core/task/scheduler/service_v2.go b/core/task/scheduler/service_v2.go index 740bf3e3..81d5907e 100644 --- a/core/task/scheduler/service_v2.go +++ b/core/task/scheduler/service_v2.go @@ -7,7 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/handler" @@ -36,40 +36,40 @@ func (svc *ServiceV2) Start() { utils.DefaultWait() } -func (svc *ServiceV2) Enqueue(t *models.TaskV2, by primitive.ObjectID) (t2 *models.TaskV2, err error) { +func (svc *ServiceV2) Enqueue(t *models2.TaskV2, by primitive.ObjectID) (t2 *models2.TaskV2, err error) { // set task status t.Status = constants.TaskStatusPending t.SetCreatedBy(by) t.SetUpdated(by) // add task - taskModelSvc := service.NewModelServiceV2[models.TaskV2]() + taskModelSvc := service.NewModelServiceV2[models2.TaskV2]() _, err = taskModelSvc.InsertOne(*t) if err != nil { return nil, err } // task queue item - tq := models.TaskQueueItemV2{ + tq := models2.TaskQueueItemV2{ Priority: t.Priority, NodeId: t.NodeId, } tq.SetId(t.Id) // task stat - ts := models.TaskStatV2{ + ts := models2.TaskStatV2{ CreateTs: time.Now(), } ts.SetId(t.Id) // enqueue task - _, err = service.NewModelServiceV2[models.TaskQueueItemV2]().InsertOne(tq) + _, err = service.NewModelServiceV2[models2.TaskQueueItemV2]().InsertOne(tq) if err != nil { return nil, trace.TraceError(err) } // add task stat - _, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(ts) + _, err = service.NewModelServiceV2[models2.TaskStatV2]().InsertOne(ts) if err != nil { return nil, trace.TraceError(err) } @@ -80,7 +80,7 @@ func (svc *ServiceV2) Enqueue(t *models.TaskV2, by primitive.ObjectID) (t2 *mode func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err error) { // task - t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id) if err != nil { return trace.TraceError(err) } @@ -95,7 +95,7 @@ func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err // set status of pending tasks as "cancelled" and remove from task item queue if initialStatus == constants.TaskStatusPending { // remove from task item queue - if err := service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteById(t.Id); err != nil { + if err := service.NewModelServiceV2[models2.TaskQueueItemV2]().DeleteById(t.Id); err != nil { return trace.TraceError(err) } return nil @@ -110,7 +110,7 @@ func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err } // node - n, err := service.NewModelServiceV2[models.NodeV2]().GetById(t.NodeId) + n, err := service.NewModelServiceV2[models2.NodeV2]().GetById(t.NodeId) if err != nil { return trace.TraceError(err) } @@ -136,22 +136,22 @@ func (svc *ServiceV2) SetInterval(interval time.Duration) { svc.interval = interval } -func (svc *ServiceV2) SaveTask(t *models.TaskV2, by primitive.ObjectID) (err error) { +func (svc *ServiceV2) SaveTask(t *models2.TaskV2, by primitive.ObjectID) (err error) { if t.Id.IsZero() { t.SetCreated(by) t.SetUpdated(by) - _, err = service.NewModelServiceV2[models.TaskV2]().InsertOne(*t) + _, err = service.NewModelServiceV2[models2.TaskV2]().InsertOne(*t) return err } else { t.SetUpdated(by) - return service.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t) + return service.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t) } } // initTaskStatus initialize task status of existing tasks func (svc *ServiceV2) initTaskStatus() { // set status of running tasks as TaskStatusAbnormal - runningTasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{ + runningTasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(bson.M{ "status": bson.M{ "$in": []string{ constants.TaskStatusPending, @@ -166,23 +166,23 @@ func (svc *ServiceV2) initTaskStatus() { trace.PrintError(err) } for _, t := range runningTasks { - go func(t *models.TaskV2) { + go func(t *models2.TaskV2) { t.Status = constants.TaskStatusAbnormal if err := svc.SaveTask(t, primitive.NilObjectID); err != nil { trace.PrintError(err) } }(&t) } - if err := service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteMany(nil); err != nil { + if err := service.NewModelServiceV2[models2.TaskQueueItemV2]().DeleteMany(nil); err != nil { return } } -func (svc *ServiceV2) isMasterNode(t *models.TaskV2) (ok bool, err error) { +func (svc *ServiceV2) isMasterNode(t *models2.TaskV2) (ok bool, err error) { if t.NodeId.IsZero() { return false, trace.TraceError(errors.ErrorTaskNoNodeId) } - n, err := service.NewModelServiceV2[models.NodeV2]().GetById(t.NodeId) + n, err := service.NewModelServiceV2[models2.NodeV2]().GetById(t.NodeId) if err != nil { if errors2.Is(err, mongo2.ErrNoDocuments) { return false, trace.TraceError(errors.ErrorTaskNodeNotFound) @@ -195,7 +195,7 @@ func (svc *ServiceV2) isMasterNode(t *models.TaskV2) (ok bool, err error) { func (svc *ServiceV2) cleanupTasks() { for { // task stats over 30 days ago - taskStats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(bson.M{ + taskStats, err := service.NewModelServiceV2[models2.TaskStatV2]().GetMany(bson.M{ "create_ts": bson.M{ "$lt": time.Now().Add(-30 * 24 * time.Hour), }, @@ -213,14 +213,14 @@ func (svc *ServiceV2) cleanupTasks() { if len(ids) > 0 { // remove tasks - if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{ + if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{ "_id": bson.M{"$in": ids}, }); err != nil { trace.PrintError(err) } // remove task stats - if err := service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{ + if err := service.NewModelServiceV2[models2.TaskStatV2]().DeleteMany(bson.M{ "_id": bson.M{"$in": ids}, }); err != nil { trace.PrintError(err) diff --git a/core/task/stats/service_v2.go b/core/task/stats/service_v2.go index fc39cbd5..0637e1e1 100644 --- a/core/task/stats/service_v2.go +++ b/core/task/stats/service_v2.go @@ -2,7 +2,7 @@ package stats import ( "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/result" @@ -63,7 +63,7 @@ func (svc *ServiceV2) getResultService(id primitive.ObjectID) (resultSvc interfa } // task - t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id) if err != nil { return nil, err } @@ -81,7 +81,7 @@ func (svc *ServiceV2) getResultService(id primitive.ObjectID) (resultSvc interfa } func (svc *ServiceV2) updateTaskStats(id primitive.ObjectID, resultCount int) { - err := service.NewModelServiceV2[models.TaskStatV2]().UpdateById(id, bson.M{ + err := service.NewModelServiceV2[models2.TaskStatV2]().UpdateById(id, bson.M{ "$inc": bson.M{ "result_count": resultCount, }, diff --git a/core/user/service_v2.go b/core/user/service_v2.go index 7784003c..c0ab759c 100644 --- a/core/user/service_v2.go +++ b/core/user/service_v2.go @@ -4,7 +4,7 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" mongo2 "github.com/crawlab-team/crawlab/db/mongo" diff --git a/core/utils/cockroachdb.go b/core/utils/cockroachdb.go index feccc5ec..3534a1de 100644 --- a/core/utils/cockroachdb.go +++ b/core/utils/cockroachdb.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/mssql" "time" @@ -59,13 +60,13 @@ func getCockroachdbSession(ctx context.Context, ds *models.DataSource) (s db.Ses return s, err } -func GetCockroachdbSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetCockroachdbSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getCockroachdbSessionV2(ctx, ds) } -func getCockroachdbSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { +func getCockroachdbSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/es.go b/core/utils/es.go index 7723205b..d2db5e56 100644 --- a/core/utils/es.go +++ b/core/utils/es.go @@ -8,6 +8,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/db/generic" "github.com/crawlab-team/crawlab/trace" "github.com/elastic/go-elasticsearch/v8" @@ -83,13 +84,13 @@ func getElasticsearchClient(ctx context.Context, ds *models.DataSource) (c *elas return c, err } -func GetElasticsearchClientWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *elasticsearch.Client, err error) { +func GetElasticsearchClientWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *elasticsearch.Client, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getElasticsearchClientV2(ctx, ds) } -func getElasticsearchClientV2(ctx context.Context, ds *models.DataSourceV2) (c *elasticsearch.Client, err error) { +func getElasticsearchClientV2(ctx context.Context, ds *models2.DataSourceV2) (c *elasticsearch.Client, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/kafka.go b/core/utils/kafka.go index e3360392..e47d2107 100644 --- a/core/utils/kafka.go +++ b/core/utils/kafka.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/segmentio/kafka-go" "time" ) @@ -40,13 +41,13 @@ func getKafkaConnection(ctx context.Context, ds *models.DataSource) (c *kafka.Co return kafka.DialLeader(ctx, network, address, topic, partition) } -func GetKafkaConnectionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *kafka.Conn, err error) { +func GetKafkaConnectionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *kafka.Conn, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getKafkaConnectionV2(ctx, ds) } -func getKafkaConnectionV2(ctx context.Context, ds *models.DataSourceV2) (c *kafka.Conn, err error) { +func getKafkaConnectionV2(ctx context.Context, ds *models2.DataSourceV2) (c *kafka.Conn, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/mongo.go b/core/utils/mongo.go index 23e92541..a2fd3b03 100644 --- a/core/utils/mongo.go +++ b/core/utils/mongo.go @@ -4,6 +4,7 @@ import ( "context" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/db/generic" "github.com/crawlab-team/crawlab/db/mongo" "go.mongodb.org/mongo-driver/bson" @@ -54,7 +55,7 @@ func GetMongoClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c return getMongoClient(ctx, ds) } -func GetMongoClientWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *mongo2.Client, err error) { +func GetMongoClientWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *mongo2.Client, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getMongoClientV2(ctx, ds) @@ -99,7 +100,7 @@ func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Clien return mongo.GetMongoClient(opts...) } -func getMongoClientV2(ctx context.Context, ds *models.DataSourceV2) (c *mongo2.Client, err error) { +func getMongoClientV2(ctx context.Context, ds *models2.DataSourceV2) (c *mongo2.Client, err error) { // normalize settings if ds.Host == "" { ds.Host = constants.DefaultHost diff --git a/core/utils/mssql.go b/core/utils/mssql.go index 03329eae..7333706b 100644 --- a/core/utils/mssql.go +++ b/core/utils/mssql.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/mssql" "time" @@ -59,13 +60,13 @@ func getMssqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, return s, err } -func GetMssqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetMssqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getMssqlSessionV2(ctx, ds) } -func getMssqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { +func getMssqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/mysql.go b/core/utils/mysql.go index c1e78a20..4ea293cf 100644 --- a/core/utils/mysql.go +++ b/core/utils/mysql.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/mysql" "time" @@ -59,13 +60,13 @@ func getMysqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, return s, err } -func GetMysqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetMysqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getMysqlSessionV2(ctx, ds) } -func getMysqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { +func getMysqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/postgresql.go b/core/utils/postgresql.go index cf3ba7dc..a3086762 100644 --- a/core/utils/postgresql.go +++ b/core/utils/postgresql.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/postgresql" "time" @@ -59,13 +60,13 @@ func getPostgresqlSession(ctx context.Context, ds *models.DataSource) (s db.Sess return s, err } -func GetPostgresqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetPostgresqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getPostgresqlSessionV2(ctx, ds) } -func getPostgresqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { +func getPostgresqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/sqlite.go b/core/utils/sqlite.go index 83a06cbc..b8e87786 100644 --- a/core/utils/sqlite.go +++ b/core/utils/sqlite.go @@ -3,6 +3,7 @@ package utils import ( "context" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/sqlite" "time" @@ -44,13 +45,13 @@ func getSqliteSession(ctx context.Context, ds *models.DataSource) (s db.Session, return s, err } -func GetSqliteSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetSqliteSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getSqliteSessionV2(ctx, ds) } -func getSqliteSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { +func getSqliteSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { // connect settings settings := sqlite.ConnectionURL{ Database: ds.Database,