git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-23 17:31:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: removed unnecessary code

Browse Source
This commit is contained in:
Marvin Zhang
2024-07-11 12:45:29 +08:00
parent 01ced1ec45
commit ce8ff0ce2f
57 changed files with 423 additions and 430 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
core/controllers/base_v2.go
View File

@@ -11,6 +11,12 @@ import (
mongo2 "go.mongodb.org/mongo-driver/mongo"
)
type Action struct {
Method string
Path string
HandlerFunc gin.HandlerFunc
}
type BaseControllerV2[T any] struct {
modelSvc *service.ModelServiceV2[T]
actions []Action

4
core/controllers/base_v2_test.go
View File

@@ -6,7 +6,7 @@ import (
"encoding/json"
"github.com/crawlab-team/crawlab/core/controllers"
"github.com/crawlab-team/crawlab/core/middlewares"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/user"
"github.com/spf13/viper"
@@ -24,7 +24,7 @@ func init() {
}
// TestModel is a simple struct to be used as a model in tests
type TestModel models.TestModel
type TestModel models.TestModelV2
var TestToken string

2
core/controllers/spider_v2_test.go
View File

@@ -5,7 +5,7 @@ import (
"encoding/json"
"github.com/crawlab-team/crawlab/core/controllers"
"github.com/crawlab-team/crawlab/core/middlewares"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"

2
core/ds/service_v2.go
View File

@@ -4,7 +4,7 @@ import (
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/constants"
constants2 "github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/result"
"github.com/crawlab-team/crawlab/core/utils"

28
core/grpc/server/dependencies_server_v2.go
View File

@@ -4,7 +4,7 @@ import (
"context"
"errors"
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
mongo2 "github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/grpc"
@@ -40,12 +40,12 @@ func (svr DependenciesServerV2) Connect(req *grpc.DependenciesServiceV2ConnectRe
}
func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.DependenciesServiceV2SyncRequest) (response *grpc.Response, err error) {
n, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": request.NodeKey}, nil)
n, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": request.NodeKey}, nil)
if err != nil {
return nil, err
}
depsDb, err := service.NewModelServiceV2[models.DependencyV2]().GetMany(bson.M{
depsDb, err := service.NewModelServiceV2[models2.DependencyV2]().GetMany(bson.M{
"node_id": n.Id,
"type": request.Lang,
}, nil)
@@ -56,15 +56,15 @@ func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.Dependen
}
}
depsDbMap := make(map[string]*models.DependencyV2)
depsDbMap := make(map[string]*models2.DependencyV2)
for _, d := range depsDb {
depsDbMap[d.Name] = &d
}
var depsToInsert []models.DependencyV2
depsMap := make(map[string]*models.DependencyV2)
var depsToInsert []models2.DependencyV2
depsMap := make(map[string]*models2.DependencyV2)
for _, dep := range request.Dependencies {
d := models.DependencyV2{
d := models2.DependencyV2{
Name: dep.Name,
NodeId: n.Id,
Type: request.Lang,
@@ -90,7 +90,7 @@ func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.Dependen
err = mongo2.RunTransaction(func(ctx mongo.SessionContext) (err error) {
if len(depIdsToDelete) > 0 {
err = service.NewModelServiceV2[models.DependencyV2]().DeleteMany(bson.M{
err = service.NewModelServiceV2[models2.DependencyV2]().DeleteMany(bson.M{
"_id": bson.M{"$in": depIdsToDelete},
})
if err != nil {
@@ -101,7 +101,7 @@ func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.Dependen
}
if len(depsToInsert) > 0 {
_, err = service.NewModelServiceV2[models.DependencyV2]().InsertMany(depsToInsert)
_, err = service.NewModelServiceV2[models2.DependencyV2]().InsertMany(depsToInsert)
if err != nil {
log.Errorf("[DependenciesServerV2] insert dependencies in db error: %v", err)
trace.PrintError(err)
@@ -116,7 +116,7 @@ func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.Dependen
}
func (svr DependenciesServerV2) UpdateTaskLog(stream grpc.DependenciesServiceV2_UpdateTaskLogServer) (err error) {
var t *models.DependencyTaskV2
var t *models2.DependencyTaskV2
for {
req, err := stream.Recv()
if err == io.EOF {
@@ -131,21 +131,21 @@ func (svr DependenciesServerV2) UpdateTaskLog(stream grpc.DependenciesServiceV2_
return err
}
if t == nil {
t, err = service.NewModelServiceV2[models.DependencyTaskV2]().GetById(taskId)
t, err = service.NewModelServiceV2[models2.DependencyTaskV2]().GetById(taskId)
if err != nil {
return err
}
}
var logs []models.DependencyLogV2
var logs []models2.DependencyLogV2
for _, line := range req.LogLines {
l := models.DependencyLogV2{
l := models2.DependencyLogV2{
TaskId: taskId,
Content: line,
}
l.SetCreated(t.CreatedBy)
logs = append(logs, l)
}
_, err = service.NewModelServiceV2[models.DependencyLogV2]().InsertMany(logs)
_, err = service.NewModelServiceV2[models2.DependencyLogV2]().InsertMany(logs)
if err != nil {
return err
}

8
core/grpc/server/metrics_server_v2.go
View File

@@ -3,7 +3,7 @@ package server
import (
"context"
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/grpc"
"go.mongodb.org/mongo-driver/bson"
@@ -17,12 +17,12 @@ type MetricsServerV2 struct {
func (svr MetricsServerV2) Send(_ context.Context, req *grpc.MetricsServiceV2SendRequest) (res *grpc.Response, err error) {
log.Info("[MetricsServerV2] received metric from node: " + req.NodeKey)
n, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil)
n, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil)
if err != nil {
log.Errorf("[MetricsServerV2] error getting node: %v", err)
return HandleError(err)
}
metric := models.MetricV2{
metric := models2.MetricV2{
Type: req.Type,
NodeId: n.Id,
CpuUsagePercent: req.CpuUsagePercent,
@@ -40,7 +40,7 @@ func (svr MetricsServerV2) Send(_ context.Context, req *grpc.MetricsServiceV2Sen
NetworkBytesRecvRate: req.NetworkBytesRecvRate,
}
metric.CreatedAt = time.Unix(req.Timestamp, 0)
_, err = service.NewModelServiceV2[models.MetricV2]().InsertOne(metric)
_, err = service.NewModelServiceV2[models2.MetricV2]().InsertOne(metric)
if err != nil {
log.Errorf("[MetricsServerV2] error inserting metric: %v", err)
return HandleError(err)

56
core/grpc/server/model_base_service_v2_server.go
View File

@@ -3,10 +3,10 @@ package server
import (
"context"
"encoding/json"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/crawlab/grpc"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"reflect"
@@ -16,32 +16,32 @@ var (
typeNameColNameMap = make(map[string]string)
typeOneNameModelMap = make(map[string]any)
typeOneInstances = []any{
*new(models.TestModel),
*new(models.DataCollectionV2),
*new(models.DataSourceV2),
*new(models.DependencyV2),
*new(models.DependencyLogV2),
*new(models.DependencySettingV2),
*new(models.DependencyTaskV2),
*new(models.EnvironmentV2),
*new(models.GitV2),
*new(models.MetricV2),
*new(models.NodeV2),
*new(models.NotificationSettingV2),
*new(models.PermissionV2),
*new(models.ProjectV2),
*new(models.RolePermissionV2),
*new(models.RoleV2),
*new(models.ScheduleV2),
*new(models.SettingV2),
*new(models.SpiderV2),
*new(models.SpiderStatV2),
*new(models.TaskQueueItemV2),
*new(models.TaskStatV2),
*new(models.TaskV2),
*new(models.TokenV2),
*new(models.UserRoleV2),
*new(models.UserV2),
*new(models2.TestModelV2),
*new(models2.DataCollectionV2),
*new(models2.DataSourceV2),
*new(models2.DependencyV2),
*new(models2.DependencyLogV2),
*new(models2.DependencySettingV2),
*new(models2.DependencyTaskV2),
*new(models2.EnvironmentV2),
*new(models2.GitV2),
*new(models2.MetricV2),
*new(models2.NodeV2),
*new(models2.NotificationSettingV2),
*new(models2.PermissionV2),
*new(models2.ProjectV2),
*new(models2.RolePermissionV2),
*new(models2.RoleV2),
*new(models2.ScheduleV2),
*new(models2.SettingV2),
*new(models2.SpiderV2),
*new(models2.SpiderStatV2),
*new(models2.TaskQueueItemV2),
*new(models2.TaskStatV2),
*new(models2.TaskV2),
*new(models2.TokenV2),
*new(models2.UserRoleV2),
*new(models2.UserV2),
}
)

2
core/grpc/server/node_server_v2.go
View File

@@ -7,7 +7,7 @@ import (
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
"github.com/crawlab-team/crawlab/grpc"

18
core/grpc/server/task_server_v2.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
"github.com/crawlab-team/crawlab/core/notification"
@@ -72,7 +72,7 @@ func (svr TaskServerV2) Fetch(ctx context.Context, request *grpc.Request) (respo
if nodeKey == "" {
return nil, errors.New("invalid node key")
}
n, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
n, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
if err != nil {
return nil, trace.TraceError(err)
}
@@ -111,11 +111,11 @@ func (svr TaskServerV2) Fetch(ctx context.Context, request *grpc.Request) (respo
func (svr TaskServerV2) SendNotification(ctx context.Context, request *grpc.Request) (response *grpc.Response, err error) {
svc := notification.GetNotificationServiceV2()
var t = new(models.TaskV2)
var t = new(models2.TaskV2)
if err := json.Unmarshal(request.Data, t); err != nil {
return nil, trace.TraceError(err)
}
t, err = service.NewModelServiceV2[models.TaskV2]().GetById(t.Id)
t, err = service.NewModelServiceV2[models2.TaskV2]().GetById(t.Id)
if err != nil {
return nil, trace.TraceError(err)
}
@@ -127,7 +127,7 @@ func (svr TaskServerV2) SendNotification(ctx context.Context, request *grpc.Requ
if err := json.Unmarshal(td, &e); err != nil {
return nil, trace.TraceError(err)
}
ts, err := service.NewModelServiceV2[models.TaskStatV2]().GetById(t.Id)
ts, err := service.NewModelServiceV2[models2.TaskStatV2]().GetById(t.Id)
if err != nil {
return nil, trace.TraceError(err)
}
@@ -190,22 +190,22 @@ func (svr TaskServerV2) handleInsertLogs(msg *grpc.StreamMessage) (err error) {
}
func (svr TaskServerV2) getTaskQueueItemIdAndDequeue(query bson.M, opts *mongo.FindOptions, nid primitive.ObjectID) (tid primitive.ObjectID, err error) {
tq, err := service.NewModelServiceV2[models.TaskQueueItemV2]().GetOne(query, opts)
tq, err := service.NewModelServiceV2[models2.TaskQueueItemV2]().GetOne(query, opts)
if err != nil {
if errors.Is(err, mongo2.ErrNoDocuments) {
return tid, nil
}
return tid, trace.TraceError(err)
}
t, err := service.NewModelServiceV2[models.TaskV2]().GetById(tq.Id)
t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(tq.Id)
if err == nil {
t.NodeId = nid
err = service.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t)
err = service.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t)
if err != nil {
return tid, trace.TraceError(err)
}
}
err = service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteById(tq.Id)
err = service.NewModelServiceV2[models2.TaskQueueItemV2]().DeleteById(tq.Id)
if err != nil {
return tid, trace.TraceError(err)
}

4
core/models/client/model_service_v2_test.go
View File

@@ -4,7 +4,7 @@ import (
"context"
"github.com/crawlab-team/crawlab/core/grpc/server"
"github.com/crawlab-team/crawlab/core/models/client"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/spf13/viper"
@@ -17,7 +17,7 @@ import (
"time"
)
type TestModel models.TestModel
type TestModel models.TestModelV2
func setupTestDB() {
viper.Set("mongo.db", "testdb")

44
core/models/common/index_service_v2.go
View File

@@ -1,7 +1,7 @@
package common
import (
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
@@ -11,7 +11,7 @@ import (
func CreateIndexesV2() {
// nodes
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.NodeV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NodeV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"key": 1}}, // key
{Keys: bson.M{"name": 1}}, // name
{Keys: bson.M{"is_master": 1}}, // is_master
@@ -21,12 +21,12 @@ func CreateIndexesV2() {
})
// projects
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.ProjectV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ProjectV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// spiders
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.SpiderV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SpiderV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
{Keys: bson.M{"type": 1}},
{Keys: bson.M{"col_id": 1}},
@@ -34,7 +34,7 @@ func CreateIndexesV2() {
})
// tasks
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"spider_id": 1}},
{Keys: bson.M{"status": 1}},
{Keys: bson.M{"node_id": 1}},
@@ -48,73 +48,73 @@ func CreateIndexesV2() {
})
// task stats
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TaskStatV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskStatV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"create_ts": 1}},
})
// schedules
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.ScheduleV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ScheduleV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
{Keys: bson.M{"spider_id": 1}},
{Keys: bson.M{"enabled": 1}},
})
// users
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.UserV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"username": 1}},
{Keys: bson.M{"role": 1}},
{Keys: bson.M{"email": 1}},
})
// settings
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"key": 1}},
})
// tokens
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TokenV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TokenV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// variables
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.VariableV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.VariableV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"key": 1}},
})
// data sources
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DataSourceV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataSourceV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// data collections
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DataCollectionV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataCollectionV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// roles
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.RoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)},
})
// user role relations
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.UserRoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserRoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.D{{"user_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)},
{Keys: bson.D{{"role_id", 1}, {"user_id", 1}}, Options: options.Index().SetUnique(true)},
})
// permissions
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.PermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.PermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)},
})
// role permission relations
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.RolePermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RolePermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.D{{"role_id", 1}, {"permission_id", 1}}, Options: options.Index().SetUnique(true)},
{Keys: bson.D{{"permission_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)},
})
// dependencies
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{
{"type", 1},
@@ -126,7 +126,7 @@ func CreateIndexesV2() {
})
// dependency settings
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencySettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencySettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{
{"type", 1},
@@ -138,7 +138,7 @@ func CreateIndexesV2() {
})
// dependency logs
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyLogV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyLogV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{{"task_id", 1}},
},
@@ -149,7 +149,7 @@ func CreateIndexesV2() {
})
// dependency tasks
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyTaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyTaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{
{"update_ts", 1},
@@ -159,7 +159,7 @@ func CreateIndexesV2() {
})
// metrics
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.MetricV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.MetricV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{
{"created_ts", -1},

10
core/models/models/v2/data_collection_v2.go
View File

@@ -5,11 +5,11 @@ import (
)
type DataCollectionV2 struct {
any `collection:"data_collections"`
BaseModelV2[DataCollection] `bson:",inline"`
Name string `json:"name" bson:"name"`
Fields []entity.DataField `json:"fields" bson:"fields"`
Dedup struct {
any `collection:"data_collections"`
BaseModelV2[DataCollectionV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Fields []entity.DataField `json:"fields" bson:"fields"`
Dedup struct {
Enabled bool `json:"enabled" bson:"enabled"`
Keys []string `json:"keys" bson:"keys"`
Type string `json:"type" bson:"type"`

32
core/models/models/v2/data_source_v2.go
View File

@@ -1,20 +1,20 @@
package models
type DataSourceV2 struct {
any `collection:"data_sources"`
BaseModelV2[DataSource] `bson:",inline"`
Name string `json:"name" bson:"name"`
Type string `json:"type" bson:"type"`
Description string `json:"description" bson:"description"`
Host string `json:"host" bson:"host"`
Port string `json:"port" bson:"port"`
Url string `json:"url" bson:"url"`
Hosts []string `json:"hosts" bson:"hosts"`
Database string `json:"database" bson:"database"`
Username string `json:"username" bson:"username"`
Password string `json:"-,omitempty" bson:"password"`
ConnectType string `json:"connect_type" bson:"connect_type"`
Status string `json:"status" bson:"status"`
Error string `json:"error" bson:"error"`
Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"`
any `collection:"data_sources"`
BaseModelV2[DataSourceV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Type string `json:"type" bson:"type"`
Description string `json:"description" bson:"description"`
Host string `json:"host" bson:"host"`
Port string `json:"port" bson:"port"`
Url string `json:"url" bson:"url"`
Hosts []string `json:"hosts" bson:"hosts"`
Database string `json:"database" bson:"database"`
Username string `json:"username" bson:"username"`
Password string `json:"-,omitempty" bson:"password"`
ConnectType string `json:"connect_type" bson:"connect_type"`
Status string `json:"status" bson:"status"`
Error string `json:"error" bson:"error"`
Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"`
}

18
core/models/models/v2/dependency_setting_v2.go
View File

@@ -5,13 +5,13 @@ import (
)
type DependencySettingV2 struct {
any `collection:"dependency_settings"`
BaseModelV2[DependencySetting] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Enabled bool `json:"enabled" bson:"enabled"`
Cmd string `json:"cmd" bson:"cmd"`
Proxy string `json:"proxy" bson:"proxy"`
LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"`
any `collection:"dependency_settings"`
BaseModelV2[DependencySettingV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Enabled bool `json:"enabled" bson:"enabled"`
Cmd string `json:"cmd" bson:"cmd"`
Proxy string `json:"proxy" bson:"proxy"`
LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"`
}

29
core/models/models/v2/git_v2.go
View File

@@ -1,26 +1,25 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/vcs"
"time"
)
type GitV2 struct {
any `collection:"gits"`
models.BaseModelV2[GitV2] `bson:",inline"`
Url string `json:"url" bson:"url"`
Name string `json:"name" bson:"name"`
AuthType string `json:"auth_type" bson:"auth_type"`
Username string `json:"username" bson:"username"`
Password string `json:"password" bson:"password"`
CurrentBranch string `json:"current_branch" bson:"current_branch"`
Status string `json:"status" bson:"status"`
Error string `json:"error" bson:"error"`
Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"`
Refs []vcs.GitRef `json:"refs" bson:"refs"`
RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"`
CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"`
any `collection:"gits"`
BaseModelV2[GitV2] `bson:",inline"`
Url string `json:"url" bson:"url"`
Name string `json:"name" bson:"name"`
AuthType string `json:"auth_type" bson:"auth_type"`
Username string `json:"username" bson:"username"`
Password string `json:"password" bson:"password"`
CurrentBranch string `json:"current_branch" bson:"current_branch"`
Status string `json:"status" bson:"status"`
Error string `json:"error" bson:"error"`
Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"`
Refs []vcs.GitRef `json:"refs" bson:"refs"`
RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"`
CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"`
// settings
AutoPull bool `json:"auto_pull" bson:"auto_pull"`

35
core/models/models/v2/metric_v2.go
View File

@@ -1,26 +1,25 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type MetricV2 struct {
any `collection:"metrics"`
models.BaseModelV2[MetricV2] `bson:",inline"`
Type string `json:"type" bson:"type"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"`
TotalMemory uint64 `json:"total_memory" bson:"total_memory"`
AvailableMemory uint64 `json:"available_memory" bson:"available_memory"`
UsedMemory uint64 `json:"used_memory" bson:"used_memory"`
UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"`
TotalDisk uint64 `json:"total_disk" bson:"total_disk"`
AvailableDisk uint64 `json:"available_disk" bson:"available_disk"`
UsedDisk uint64 `json:"used_disk" bson:"used_disk"`
UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"`
DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"`
DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"`
NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"`
NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"`
any `collection:"metrics"`
BaseModelV2[MetricV2] `bson:",inline"`
Type string `json:"type" bson:"type"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"`
TotalMemory uint64 `json:"total_memory" bson:"total_memory"`
AvailableMemory uint64 `json:"available_memory" bson:"available_memory"`
UsedMemory uint64 `json:"used_memory" bson:"used_memory"`
UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"`
TotalDisk uint64 `json:"total_disk" bson:"total_disk"`
AvailableDisk uint64 `json:"available_disk" bson:"available_disk"`
UsedDisk uint64 `json:"used_disk" bson:"used_disk"`
UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"`
DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"`
DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"`
NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"`
NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"`
}

33
core/models/models/v2/node_v2.go
View File

@@ -1,25 +1,24 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"time"
)
type NodeV2 struct {
any `collection:"nodes"`
models.BaseModelV2[NodeV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Ip string `json:"ip" bson:"ip"`
Port string `json:"port" bson:"port"`
Mac string `json:"mac" bson:"mac"`
Hostname string `json:"hostname" bson:"hostname"`
Description string `json:"description" bson:"description"`
IsMaster bool `json:"is_master" bson:"is_master"`
Status string `json:"status" bson:"status"`
Enabled bool `json:"enabled" bson:"enabled"`
Active bool `json:"active" bson:"active"`
ActiveAt time.Time `json:"active_at" bson:"active_ts"`
AvailableRunners int `json:"available_runners" bson:"available_runners"`
MaxRunners int `json:"max_runners" bson:"max_runners"`
any `collection:"nodes"`
BaseModelV2[NodeV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Ip string `json:"ip" bson:"ip"`
Port string `json:"port" bson:"port"`
Mac string `json:"mac" bson:"mac"`
Hostname string `json:"hostname" bson:"hostname"`
Description string `json:"description" bson:"description"`
IsMaster bool `json:"is_master" bson:"is_master"`
Status string `json:"status" bson:"status"`
Enabled bool `json:"enabled" bson:"enabled"`
Active bool `json:"active" bson:"active"`
ActiveAt time.Time `json:"active_at" bson:"active_ts"`
AvailableRunners int `json:"available_runners" bson:"available_runners"`
MaxRunners int `json:"max_runners" bson:"max_runners"`
}

3
core/models/models/v2/notification_channel_v2.go
View File

@@ -1 +1,4 @@
package models
type NotificationChannel struct {
}

26
core/models/models/v2/notification_setting_v2.go
View File

@@ -1,20 +1,18 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type NotificationSettingV2 struct {
any `collection:"notification_settings"`
models.BaseModelV2[NotificationSettingV2] `bson:",inline"`
Type string `json:"type" bson:"type"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Enabled bool `json:"enabled" bson:"enabled"`
Global bool `json:"global" bson:"global"`
Title string `json:"title,omitempty" bson:"title,omitempty"`
Template string `json:"template,omitempty" bson:"template,omitempty"`
TaskTrigger string `json:"task_trigger" bson:"task_trigger"`
Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"`
Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"`
any `collection:"notification_settings"`
BaseModelV2[NotificationSettingV2] `bson:",inline"`
Type string `json:"type" bson:"type"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Enabled bool `json:"enabled" bson:"enabled"`
Global bool `json:"global" bson:"global"`
Title string `json:"title,omitempty" bson:"title,omitempty"`
Template string `json:"template,omitempty" bson:"template,omitempty"`
TaskTrigger string `json:"task_trigger" bson:"task_trigger"`
Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"`
Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"`
}
type NotificationSettingMail struct {

20
core/models/models/v2/permission_v2.go
View File

@@ -1,15 +1,13 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type PermissionV2 struct {
any `collection:"permissions"`
models.BaseModelV2[PermissionV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Type string `json:"type" bson:"type"`
Target []string `json:"target" bson:"target"`
Allow []string `json:"allow" bson:"allow"`
Deny []string `json:"deny" bson:"deny"`
any `collection:"permissions"`
BaseModelV2[PermissionV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Type string `json:"type" bson:"type"`
Target []string `json:"target" bson:"target"`
Allow []string `json:"allow" bson:"allow"`
Deny []string `json:"deny" bson:"deny"`
}

12
core/models/models/v2/project_v2.go
View File

@@ -1,11 +1,9 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type ProjectV2 struct {
any `collection:"projects"`
models.BaseModelV2[ProjectV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Spiders int `json:"spiders" bson:"-"`
any `collection:"projects"`
BaseModelV2[ProjectV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Spiders int `json:"spiders" bson:"-"`
}

9
core/models/models/v2/role_permission_v2.go
View File

@@ -1,13 +1,12 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type RolePermissionV2 struct {
any `collection:"role_permissions"`
models.BaseModelV2[RolePermissionV2] `bson:",inline"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"`
any `collection:"role_permissions"`
BaseModelV2[RolePermissionV2] `bson:",inline"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"`
}

12
core/models/models/v2/role_v2.go
View File

@@ -1,11 +1,9 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type RoleV2 struct {
any `collection:"roles"`
models.BaseModelV2[RoleV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
any `collection:"roles"`
BaseModelV2[RoleV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
}

29
core/models/models/v2/schedule_v2.go
View File

@@ -1,24 +1,23 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/robfig/cron/v3"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type ScheduleV2 struct {
any `collection:"schedules"`
models.BaseModelV2[ScheduleV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Cron string `json:"cron" bson:"cron"`
EntryId cron.EntryID `json:"entry_id" bson:"entry_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
Priority int `json:"priority" bson:"priority"`
Enabled bool `json:"enabled" bson:"enabled"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
any `collection:"schedules"`
BaseModelV2[ScheduleV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Cron string `json:"cron" bson:"cron"`
EntryId cron.EntryID `json:"entry_id" bson:"entry_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
Priority int `json:"priority" bson:"priority"`
Enabled bool `json:"enabled" bson:"enabled"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
}

9
core/models/models/v2/setting_v2.go
View File

@@ -1,13 +1,12 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson"
)
type SettingV2 struct {
any `collection:"settings"`
models.BaseModelV2[SettingV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Value bson.M `json:"value" bson:"value"`
any `collection:"settings"`
BaseModelV2[SettingV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Value bson.M `json:"value" bson:"value"`
}

25
core/models/models/v2/spider_stat_v2.go
View File

@@ -1,21 +1,20 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type SpiderStatV2 struct {
any `collection:"spider_stats"`
models.BaseModelV2[SpiderStatV2] `bson:",inline"`
LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"`
LastTask *TaskV2 `json:"last_task,omitempty" bson:"-"`
Tasks int `json:"tasks" bson:"tasks"`
Results int `json:"results" bson:"results"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second
AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second
AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second
AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second
any `collection:"spider_stats"`
BaseModelV2[SpiderStatV2] `bson:",inline"`
LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"`
LastTask *TaskV2 `json:"last_task,omitempty" bson:"-"`
Tasks int `json:"tasks" bson:"tasks"`
Results int `json:"results" bson:"results"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second
AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second
AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second
AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second
}

31
core/models/models/v2/spider_v2.go
View File

@@ -1,26 +1,25 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type SpiderV2 struct {
any `collection:"spiders"`
models.BaseModelV2[SpiderV2] `bson:",inline"`
Name string `json:"name" bson:"name"` // spider name
Type string `json:"type" bson:"type"` // spider type
ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id
ColName string `json:"col_name,omitempty" bson:"-"` // data collection name
DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id
DataSource *models.DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source
Description string `json:"description" bson:"description"` // description
ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id
Mode string `json:"mode" bson:"mode"` // default Task.Mode
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds
GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id
GitRootPath string `json:"git_root_path" bson:"git_root_path"`
Git *GitV2 `json:"git,omitempty" bson:"-"`
any `collection:"spiders"`
BaseModelV2[SpiderV2] `bson:",inline"`
Name string `json:"name" bson:"name"` // spider name
Type string `json:"type" bson:"type"` // spider type
ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id
ColName string `json:"col_name,omitempty" bson:"-"` // data collection name
DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id
DataSource *DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source
Description string `json:"description" bson:"description"` // description
ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id
Mode string `json:"mode" bson:"mode"` // default Task.Mode
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds
GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id
GitRootPath string `json:"git_root_path" bson:"git_root_path"`
Git *GitV2 `json:"git,omitempty" bson:"-"`
// stats
Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"`

9
core/models/models/v2/task_queue_item_v2.go
View File

@@ -1,13 +1,12 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type TaskQueueItemV2 struct {
any `collection:"task_queue"`
models.BaseModelV2[TaskQueueItemV2] `bson:",inline"`
Priority int `json:"p" bson:"p"`
NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"`
any `collection:"task_queue"`
BaseModelV2[TaskQueueItemV2] `bson:",inline"`
Priority int `json:"p" bson:"p"`
NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"`
}

21
core/models/models/v2/task_stat_v2.go
View File

@@ -1,19 +1,18 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"time"
)
type TaskStatV2 struct {
any `collection:"task_stats"`
models.BaseModelV2[TaskStatV2] `bson:",inline"`
CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"`
StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"`
EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond
ResultCount int64 `json:"result_count" bson:"result_count"`
ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"`
any `collection:"task_stats"`
BaseModelV2[TaskStatV2] `bson:",inline"`
CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"`
StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"`
EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond
ResultCount int64 `json:"result_count" bson:"result_count"`
ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"`
}

43
core/models/models/v2/task_v2.go
View File

@@ -1,31 +1,30 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
"time"
)
type TaskV2 struct {
any `collection:"tasks"`
models.BaseModelV2[TaskV2] `bson:",inline"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Status string `json:"status" bson:"status"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Error string `json:"error" bson:"error"`
Pid int `json:"pid" bson:"pid"`
ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"`
Type string `json:"type" bson:"type"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"`
Priority int `json:"priority" bson:"priority"`
Stat *TaskStatV2 `json:"stat,omitempty" bson:"-"`
HasSub bool `json:"has_sub" json:"has_sub"`
SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"`
Spider *SpiderV2 `json:"spider,omitempty" bson:"-"`
UserId primitive.ObjectID `json:"-" bson:"-"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
any `collection:"tasks"`
BaseModelV2[TaskV2] `bson:",inline"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Status string `json:"status" bson:"status"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Error string `json:"error" bson:"error"`
Pid int `json:"pid" bson:"pid"`
ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"`
Type string `json:"type" bson:"type"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"`
Priority int `json:"priority" bson:"priority"`
Stat *TaskStatV2 `json:"stat,omitempty" bson:"-"`
HasSub bool `json:"has_sub" json:"has_sub"`
SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"`
Spider *SpiderV2 `json:"spider,omitempty" bson:"-"`
UserId primitive.ObjectID `json:"-" bson:"-"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
}

8
core/models/models/v2/test_v2.go
View File

@@ -1,7 +1,7 @@
package models
type TestModel struct {
any `collection:"testmodels"`
BaseModelV2[TestModel] `bson:",inline"`
Name string `json:"name" bson:"name"`
type TestModelV2 struct {
any `collection:"testmodels"`
BaseModelV2[TestModelV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
}

10
core/models/models/v2/token_v2.go
View File

@@ -1,10 +1,8 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type TokenV2 struct {
any `collection:"tokens"`
models.BaseModelV2[TokenV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Token string `json:"token" bson:"token"`
any `collection:"tokens"`
BaseModelV2[TokenV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Token string `json:"token" bson:"token"`
}

9
core/models/models/v2/user_role_v2.go
View File

@@ -1,13 +1,12 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type UserRoleV2 struct {
any `collection:"user_roles"`
models.BaseModelV2[UserRoleV2] `bson:",inline"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
any `collection:"user_roles"`
BaseModelV2[UserRoleV2] `bson:",inline"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
}

14
core/models/models/v2/user_v2.go
View File

@@ -1,12 +1,10 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type UserV2 struct {
any `collection:"users"`
models.BaseModelV2[UserV2] `bson:",inline"`
Username string `json:"username" bson:"username"`
Password string `json:"-,omitempty" bson:"password"`
Role string `json:"role" bson:"role"`
Email string `json:"email" bson:"email"`
any `collection:"users"`
BaseModelV2[UserV2] `bson:",inline"`
Username string `json:"username" bson:"username"`
Password string `json:"-,omitempty" bson:"password"`
Role string `json:"role" bson:"role"`
Email string `json:"email" bson:"email"`
}

12
core/models/models/v2/variable_v2.go
View File

@@ -1,11 +1,9 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type VariableV2 struct {
any `collection:"variables"`
models.BaseModelV2[VariableV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Value string `json:"value" bson:"value"`
Remark string `json:"remark" bson:"remark"`
any `collection:"variables"`
BaseModelV2[VariableV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Value string `json:"value" bson:"value"`
Remark string `json:"remark" bson:"remark"`
}

2
core/models/service/base_service_v2_test.go
View File

@@ -2,7 +2,7 @@ package service_test
import (
"context"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/spf13/viper"

34
core/node/service/master_service_v2.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/crawlab-team/crawlab/core/grpc/server"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/common"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/node/config"
"github.com/crawlab-team/crawlab/core/notification"
@@ -142,11 +142,11 @@ func (svc *MasterServiceV2) SetMonitorInterval(duration time.Duration) {
func (svc *MasterServiceV2) Register() (err error) {
nodeKey := svc.GetConfigService().GetNodeKey()
nodeName := svc.GetConfigService().GetNodeName()
node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
node, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
if err != nil && err.Error() == mongo2.ErrNoDocuments.Error() {
// not exists
log.Infof("master[%s] does not exist in db", nodeKey)
node := models.NodeV2{
node := models2.NodeV2{
Key: nodeKey,
Name: nodeName,
MaxRunners: config.DefaultConfigOptions.MaxRunners,
@@ -158,7 +158,7 @@ func (svc *MasterServiceV2) Register() (err error) {
}
node.SetCreated(primitive.NilObjectID)
node.SetUpdated(primitive.NilObjectID)
id, err := service.NewModelServiceV2[models.NodeV2]().InsertOne(node)
id, err := service.NewModelServiceV2[models2.NodeV2]().InsertOne(node)
if err != nil {
return err
}
@@ -170,7 +170,7 @@ func (svc *MasterServiceV2) Register() (err error) {
node.Status = constants.NodeStatusOnline
node.Active = true
node.ActiveAt = time.Now()
err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node)
err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node)
if err != nil {
return err
}
@@ -209,7 +209,7 @@ func (svc *MasterServiceV2) monitor() (err error) {
wg := sync.WaitGroup{}
wg.Add(len(workerNodes))
for _, n := range workerNodes {
go func(n *models.NodeV2) {
go func(n *models2.NodeV2) {
// subscribe
ok := svc.subscribeNode(n)
if !ok {
@@ -243,12 +243,12 @@ func (svc *MasterServiceV2) monitor() (err error) {
return nil
}
func (svc *MasterServiceV2) getAllWorkerNodes() (nodes []models.NodeV2, err error) {
func (svc *MasterServiceV2) getAllWorkerNodes() (nodes []models2.NodeV2, err error) {
query := bson.M{
"key": bson.M{"$ne": svc.cfgSvc.GetNodeKey()}, // not self
"active": true, // active
}
nodes, err = service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil)
nodes, err = service.NewModelServiceV2[models2.NodeV2]().GetMany(query, nil)
if err != nil {
if errors.Is(err, mongo2.ErrNoDocuments) {
return nil, nil
@@ -260,32 +260,32 @@ func (svc *MasterServiceV2) getAllWorkerNodes() (nodes []models.NodeV2, err erro
func (svc *MasterServiceV2) updateMasterNodeStatus() (err error) {
nodeKey := svc.GetConfigService().GetNodeKey()
node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
node, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
if err != nil {
return err
}
node.Status = constants.NodeStatusOnline
node.Active = true
node.ActiveAt = time.Now()
err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node)
err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node)
if err != nil {
return err
}
return nil
}
func (svc *MasterServiceV2) setWorkerNodeOffline(node *models.NodeV2) {
func (svc *MasterServiceV2) setWorkerNodeOffline(node *models2.NodeV2) {
node.Status = constants.NodeStatusOffline
node.Active = false
err := backoff.Retry(func() error {
return service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node)
return service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node)
}, backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 3))
if err != nil {
trace.PrintError(err)
}
}
func (svc *MasterServiceV2) subscribeNode(n *models.NodeV2) (ok bool) {
func (svc *MasterServiceV2) subscribeNode(n *models2.NodeV2) (ok bool) {
_, err := svc.server.GetSubscribe("node:" + n.Key)
if err != nil {
log.Errorf("cannot subscribe worker node[%s]: %v", n.Key, err)
@@ -294,7 +294,7 @@ func (svc *MasterServiceV2) subscribeNode(n *models.NodeV2) (ok bool) {
return true
}
func (svc *MasterServiceV2) pingNodeClient(n *models.NodeV2) (ok bool) {
func (svc *MasterServiceV2) pingNodeClient(n *models2.NodeV2) (ok bool) {
if err := svc.server.SendStreamMessage("node:"+n.Key, grpc.StreamMessageCode_PING); err != nil {
log.Errorf("cannot ping worker node client[%s]: %v", n.Key, err)
return false
@@ -302,17 +302,17 @@ func (svc *MasterServiceV2) pingNodeClient(n *models.NodeV2) (ok bool) {
return true
}
func (svc *MasterServiceV2) updateNodeAvailableRunners(node *models.NodeV2) (err error) {
func (svc *MasterServiceV2) updateNodeAvailableRunners(node *models2.NodeV2) (err error) {
query := bson.M{
"node_id": node.Id,
"status": constants.TaskStatusRunning,
}
runningTasksCount, err := service.NewModelServiceV2[models.TaskV2]().Count(query)
runningTasksCount, err := service.NewModelServiceV2[models2.TaskV2]().Count(query)
if err != nil {
return trace.TraceError(err)
}
node.AvailableRunners = node.MaxRunners - runningTasksCount
err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node)
err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node)
if err != nil {
return err
}

5
core/node/service/worker_service_v2.go
View File

@@ -9,6 +9,7 @@ import (
"github.com/crawlab-team/crawlab/core/interfaces"
client2 "github.com/crawlab-team/crawlab/core/models/client"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
"github.com/crawlab-team/crawlab/core/task/handler"
"github.com/crawlab-team/crawlab/core/utils"
@@ -31,7 +32,7 @@ type WorkerServiceV2 struct {
heartbeatInterval time.Duration
// internals
n *models.NodeV2
n *models2.NodeV2
s grpc.NodeService_SubscribeClient
}
@@ -87,7 +88,7 @@ func (svc *WorkerServiceV2) Register() {
if err != nil {
panic(err)
}
svc.n, err = client2.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": svc.GetConfigService().GetNodeKey()}, nil)
svc.n, err = client2.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": svc.GetConfigService().GetNodeKey()}, nil)
if err != nil {
panic(err)
}

2
core/notification/mail.go
View File

@@ -3,7 +3,7 @@ package notification
import (
"errors"
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/matcornic/hermes/v2"
"gopkg.in/gomail.v2"
"net/mail"

2
core/notification/service.go
View File

@@ -4,7 +4,7 @@ import (
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
mongo2 "github.com/crawlab-team/crawlab/db/mongo"

2
core/notification/service_v2.go
View File

@@ -4,7 +4,7 @@ import (
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
mongo2 "github.com/crawlab-team/crawlab/db/mongo"
parser "github.com/crawlab-team/crawlab/template-parser"

14
core/schedule/service_v2.go
View File

@@ -4,7 +4,7 @@ import (
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/config"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/spider/admin"
"github.com/crawlab-team/crawlab/core/utils"
@@ -19,7 +19,7 @@ import (
type ServiceV2 struct {
// dependencies
interfaces.WithConfigPath
modelSvc *service.ModelServiceV2[models.ScheduleV2]
modelSvc *service.ModelServiceV2[models2.ScheduleV2]
adminSvc *admin.ServiceV2
// settings variables
@@ -31,7 +31,7 @@ type ServiceV2 struct {
// internals
cron *cron.Cron
logger cron.Logger
schedules []models.ScheduleV2
schedules []models2.ScheduleV2
stopped bool
mu sync.Mutex
}
@@ -87,7 +87,7 @@ func (svc *ServiceV2) Stop() {
svc.cron.Stop()
}
func (svc *ServiceV2) Enable(s models.ScheduleV2, by primitive.ObjectID) (err error) {
func (svc *ServiceV2) Enable(s models2.ScheduleV2, by primitive.ObjectID) (err error) {
svc.mu.Lock()
defer svc.mu.Unlock()
@@ -101,7 +101,7 @@ func (svc *ServiceV2) Enable(s models.ScheduleV2, by primitive.ObjectID) (err er
return svc.modelSvc.ReplaceById(s.Id, s)
}
func (svc *ServiceV2) Disable(s models.ScheduleV2, by primitive.ObjectID) (err error) {
func (svc *ServiceV2) Disable(s models2.ScheduleV2, by primitive.ObjectID) (err error) {
svc.mu.Lock()
defer svc.mu.Unlock()
@@ -191,7 +191,7 @@ func (svc *ServiceV2) schedule(id primitive.ObjectID) (fn func()) {
}
// spider
spider, err := service.NewModelServiceV2[models.SpiderV2]().GetById(s.SpiderId)
spider, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(s.SpiderId)
if err != nil {
trace.PrintError(err)
return
@@ -250,7 +250,7 @@ func NewScheduleServiceV2() (svc2 *ServiceV2, err error) {
if err != nil {
return nil, err
}
svc.modelSvc = service.NewModelServiceV2[models.ScheduleV2]()
svc.modelSvc = service.NewModelServiceV2[models2.ScheduleV2]()
// logger
svc.logger = NewLogger()

14
core/spider/admin/service_v2.go
View File

@@ -6,7 +6,7 @@ import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/node/config"
"github.com/crawlab-team/crawlab/core/task/scheduler"
@@ -35,7 +35,7 @@ func (svc *ServiceV2) Start() (err error) {
func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) {
// spider
s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id)
s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id)
if err != nil {
return nil, err
}
@@ -44,9 +44,9 @@ func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRun
return svc.scheduleTasks(s, opts)
}
func (svc *ServiceV2) scheduleTasks(s *models.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) {
func (svc *ServiceV2) scheduleTasks(s *models2.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) {
// main task
mainTask := &models.TaskV2{
mainTask := &models2.TaskV2{
SpiderId: s.Id,
Mode: opts.Mode,
NodeIds: opts.NodeIds,
@@ -83,7 +83,7 @@ func (svc *ServiceV2) scheduleTasks(s *models.SpiderV2, opts *interfaces.SpiderR
return nil, err
}
for _, nodeId := range nodeIds {
t := &models.TaskV2{
t := &models2.TaskV2{
SpiderId: s.Id,
Mode: opts.Mode,
Cmd: opts.Cmd,
@@ -127,7 +127,7 @@ func (svc *ServiceV2) getNodeIds(opts *interfaces.SpiderRunOptions) (nodeIds []p
"enabled": true,
"status": constants.NodeStatusOnline,
}
nodes, err := service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil)
nodes, err := service.NewModelServiceV2[models2.NodeV2]().GetMany(query, nil)
if err != nil {
return nil, err
}
@@ -147,7 +147,7 @@ func (svc *ServiceV2) isMultiTask(opts *interfaces.SpiderRunOptions) (res bool)
"enabled": true,
"status": constants.NodeStatusOnline,
}
nodes, err := service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil)
nodes, err := service.NewModelServiceV2[models2.NodeV2]().GetMany(query, nil)
if err != nil {
trace.PrintError(err)
return false

2
core/system/service_v2.go
View File

@@ -1,7 +1,7 @@
package system
import (
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"go.mongodb.org/mongo-driver/bson"
"sync"

23
core/task/handler/runner_v2.go
View File

@@ -15,6 +15,7 @@ import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/client"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
service2 "github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/sys_exec"
"github.com/crawlab-team/crawlab/core/utils"
@@ -47,8 +48,8 @@ type RunnerV2 struct {
cmd *exec.Cmd // process command instance
pid int // process id
tid primitive.ObjectID // task id
t *models.TaskV2 // task model.Task
s *models.SpiderV2 // spider model.Spider
t *models2.TaskV2 // task model.Task
s *models2.SpiderV2 // spider model.Spider
ch chan constants.TaskSignal // channel to communicate between Service and RunnerV2
err error // standard process error
envs []models.Env // environment variables
@@ -315,7 +316,7 @@ func (r *RunnerV2) configureEnv() {
}
// global environment variables
envs, err := client.NewModelServiceV2[models.EnvironmentV2]().GetMany(nil, nil)
envs, err := client.NewModelServiceV2[models2.EnvironmentV2]().GetMany(nil, nil)
if err != nil {
trace.PrintError(err)
return
@@ -510,12 +511,12 @@ func (r *RunnerV2) updateTask(status string, e error) (err error) {
r.t.Error = e.Error()
}
if r.svc.GetNodeConfigService().IsMaster() {
err = service2.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t)
err = service2.NewModelServiceV2[models2.TaskV2]().ReplaceById(r.t.Id, *r.t)
if err != nil {
return err
}
} else {
err = client.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t)
err = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(r.t.Id, *r.t)
if err != nil {
return err
}
@@ -568,7 +569,7 @@ func (r *RunnerV2) writeLogLines(lines []string) {
}
func (r *RunnerV2) _updateTaskStat(status string) {
ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid)
ts, err := client.NewModelServiceV2[models2.TaskStatV2]().GetById(r.tid)
if err != nil {
trace.PrintError(err)
return
@@ -589,13 +590,13 @@ func (r *RunnerV2) _updateTaskStat(status string) {
ts.TotalDuration = ts.EndTs.Sub(ts.CreateTs).Milliseconds()
}
if r.svc.GetNodeConfigService().IsMaster() {
err = service2.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts)
err = service2.NewModelServiceV2[models2.TaskStatV2]().ReplaceById(ts.Id, *ts)
if err != nil {
trace.PrintError(err)
return
}
} else {
err = client.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts)
err = client.NewModelServiceV2[models2.TaskStatV2]().ReplaceById(ts.Id, *ts)
if err != nil {
trace.PrintError(err)
return
@@ -622,7 +623,7 @@ func (r *RunnerV2) sendNotification() {
func (r *RunnerV2) _updateSpiderStat(status string) {
// task stat
ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid)
ts, err := client.NewModelServiceV2[models2.TaskStatV2]().GetById(r.tid)
if err != nil {
trace.PrintError(err)
return
@@ -660,13 +661,13 @@ func (r *RunnerV2) _updateSpiderStat(status string) {
// perform update
if r.svc.GetNodeConfigService().IsMaster() {
err = service2.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update)
err = service2.NewModelServiceV2[models2.SpiderStatV2]().UpdateById(r.s.Id, update)
if err != nil {
trace.PrintError(err)
return
}
} else {
err = client.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update)
err = client.NewModelServiceV2[models2.SpiderStatV2]().UpdateById(r.s.Id, update)
if err != nil {
trace.PrintError(err)
return

30
core/task/handler/service_v2.go
View File

@@ -10,7 +10,7 @@ import (
grpcclient "github.com/crawlab-team/crawlab/core/grpc/client"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/client"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
"github.com/crawlab-team/crawlab/trace"
@@ -121,7 +121,7 @@ func (svc *ServiceV2) Fetch() {
t.Error = err.Error()
t.Status = constants.TaskStatusError
t.SetUpdated(t.CreatedBy)
_ = client.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t)
_ = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t)
continue
}
continue
@@ -202,15 +202,15 @@ func (svc *ServiceV2) GetNodeConfigService() (cfgSvc interfaces.NodeConfigServic
return svc.cfgSvc
}
func (svc *ServiceV2) GetCurrentNode() (n *models.NodeV2, err error) {
func (svc *ServiceV2) GetCurrentNode() (n *models2.NodeV2, err error) {
// node key
nodeKey := svc.cfgSvc.GetNodeKey()
// current node
if svc.cfgSvc.IsMaster() {
n, err = service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
n, err = service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
} else {
n, err = client.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
n, err = client.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil)
}
if err != nil {
return nil, err
@@ -219,11 +219,11 @@ func (svc *ServiceV2) GetCurrentNode() (n *models.NodeV2, err error) {
return n, nil
}
func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models.TaskV2, err error) {
func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models2.TaskV2, err error) {
if svc.cfgSvc.IsMaster() {
t, err = service.NewModelServiceV2[models.TaskV2]().GetById(id)
t, err = service.NewModelServiceV2[models2.TaskV2]().GetById(id)
} else {
t, err = client.NewModelServiceV2[models.TaskV2]().GetById(id)
t, err = client.NewModelServiceV2[models2.TaskV2]().GetById(id)
}
if err != nil {
return nil, err
@@ -232,11 +232,11 @@ func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models.TaskV2, err
return t, nil
}
func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models.SpiderV2, err error) {
func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models2.SpiderV2, err error) {
if svc.cfgSvc.IsMaster() {
s, err = service.NewModelServiceV2[models.SpiderV2]().GetById(id)
s, err = service.NewModelServiceV2[models2.SpiderV2]().GetById(id)
} else {
s, err = client.NewModelServiceV2[models.SpiderV2]().GetById(id)
s, err = client.NewModelServiceV2[models2.SpiderV2]().GetById(id)
}
if err != nil {
return nil, err
@@ -267,13 +267,13 @@ func (svc *ServiceV2) getRunnerCount() (count int) {
"status": constants.TaskStatusRunning,
}
if svc.cfgSvc.IsMaster() {
count, err = service.NewModelServiceV2[models.TaskV2]().Count(query)
count, err = service.NewModelServiceV2[models2.TaskV2]().Count(query)
if err != nil {
trace.PrintError(err)
return
}
} else {
count, err = client.NewModelServiceV2[models.TaskV2]().Count(query)
count, err = client.NewModelServiceV2[models2.TaskV2]().Count(query)
if err != nil {
trace.PrintError(err)
return
@@ -323,9 +323,9 @@ func (svc *ServiceV2) reportStatus() (err error) {
// save node
n.SetUpdated(n.CreatedBy)
if svc.cfgSvc.IsMaster() {
err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(n.Id, *n)
err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(n.Id, *n)
} else {
err = client.NewModelServiceV2[models.NodeV2]().ReplaceById(n.Id, *n)
err = client.NewModelServiceV2[models2.NodeV2]().ReplaceById(n.Id, *n)
}
if err != nil {
return err

42
core/task/scheduler/service_v2.go
View File

@@ -7,7 +7,7 @@ import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/grpc/server"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
"github.com/crawlab-team/crawlab/core/task/handler"
@@ -36,40 +36,40 @@ func (svc *ServiceV2) Start() {
utils.DefaultWait()
}
func (svc *ServiceV2) Enqueue(t *models.TaskV2, by primitive.ObjectID) (t2 *models.TaskV2, err error) {
func (svc *ServiceV2) Enqueue(t *models2.TaskV2, by primitive.ObjectID) (t2 *models2.TaskV2, err error) {
// set task status
t.Status = constants.TaskStatusPending
t.SetCreatedBy(by)
t.SetUpdated(by)
// add task
taskModelSvc := service.NewModelServiceV2[models.TaskV2]()
taskModelSvc := service.NewModelServiceV2[models2.TaskV2]()
_, err = taskModelSvc.InsertOne(*t)
if err != nil {
return nil, err
}
// task queue item
tq := models.TaskQueueItemV2{
tq := models2.TaskQueueItemV2{
Priority: t.Priority,
NodeId: t.NodeId,
}
tq.SetId(t.Id)
// task stat
ts := models.TaskStatV2{
ts := models2.TaskStatV2{
CreateTs: time.Now(),
}
ts.SetId(t.Id)
// enqueue task
_, err = service.NewModelServiceV2[models.TaskQueueItemV2]().InsertOne(tq)
_, err = service.NewModelServiceV2[models2.TaskQueueItemV2]().InsertOne(tq)
if err != nil {
return nil, trace.TraceError(err)
}
// add task stat
_, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(ts)
_, err = service.NewModelServiceV2[models2.TaskStatV2]().InsertOne(ts)
if err != nil {
return nil, trace.TraceError(err)
}
@@ -80,7 +80,7 @@ func (svc *ServiceV2) Enqueue(t *models.TaskV2, by primitive.ObjectID) (t2 *mode
func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err error) {
// task
t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id)
t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id)
if err != nil {
return trace.TraceError(err)
}
@@ -95,7 +95,7 @@ func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err
// set status of pending tasks as "cancelled" and remove from task item queue
if initialStatus == constants.TaskStatusPending {
// remove from task item queue
if err := service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteById(t.Id); err != nil {
if err := service.NewModelServiceV2[models2.TaskQueueItemV2]().DeleteById(t.Id); err != nil {
return trace.TraceError(err)
}
return nil
@@ -110,7 +110,7 @@ func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err
}
// node
n, err := service.NewModelServiceV2[models.NodeV2]().GetById(t.NodeId)
n, err := service.NewModelServiceV2[models2.NodeV2]().GetById(t.NodeId)
if err != nil {
return trace.TraceError(err)
}
@@ -136,22 +136,22 @@ func (svc *ServiceV2) SetInterval(interval time.Duration) {
svc.interval = interval
}
func (svc *ServiceV2) SaveTask(t *models.TaskV2, by primitive.ObjectID) (err error) {
func (svc *ServiceV2) SaveTask(t *models2.TaskV2, by primitive.ObjectID) (err error) {
if t.Id.IsZero() {
t.SetCreated(by)
t.SetUpdated(by)
_, err = service.NewModelServiceV2[models.TaskV2]().InsertOne(*t)
_, err = service.NewModelServiceV2[models2.TaskV2]().InsertOne(*t)
return err
} else {
t.SetUpdated(by)
return service.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t)
return service.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t)
}
}
// initTaskStatus initialize task status of existing tasks
func (svc *ServiceV2) initTaskStatus() {
// set status of running tasks as TaskStatusAbnormal
runningTasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{
runningTasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(bson.M{
"status": bson.M{
"$in": []string{
constants.TaskStatusPending,
@@ -166,23 +166,23 @@ func (svc *ServiceV2) initTaskStatus() {
trace.PrintError(err)
}
for _, t := range runningTasks {
go func(t *models.TaskV2) {
go func(t *models2.TaskV2) {
t.Status = constants.TaskStatusAbnormal
if err := svc.SaveTask(t, primitive.NilObjectID); err != nil {
trace.PrintError(err)
}
}(&t)
}
if err := service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteMany(nil); err != nil {
if err := service.NewModelServiceV2[models2.TaskQueueItemV2]().DeleteMany(nil); err != nil {
return
}
}
func (svc *ServiceV2) isMasterNode(t *models.TaskV2) (ok bool, err error) {
func (svc *ServiceV2) isMasterNode(t *models2.TaskV2) (ok bool, err error) {
if t.NodeId.IsZero() {
return false, trace.TraceError(errors.ErrorTaskNoNodeId)
}
n, err := service.NewModelServiceV2[models.NodeV2]().GetById(t.NodeId)
n, err := service.NewModelServiceV2[models2.NodeV2]().GetById(t.NodeId)
if err != nil {
if errors2.Is(err, mongo2.ErrNoDocuments) {
return false, trace.TraceError(errors.ErrorTaskNodeNotFound)
@@ -195,7 +195,7 @@ func (svc *ServiceV2) isMasterNode(t *models.TaskV2) (ok bool, err error) {
func (svc *ServiceV2) cleanupTasks() {
for {
// task stats over 30 days ago
taskStats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(bson.M{
taskStats, err := service.NewModelServiceV2[models2.TaskStatV2]().GetMany(bson.M{
"create_ts": bson.M{
"$lt": time.Now().Add(-30 * 24 * time.Hour),
},
@@ -213,14 +213,14 @@ func (svc *ServiceV2) cleanupTasks() {
if len(ids) > 0 {
// remove tasks
if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{
if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{
"_id": bson.M{"$in": ids},
}); err != nil {
trace.PrintError(err)
}
// remove task stats
if err := service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{
if err := service.NewModelServiceV2[models2.TaskStatV2]().DeleteMany(bson.M{
"_id": bson.M{"$in": ids},
}); err != nil {
trace.PrintError(err)

6
core/task/stats/service_v2.go
View File

@@ -2,7 +2,7 @@ package stats
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
"github.com/crawlab-team/crawlab/core/result"
@@ -63,7 +63,7 @@ func (svc *ServiceV2) getResultService(id primitive.ObjectID) (resultSvc interfa
}
// task
t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id)
t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id)
if err != nil {
return nil, err
}
@@ -81,7 +81,7 @@ func (svc *ServiceV2) getResultService(id primitive.ObjectID) (resultSvc interfa
}
func (svc *ServiceV2) updateTaskStats(id primitive.ObjectID, resultCount int) {
err := service.NewModelServiceV2[models.TaskStatV2]().UpdateById(id, bson.M{
err := service.NewModelServiceV2[models2.TaskStatV2]().UpdateById(id, bson.M{
"$inc": bson.M{
"result_count": resultCount,
},

2
core/user/service_v2.go
View File

@@ -4,7 +4,7 @@ import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
mongo2 "github.com/crawlab-team/crawlab/db/mongo"

5
core/utils/cockroachdb.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/upper/db/v4"
"github.com/upper/db/v4/adapter/mssql"
"time"
@@ -59,13 +60,13 @@ func getCockroachdbSession(ctx context.Context, ds *models.DataSource) (s db.Ses
return s, err
}
func GetCockroachdbSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
func GetCockroachdbSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getCockroachdbSessionV2(ctx, ds)
}
func getCockroachdbSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) {
func getCockroachdbSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) {
// normalize settings
host := ds.Host
port := ds.Port

5
core/utils/es.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/cenkalti/backoff/v4"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/trace"
"github.com/elastic/go-elasticsearch/v8"
@@ -83,13 +84,13 @@ func getElasticsearchClient(ctx context.Context, ds *models.DataSource) (c *elas
return c, err
}
func GetElasticsearchClientWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *elasticsearch.Client, err error) {
func GetElasticsearchClientWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *elasticsearch.Client, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getElasticsearchClientV2(ctx, ds)
}
func getElasticsearchClientV2(ctx context.Context, ds *models.DataSourceV2) (c *elasticsearch.Client, err error) {
func getElasticsearchClientV2(ctx context.Context, ds *models2.DataSourceV2) (c *elasticsearch.Client, err error) {
// normalize settings
host := ds.Host
port := ds.Port

5
core/utils/kafka.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/segmentio/kafka-go"
"time"
)
@@ -40,13 +41,13 @@ func getKafkaConnection(ctx context.Context, ds *models.DataSource) (c *kafka.Co
return kafka.DialLeader(ctx, network, address, topic, partition)
}
func GetKafkaConnectionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *kafka.Conn, err error) {
func GetKafkaConnectionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *kafka.Conn, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getKafkaConnectionV2(ctx, ds)
}
func getKafkaConnectionV2(ctx context.Context, ds *models.DataSourceV2) (c *kafka.Conn, err error) {
func getKafkaConnectionV2(ctx context.Context, ds *models2.DataSourceV2) (c *kafka.Conn, err error) {
// normalize settings
host := ds.Host
port := ds.Port

5
core/utils/mongo.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
@@ -54,7 +55,7 @@ func GetMongoClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c
return getMongoClient(ctx, ds)
}
func GetMongoClientWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *mongo2.Client, err error) {
func GetMongoClientWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *mongo2.Client, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getMongoClientV2(ctx, ds)
@@ -99,7 +100,7 @@ func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Clien
return mongo.GetMongoClient(opts...)
}
func getMongoClientV2(ctx context.Context, ds *models.DataSourceV2) (c *mongo2.Client, err error) {
func getMongoClientV2(ctx context.Context, ds *models2.DataSourceV2) (c *mongo2.Client, err error) {
// normalize settings
if ds.Host == "" {
ds.Host = constants.DefaultHost

5
core/utils/mssql.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/upper/db/v4"
"github.com/upper/db/v4/adapter/mssql"
"time"
@@ -59,13 +60,13 @@ func getMssqlSession(ctx context.Context, ds *models.DataSource) (s db.Session,
return s, err
}
func GetMssqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
func GetMssqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getMssqlSessionV2(ctx, ds)
}
func getMssqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) {
func getMssqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) {
// normalize settings
host := ds.Host
port := ds.Port

5
core/utils/mysql.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/upper/db/v4"
"github.com/upper/db/v4/adapter/mysql"
"time"
@@ -59,13 +60,13 @@ func getMysqlSession(ctx context.Context, ds *models.DataSource) (s db.Session,
return s, err
}
func GetMysqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
func GetMysqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getMysqlSessionV2(ctx, ds)
}
func getMysqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) {
func getMysqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) {
// normalize settings
host := ds.Host
port := ds.Port

5
core/utils/postgresql.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/upper/db/v4"
"github.com/upper/db/v4/adapter/postgresql"
"time"
@@ -59,13 +60,13 @@ func getPostgresqlSession(ctx context.Context, ds *models.DataSource) (s db.Sess
return s, err
}
func GetPostgresqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
func GetPostgresqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getPostgresqlSessionV2(ctx, ds)
}
func getPostgresqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) {
func getPostgresqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) {
// normalize settings
host := ds.Host
port := ds.Port

5
core/utils/sqlite.go
View File

@@ -3,6 +3,7 @@ package utils
import (
"context"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/upper/db/v4"
"github.com/upper/db/v4/adapter/sqlite"
"time"
@@ -44,13 +45,13 @@ func getSqliteSession(ctx context.Context, ds *models.DataSource) (s db.Session,
return s, err
}
func GetSqliteSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
func GetSqliteSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getSqliteSessionV2(ctx, ds)
}
func getSqliteSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) {
func getSqliteSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) {
// connect settings
settings := sqlite.ConnectionURL{
Database: ds.Database,