git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-29 18:00:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: added modules

Browse Source
This commit is contained in:
Marvin Zhang
2024-06-14 15:42:50 +08:00
parent 4d0adcb6f0
commit c4d795f47f
626 changed files with 60104 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
core/models/client/binder_basic.go Normal file
View File

@@ -0,0 +1,92 @@
package client
import (
"encoding/json"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/go-trace"
)
func NewBasicBinder(id interfaces.ModelId, res *grpc.Response) (b interfaces.GrpcModelBinder) {
return &BasicBinder{
id: id,
res: res,
}
}
type BasicBinder struct {
id interfaces.ModelId
res *grpc.Response
}
func (b *BasicBinder) Bind() (res interfaces.Model, err error) {
m := models.NewModelMap()
switch b.id {
case interfaces.ModelIdArtifact:
return b.Process(&m.Artifact)
case interfaces.ModelIdTag:
return b.Process(&m.Tag)
case interfaces.ModelIdNode:
return b.Process(&m.Node)
case interfaces.ModelIdProject:
return b.Process(&m.Project)
case interfaces.ModelIdSpider:
return b.Process(&m.Spider)
case interfaces.ModelIdTask:
return b.Process(&m.Task)
case interfaces.ModelIdJob:
return b.Process(&m.Job)
case interfaces.ModelIdSchedule:
return b.Process(&m.Schedule)
case interfaces.ModelIdUser:
return b.Process(&m.User)
case interfaces.ModelIdSetting:
return b.Process(&m.Setting)
case interfaces.ModelIdToken:
return b.Process(&m.Token)
case interfaces.ModelIdVariable:
return b.Process(&m.Variable)
case interfaces.ModelIdTaskQueue:
return b.Process(&m.TaskQueueItem)
case interfaces.ModelIdTaskStat:
return b.Process(&m.TaskStat)
case interfaces.ModelIdSpiderStat:
return b.Process(&m.SpiderStat)
case interfaces.ModelIdDataSource:
return b.Process(&m.DataSource)
case interfaces.ModelIdDataCollection:
return b.Process(&m.DataCollection)
case interfaces.ModelIdResult:
return b.Process(&m.Result)
case interfaces.ModelIdPassword:
return b.Process(&m.Password)
case interfaces.ModelIdExtraValue:
return b.Process(&m.ExtraValue)
case interfaces.ModelIdGit:
return b.Process(&m.Git)
case interfaces.ModelIdRole:
return b.Process(&m.Role)
case interfaces.ModelIdUserRole:
return b.Process(&m.UserRole)
case interfaces.ModelIdPermission:
return b.Process(&m.Permission)
case interfaces.ModelIdRolePermission:
return b.Process(&m.RolePermission)
case interfaces.ModelIdEnvironment:
return b.Process(&m.Environment)
case interfaces.ModelIdDependencySetting:
return b.Process(&m.DependencySetting)
default:
return nil, errors.ErrorModelInvalidModelId
}
}
func (b *BasicBinder) Process(d interfaces.Model) (res interfaces.Model, err error) {
if err := json.Unmarshal(b.res.Data, d); err != nil {
return nil, trace.TraceError(err)
}
return d, nil
}

100
core/models/client/binder_list.go Normal file
View File

@@ -0,0 +1,100 @@
package client
import (
"encoding/json"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/go-trace"
)
func NewListBinder(id interfaces.ModelId, res *grpc.Response) (b interfaces.GrpcModelListBinder) {
return &ListBinder{
id: id,
res: res,
}
}
type ListBinder struct {
id interfaces.ModelId
res *grpc.Response
}
func (b *ListBinder) Bind() (l interfaces.List, err error) {
m := models.NewModelListMap()
switch b.id {
case interfaces.ModelIdArtifact:
return b.Process(&m.Artifacts)
case interfaces.ModelIdTag:
return b.Process(&m.Tags)
case interfaces.ModelIdNode:
return b.Process(&m.Nodes)
case interfaces.ModelIdProject:
return b.Process(&m.Projects)
case interfaces.ModelIdSpider:
return b.Process(&m.Spiders)
case interfaces.ModelIdTask:
return b.Process(&m.Tasks)
case interfaces.ModelIdJob:
return b.Process(&m.Jobs)
case interfaces.ModelIdSchedule:
return b.Process(&m.Schedules)
case interfaces.ModelIdUser:
return b.Process(&m.Users)
case interfaces.ModelIdSetting:
return b.Process(&m.Settings)
case interfaces.ModelIdToken:
return b.Process(&m.Tokens)
case interfaces.ModelIdVariable:
return b.Process(&m.Variables)
case interfaces.ModelIdTaskQueue:
return b.Process(&m.TaskQueueItems)
case interfaces.ModelIdTaskStat:
return b.Process(&m.TaskStats)
case interfaces.ModelIdSpiderStat:
return b.Process(&m.SpiderStats)
case interfaces.ModelIdDataSource:
return b.Process(&m.DataSources)
case interfaces.ModelIdDataCollection:
return b.Process(&m.DataCollections)
case interfaces.ModelIdResult:
return b.Process(&m.Results)
case interfaces.ModelIdPassword:
return b.Process(&m.Passwords)
case interfaces.ModelIdExtraValue:
return b.Process(&m.ExtraValues)
case interfaces.ModelIdGit:
return b.Process(&m.Gits)
case interfaces.ModelIdRole:
return b.Process(&m.Roles)
case interfaces.ModelIdUserRole:
return b.Process(&m.UserRoles)
case interfaces.ModelIdPermission:
return b.Process(&m.PermissionList)
case interfaces.ModelIdRolePermission:
return b.Process(&m.RolePermissionList)
case interfaces.ModelIdEnvironment:
return b.Process(&m.Environments)
case interfaces.ModelIdDependencySetting:
return b.Process(&m.DependencySettings)
default:
return l, errors.ErrorModelInvalidModelId
}
}
func (b *ListBinder) MustBind() (res interface{}) {
res, err := b.Bind()
if err != nil {
panic(err)
}
return res
}
func (b *ListBinder) Process(d interface{}) (l interfaces.List, err error) {
if err := json.Unmarshal(b.res.Data, d); err != nil {
return l, trace.TraceError(err)
}
return d.(interfaces.List), nil
}

251
core/models/client/model_base_service.go Normal file
View File

@@ -0,0 +1,251 @@
package client
import (
"encoding/json"
"github.com/apex/log"
"github.com/cenkalti/backoff/v4"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/utils"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/go-trace"
errors2 "github.com/pkg/errors"
"github.com/spf13/viper"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"time"
)
type BaseServiceDelegate struct {
// settings
cfgPath string
// internals
id interfaces.ModelId
c interfaces.GrpcClient
}
func (d *BaseServiceDelegate) GetModelId() (id interfaces.ModelId) {
return d.id
}
func (d *BaseServiceDelegate) SetModelId(id interfaces.ModelId) {
d.id = id
}
func (d *BaseServiceDelegate) GetConfigPath() (path string) {
return d.cfgPath
}
func (d *BaseServiceDelegate) SetConfigPath(path string) {
d.cfgPath = path
}
func (d *BaseServiceDelegate) GetById(id primitive.ObjectID) (doc interfaces.Model, err error) {
log.Debugf("[BaseServiceDelegate] get by id[%s]", id.Hex())
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id})
c := d.getModelBaseServiceClient()
if c == nil {
return nil, trace.TraceError(errors.ErrorModelNilPointer)
}
log.Debugf("[BaseServiceDelegate] get by id[%s] req: %v", id.Hex(), req)
res, err := c.GetById(ctx, req)
if err != nil {
return nil, trace.TraceError(err)
}
log.Debugf("[BaseServiceDelegate] get by id[%s] res: %v", id.Hex(), res)
return NewBasicBinder(d.id, res).Bind()
}
func (d *BaseServiceDelegate) Get(query bson.M, opts *mongo.FindOptions) (doc interfaces.Model, err error) {
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, FindOptions: opts})
res, err := d.getModelBaseServiceClient().Get(ctx, req)
if err != nil {
return nil, err
}
return NewBasicBinder(d.id, res).Bind()
}
func (d *BaseServiceDelegate) GetList(query bson.M, opts *mongo.FindOptions) (l interfaces.List, err error) {
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, FindOptions: opts})
res, err := d.getModelBaseServiceClient().GetList(ctx, req)
if err != nil {
return l, err
}
return NewListBinder(d.id, res).Bind()
}
func (d *BaseServiceDelegate) DeleteById(id primitive.ObjectID, args ...interface{}) (err error) {
u := utils.GetUserFromArgs(args...)
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id, User: u})
_, err = d.getModelBaseServiceClient().DeleteById(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) Delete(query bson.M, args ...interface{}) (err error) {
u := utils.GetUserFromArgs(args...)
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u})
_, err = d.getModelBaseServiceClient().Delete(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) DeleteList(query bson.M, args ...interface{}) (err error) {
u := utils.GetUserFromArgs(args...)
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u})
_, err = d.getModelBaseServiceClient().DeleteList(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) ForceDeleteList(query bson.M, args ...interface{}) (err error) {
u := utils.GetUserFromArgs(args...)
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u})
_, err = d.getModelBaseServiceClient().ForceDeleteList(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) UpdateById(id primitive.ObjectID, update bson.M, args ...interface{}) (err error) {
u := utils.GetUserFromArgs(args...)
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id, Update: update, User: u})
_, err = d.getModelBaseServiceClient().UpdateById(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) Update(query bson.M, update bson.M, fields []string, args ...interface{}) (err error) {
u := utils.GetUserFromArgs(args...)
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, Update: update, Fields: fields, User: u})
_, err = d.getModelBaseServiceClient().Update(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) UpdateDoc(query bson.M, doc interfaces.Model, fields []string, args ...interface{}) (err error) {
u := utils.GetUserFromArgs(args...)
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, Doc: doc, Fields: fields, User: u})
_, err = d.getModelBaseServiceClient().UpdateDoc(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) Insert(u interfaces.User, docs ...interface{}) (err error) {
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Docs: docs, User: u})
_, err = d.getModelBaseServiceClient().Insert(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) Count(query bson.M) (total int, err error) {
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query})
res, err := d.getModelBaseServiceClient().Count(ctx, req)
if err != nil {
return total, err
}
if err := json.Unmarshal(res.Data, &total); err != nil {
return total, err
}
return total, nil
}
func (d *BaseServiceDelegate) newRequest(params interfaces.GrpcBaseServiceParams) (req *grpc.Request, err error) {
return d.c.NewModelBaseServiceRequest(d.id, params)
}
func (d *BaseServiceDelegate) mustNewRequest(params *entity.GrpcBaseServiceParams) (req *grpc.Request) {
req, err := d.newRequest(params)
if err != nil {
panic(err)
}
return req
}
func (d *BaseServiceDelegate) getModelBaseServiceClient() (c grpc.ModelBaseServiceClient) {
if err := backoff.Retry(func() (err error) {
c = d.c.GetModelBaseServiceClient()
if c == nil {
err = errors2.New("unable to get model base service client")
log.Debugf("[BaseServiceDelegate] err: %v", err)
return err
}
return nil
}, backoff.NewConstantBackOff(1*time.Second)); err != nil {
trace.PrintError(err)
}
return c
}
func NewBaseServiceDelegate(opts ...ModelBaseServiceDelegateOption) (svc2 interfaces.GrpcClientModelBaseService, err error) {
// base service
svc := &BaseServiceDelegate{}
// apply options
for _, opt := range opts {
opt(svc)
}
// config path
if viper.GetString("config.path") != "" {
svc.cfgPath = viper.GetString("config.path")
}
// dependency injection
if err := container.GetContainer().Invoke(func(client interfaces.GrpcClient) {
svc.c = client
}); err != nil {
return nil, err
}
return svc, nil
}
func ProvideBaseServiceDelegate(id interfaces.ModelId) func() (svc interfaces.GrpcClientModelBaseService, err error) {
return func() (svc interfaces.GrpcClientModelBaseService, err error) {
return NewBaseServiceDelegate(WithBaseServiceModelId(id))
}
}

329
core/models/client/model_delegate.go Normal file
View File

@@ -0,0 +1,329 @@
package client
import (
"encoding/json"
config2 "github.com/crawlab-team/crawlab/core/config"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/grpc/client"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/utils"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/go-trace"
"github.com/spf13/viper"
)
func NewModelDelegate(doc interfaces.Model, opts ...ModelDelegateOption) interfaces.GrpcClientModelDelegate {
switch doc.(type) {
case *models.Artifact:
return newModelDelegate(interfaces.ModelIdArtifact, doc, opts...)
case *models.Tag:
return newModelDelegate(interfaces.ModelIdTag, doc, opts...)
case *models.Node:
return newModelDelegate(interfaces.ModelIdNode, doc, opts...)
case *models.Project:
return newModelDelegate(interfaces.ModelIdProject, doc, opts...)
case *models.Spider:
return newModelDelegate(interfaces.ModelIdSpider, doc, opts...)
case *models.Task:
return newModelDelegate(interfaces.ModelIdTask, doc, opts...)
case *models.Job:
return newModelDelegate(interfaces.ModelIdJob, doc, opts...)
case *models.Schedule:
return newModelDelegate(interfaces.ModelIdSchedule, doc, opts...)
case *models.User:
return newModelDelegate(interfaces.ModelIdUser, doc, opts...)
case *models.Setting:
return newModelDelegate(interfaces.ModelIdSetting, doc, opts...)
case *models.Token:
return newModelDelegate(interfaces.ModelIdToken, doc, opts...)
case *models.Variable:
return newModelDelegate(interfaces.ModelIdVariable, doc, opts...)
case *models.TaskQueueItem:
return newModelDelegate(interfaces.ModelIdTaskQueue, doc, opts...)
case *models.TaskStat:
return newModelDelegate(interfaces.ModelIdTaskStat, doc, opts...)
case *models.SpiderStat:
return newModelDelegate(interfaces.ModelIdSpiderStat, doc, opts...)
case *models.DataSource:
return newModelDelegate(interfaces.ModelIdDataSource, doc, opts...)
case *models.DataCollection:
return newModelDelegate(interfaces.ModelIdDataCollection, doc, opts...)
case *models.Result:
return newModelDelegate(interfaces.ModelIdResult, doc, opts...)
case *models.Password:
return newModelDelegate(interfaces.ModelIdPassword, doc, opts...)
case *models.ExtraValue:
return newModelDelegate(interfaces.ModelIdExtraValue, doc, opts...)
case *models.Git:
return newModelDelegate(interfaces.ModelIdGit, doc, opts...)
case *models.UserRole:
return newModelDelegate(interfaces.ModelIdUserRole, doc, opts...)
case *models.Permission:
return newModelDelegate(interfaces.ModelIdPermission, doc, opts...)
case *models.RolePermission:
return newModelDelegate(interfaces.ModelIdRolePermission, doc, opts...)
case *models.Environment:
return newModelDelegate(interfaces.ModelIdEnvironment, doc, opts...)
case *models.DependencySetting:
return newModelDelegate(interfaces.ModelIdDependencySetting, doc, opts...)
default:
_ = trace.TraceError(errors.ErrorModelInvalidType)
return nil
}
}
func newModelDelegate(id interfaces.ModelId, doc interfaces.Model, opts ...ModelDelegateOption) interfaces.GrpcClientModelDelegate {
var err error
// collection name
colName := models.GetModelColName(id)
// model delegate
d := &ModelDelegate{
id: id,
colName: colName,
doc: doc,
cfgPath: config2.GetConfigPath(),
a: &models.Artifact{
Col: colName,
},
}
// config path
if viper.GetString("config.path") != "" {
d.cfgPath = viper.GetString("config.path")
}
// apply options
for _, opt := range opts {
opt(d)
}
// grpc client
d.c, err = client.GetClient()
if err != nil {
trace.PrintError(errors.ErrorModelInvalidType)
return nil
}
if !d.c.IsStarted() {
if err := d.c.Start(); err != nil {
trace.PrintError(err)
return nil
}
} else if d.c.IsClosed() {
if err := d.c.Restart(); err != nil {
trace.PrintError(err)
return nil
}
}
return d
}
type ModelDelegate struct {
// settings
cfgPath string
// internals
id interfaces.ModelId
colName string
c interfaces.GrpcClient
doc interfaces.Model
a interfaces.ModelArtifact
}
func (d *ModelDelegate) Add() (err error) {
return d.do(interfaces.ModelDelegateMethodAdd)
}
func (d *ModelDelegate) Save() (err error) {
return d.do(interfaces.ModelDelegateMethodSave)
}
func (d *ModelDelegate) Delete() (err error) {
return d.do(interfaces.ModelDelegateMethodDelete)
}
func (d *ModelDelegate) GetArtifact() (res interfaces.ModelArtifact, err error) {
if err := d.do(interfaces.ModelDelegateMethodGetArtifact); err != nil {
return nil, err
}
return d.a, nil
}
func (d *ModelDelegate) GetModel() (res interfaces.Model) {
return d.doc
}
func (d *ModelDelegate) Refresh() (err error) {
return d.refresh()
}
func (d *ModelDelegate) GetConfigPath() (path string) {
return d.cfgPath
}
func (d *ModelDelegate) SetConfigPath(path string) {
d.cfgPath = path
}
func (d *ModelDelegate) Close() (err error) {
return d.c.Stop()
}
func (d *ModelDelegate) ToBytes(m interface{}) (bytes []byte, err error) {
if m != nil {
return utils.JsonToBytes(m)
}
return json.Marshal(d.doc)
}
func (d *ModelDelegate) do(method interfaces.ModelDelegateMethod) (err error) {
switch method {
case interfaces.ModelDelegateMethodAdd:
err = d.add()
case interfaces.ModelDelegateMethodSave:
err = d.save()
case interfaces.ModelDelegateMethodDelete:
err = d.delete()
case interfaces.ModelDelegateMethodGetArtifact, interfaces.ModelDelegateMethodRefresh:
return d.refresh()
default:
return trace.TraceError(errors.ErrorModelInvalidType)
}
if err != nil {
return err
}
return nil
}
func (d *ModelDelegate) add() (err error) {
ctx, cancel := d.c.Context()
defer cancel()
method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodAdd)
res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
ModelId: d.id,
Method: method,
Data: d.mustGetData(),
}))
if err != nil {
return trace.TraceError(err)
}
if err := d.deserialize(res, method); err != nil {
return err
}
return d.refreshArtifact()
}
func (d *ModelDelegate) save() (err error) {
ctx, cancel := d.c.Context()
defer cancel()
method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodSave)
res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
ModelId: d.id,
Method: method,
Data: d.mustGetData(),
}))
if err != nil {
return trace.TraceError(err)
}
if err := d.deserialize(res, method); err != nil {
return err
}
return d.refreshArtifact()
}
func (d *ModelDelegate) delete() (err error) {
ctx, cancel := d.c.Context()
defer cancel()
method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodDelete)
res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
ModelId: d.id,
Method: method,
Data: d.mustGetData(),
}))
if err != nil {
return trace.TraceError(err)
}
if err := d.deserialize(res, method); err != nil {
return err
}
return d.refreshArtifact()
}
func (d *ModelDelegate) refresh() (err error) {
ctx, cancel := d.c.Context()
defer cancel()
method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodRefresh)
res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
ModelId: d.id,
Method: method,
Data: d.mustGetData(),
}))
if err != nil {
return trace.TraceError(err)
}
if err := d.deserialize(res, method); err != nil {
return err
}
return nil
}
func (d *ModelDelegate) refreshArtifact() (err error) {
_, err = d.getArtifact()
return err
}
func (d *ModelDelegate) getArtifact() (res2 interfaces.ModelArtifact, err error) {
ctx, cancel := d.c.Context()
defer cancel()
method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodGetArtifact)
res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
ModelId: d.id,
Method: method,
Data: d.mustGetData(),
}))
if err != nil {
return nil, err
}
if err := d.deserialize(res, method); err != nil {
return nil, err
}
return d.a, nil
}
func (d *ModelDelegate) mustGetData() (data []byte) {
data, err := d.getData()
if err != nil {
panic(err)
}
return data
}
func (d *ModelDelegate) getData() (data []byte, err error) {
return json.Marshal(d.doc)
}
func (d *ModelDelegate) deserialize(res *grpc.Response, method interfaces.ModelDelegateMethod) (err error) {
if method == interfaces.ModelDelegateMethodGetArtifact {
res, err := NewBasicBinder(interfaces.ModelIdArtifact, res).Bind()
if err != nil {
return err
}
a, ok := res.(interfaces.ModelArtifact)
if !ok {
return trace.TraceError(errors.ErrorModelInvalidType)
}
d.a = a
} else {
d.doc, err = NewBasicBinder(d.id, res).Bind()
if err != nil {
return err
}
}
return nil
}

70
core/models/client/model_environment_service.go Normal file
View File

@@ -0,0 +1,70 @@
package client
import (
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type EnvironmentServiceDelegate struct {
interfaces.GrpcClientModelBaseService
}
func (svc *EnvironmentServiceDelegate) GetEnvironmentById(id primitive.ObjectID) (e interfaces.Environment, err error) {
res, err := svc.GetById(id)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Environment)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *EnvironmentServiceDelegate) GetEnvironment(query bson.M, opts *mongo.FindOptions) (e interfaces.Environment, err error) {
res, err := svc.Get(query, opts)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Environment)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *EnvironmentServiceDelegate) GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Environment, err error) {
list, err := svc.GetList(query, opts)
if err != nil {
return nil, err
}
for _, item := range list.GetModels() {
s, ok := item.(interfaces.Environment)
if !ok {
return nil, errors.ErrorModelInvalidType
}
res = append(res, s)
}
return res, nil
}
func NewEnvironmentServiceDelegate() (svc2 interfaces.GrpcClientModelEnvironmentService, err error) {
var opts []ModelBaseServiceDelegateOption
// apply options
opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdEnvironment))
// base service
baseSvc, err := NewBaseServiceDelegate(opts...)
if err != nil {
return nil, err
}
// service
svc := &EnvironmentServiceDelegate{baseSvc}
return svc, nil
}

37
core/models/client/model_node_delegate.go Normal file
View File

@@ -0,0 +1,37 @@
package client
import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/interfaces"
"time"
)
type ModelNodeDelegate struct {
n interfaces.Node
interfaces.GrpcClientModelDelegate
}
func (d *ModelNodeDelegate) UpdateStatus(active bool, activeTs *time.Time, status string) (err error) {
d.n.SetActive(active)
if activeTs != nil {
d.n.SetActiveTs(*activeTs)
}
d.n.SetStatus(status)
return d.Save()
}
func (d *ModelNodeDelegate) UpdateStatusOnline() (err error) {
now := time.Now()
return d.UpdateStatus(true, &now, constants.NodeStatusOnline)
}
func (d *ModelNodeDelegate) UpdateStatusOffline() (err error) {
return d.UpdateStatus(false, nil, constants.NodeStatusOffline)
}
func NewModelNodeDelegate(n interfaces.Node) interfaces.ModelNodeDelegate {
return &ModelNodeDelegate{
n: n,
GrpcClientModelDelegate: NewModelDelegate(n),
}
}

74
core/models/client/model_node_service.go Normal file
View File

@@ -0,0 +1,74 @@
package client
import (
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type NodeServiceDelegate struct {
interfaces.GrpcClientModelBaseService
}
func (svc *NodeServiceDelegate) GetNodeById(id primitive.ObjectID) (n interfaces.Node, err error) {
res, err := svc.GetById(id)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Node)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *NodeServiceDelegate) GetNode(query bson.M, opts *mongo.FindOptions) (n interfaces.Node, err error) {
res, err := svc.Get(query, opts)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Node)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *NodeServiceDelegate) GetNodeByKey(key string) (n interfaces.Node, err error) {
return svc.GetNode(bson.M{"key": key}, nil)
}
func (svc *NodeServiceDelegate) GetNodeList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Node, err error) {
list, err := svc.GetList(query, opts)
if err != nil {
return nil, err
}
for _, item := range list.GetModels() {
s, ok := item.(interfaces.Node)
if !ok {
return nil, errors.ErrorModelInvalidType
}
res = append(res, s)
}
return res, nil
}
func NewNodeServiceDelegate() (svc2 interfaces.GrpcClientModelNodeService, err error) {
var opts []ModelBaseServiceDelegateOption
// apply options
opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdNode))
// base service
baseSvc, err := NewBaseServiceDelegate(opts...)
if err != nil {
return nil, err
}
// service
svc := &NodeServiceDelegate{baseSvc}
return svc, nil
}

48
core/models/client/model_service.go Normal file
View File

@@ -0,0 +1,48 @@
package client
import (
config2 "github.com/crawlab-team/crawlab/core/config"
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/interfaces"
)
type ServiceDelegate struct {
// settings
cfgPath string
// internals
c interfaces.GrpcClient
}
func (d *ServiceDelegate) GetConfigPath() string {
return d.cfgPath
}
func (d *ServiceDelegate) SetConfigPath(path string) {
d.cfgPath = path
}
func (d *ServiceDelegate) NewBaseServiceDelegate(id interfaces.ModelId) (svc interfaces.GrpcClientModelBaseService, err error) {
var opts []ModelBaseServiceDelegateOption
opts = append(opts, WithBaseServiceModelId(id))
if d.cfgPath != "" {
opts = append(opts, WithBaseServiceConfigPath(d.cfgPath))
}
return NewBaseServiceDelegate(opts...)
}
func NewServiceDelegate() (svc2 interfaces.GrpcClientModelService, err error) {
// service delegate
svc := &ServiceDelegate{
cfgPath: config2.GetConfigPath(),
}
// dependency injection
if err := container.GetContainer().Invoke(func(client interfaces.GrpcClient) {
svc.c = client
}); err != nil {
return nil, err
}
return svc, nil
}

360
core/models/client/model_service_v2.go Normal file
View File

@@ -0,0 +1,360 @@
package client
import (
"encoding/json"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/grpc/client"
"github.com/crawlab-team/crawlab/core/interfaces"
nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
grpc "github.com/crawlab-team/crawlab/grpc"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"reflect"
"sync"
)
var (
instanceMap = make(map[string]interface{})
onceMap = make(map[string]*sync.Once)
mu sync.Mutex
)
type ModelServiceV2[T any] struct {
cfg interfaces.NodeConfigService
c *client.GrpcClientV2
modelType string
}
func (svc *ModelServiceV2[T]) GetById(id primitive.ObjectID) (model *T, err error) {
ctx, cancel := svc.c.Context()
defer cancel()
res, err := svc.c.ModelBaseServiceV2Client.GetById(ctx, &grpc.ModelServiceV2GetByIdRequest{
NodeKey: svc.cfg.GetNodeKey(),
ModelType: svc.modelType,
Id: id.Hex(),
})
if err != nil {
return nil, err
}
return svc.deserializeOne(res)
}
func (svc *ModelServiceV2[T]) GetOne(query bson.M, options *mongo.FindOptions) (model *T, err error) {
ctx, cancel := svc.c.Context()
defer cancel()
queryData, err := json.Marshal(query)
if err != nil {
return nil, err
}
findOptionsData, err := json.Marshal(options)
if err != nil {
return nil, err
}
res, err := svc.c.ModelBaseServiceV2Client.GetOne(ctx, &grpc.ModelServiceV2GetOneRequest{
NodeKey: svc.cfg.GetNodeKey(),
ModelType: svc.modelType,
Query: queryData,
FindOptions: findOptionsData,
})
if err != nil {
return nil, err
}
return svc.deserializeOne(res)
}
func (svc *ModelServiceV2[T]) GetMany(query bson.M, options *mongo.FindOptions) (models []T, err error) {
ctx, cancel := svc.c.Context()
defer cancel()
queryData, err := json.Marshal(query)
if err != nil {
return nil, err
}
findOptionsData, err := json.Marshal(options)
if err != nil {
return nil, err
}
res, err := svc.c.ModelBaseServiceV2Client.GetMany(ctx, &grpc.ModelServiceV2GetManyRequest{
NodeKey: svc.cfg.GetNodeKey(),
ModelType: svc.modelType,
Query: queryData,
FindOptions: findOptionsData,
})
if err != nil {
return nil, err
}
return svc.deserializeMany(res)
}
func (svc *ModelServiceV2[T]) DeleteById(id primitive.ObjectID) (err error) {
ctx, cancel := svc.c.Context()
defer cancel()
_, err = svc.c.ModelBaseServiceV2Client.DeleteById(ctx, &grpc.ModelServiceV2DeleteByIdRequest{
NodeKey: svc.cfg.GetNodeKey(),
ModelType: svc.modelType,
Id: id.Hex(),
})
if err != nil {
return err
}
return nil
}
func (svc *ModelServiceV2[T]) DeleteOne(query bson.M) (err error) {
ctx, cancel := svc.c.Context()
defer cancel()
queryData, err := json.Marshal(query)
if err != nil {
return err
}
_, err = svc.c.ModelBaseServiceV2Client.DeleteOne(ctx, &grpc.ModelServiceV2DeleteOneRequest{
NodeKey: svc.cfg.GetNodeKey(),
ModelType: svc.modelType,
Query: queryData,
})
if err != nil {
return err
}
return nil
}
func (svc *ModelServiceV2[T]) DeleteMany(query bson.M) (err error) {
ctx, cancel := svc.c.Context()
defer cancel()
queryData, err := json.Marshal(query)
if err != nil {
return err
}
_, err = svc.c.ModelBaseServiceV2Client.DeleteMany(ctx, &grpc.ModelServiceV2DeleteManyRequest{
NodeKey: svc.cfg.GetNodeKey(),
ModelType: svc.modelType,
Query: queryData,
})
if err != nil {
return err
}
return nil
}
func (svc *ModelServiceV2[T]) UpdateById(id primitive.ObjectID, update bson.M) (err error) {
ctx, cancel := svc.c.Context()
defer cancel()
updateData, err := json.Marshal(update)
if err != nil {
return err
}
_, err = svc.c.ModelBaseServiceV2Client.UpdateById(ctx, &grpc.ModelServiceV2UpdateByIdRequest{
NodeKey: svc.cfg.GetNodeKey(),
ModelType: svc.modelType,
Id: id.Hex(),
Update: updateData,
})
if err != nil {
return err
}
return nil
}
func (svc *ModelServiceV2[T]) UpdateOne(query bson.M, update bson.M) (err error) {
ctx, cancel := svc.c.Context()
defer cancel()
queryData, err := json.Marshal(query)
if err != nil {
return err
}
updateData, err := json.Marshal(update)
if err != nil {
return err
}
_, err = svc.c.ModelBaseServiceV2Client.UpdateOne(ctx, &grpc.ModelServiceV2UpdateOneRequest{
NodeKey: svc.cfg.GetNodeKey(),
ModelType: svc.modelType,
Query: queryData,
Update: updateData,
})
if err != nil {
return err
}
return nil
}
func (svc *ModelServiceV2[T]) UpdateMany(query bson.M, update bson.M) (err error) {
ctx, cancel := svc.c.Context()
defer cancel()
queryData, err := json.Marshal(query)
if err != nil {
return err
}
updateData, err := json.Marshal(update)
if err != nil {
return err
}
_, err = svc.c.ModelBaseServiceV2Client.UpdateMany(ctx, &grpc.ModelServiceV2UpdateManyRequest{
NodeKey: svc.cfg.GetNodeKey(),
ModelType: svc.modelType,
Query: queryData,
Update: updateData,
})
return nil
}
func (svc *ModelServiceV2[T]) ReplaceById(id primitive.ObjectID, model T) (err error) {
ctx, cancel := svc.c.Context()
defer cancel()
modelData, err := json.Marshal(model)
if err != nil {
return err
}
_, err = svc.c.ModelBaseServiceV2Client.ReplaceById(ctx, &grpc.ModelServiceV2ReplaceByIdRequest{
NodeKey: svc.cfg.GetNodeKey(),
ModelType: svc.modelType,
Id: id.Hex(),
Model: modelData,
})
if err != nil {
return err
}
return nil
}
func (svc *ModelServiceV2[T]) ReplaceOne(query bson.M, model T) (err error) {
ctx, cancel := svc.c.Context()
defer cancel()
queryData, err := json.Marshal(query)
if err != nil {
return err
}
modelData, err := json.Marshal(model)
if err != nil {
return err
}
_, err = svc.c.ModelBaseServiceV2Client.ReplaceOne(ctx, &grpc.ModelServiceV2ReplaceOneRequest{
NodeKey: svc.cfg.GetNodeKey(),
ModelType: svc.modelType,
Query: queryData,
Model: modelData,
})
if err != nil {
return err
}
return nil
}
func (svc *ModelServiceV2[T]) InsertOne(model T) (id primitive.ObjectID, err error) {
ctx, cancel := svc.c.Context()
defer cancel()
modelData, err := json.Marshal(model)
if err != nil {
return primitive.NilObjectID, err
}
res, err := svc.c.ModelBaseServiceV2Client.InsertOne(ctx, &grpc.ModelServiceV2InsertOneRequest{
NodeKey: svc.cfg.GetNodeKey(),
ModelType: svc.modelType,
Model: modelData,
})
if err != nil {
return primitive.NilObjectID, err
}
return deserialize[primitive.ObjectID](res)
//idStr, err := deserialize[string](res)
//if err != nil {
// return primitive.NilObjectID, err
//}
//return primitive.ObjectIDFromHex(idStr)
}
func (svc *ModelServiceV2[T]) InsertMany(models []T) (ids []primitive.ObjectID, err error) {
ctx, cancel := svc.c.Context()
defer cancel()
modelsData, err := json.Marshal(models)
if err != nil {
return nil, err
}
res, err := svc.c.ModelBaseServiceV2Client.InsertMany(ctx, &grpc.ModelServiceV2InsertManyRequest{
NodeKey: svc.cfg.GetNodeKey(),
ModelType: svc.modelType,
Models: modelsData,
})
if err != nil {
return nil, err
}
return deserialize[[]primitive.ObjectID](res)
}
func (svc *ModelServiceV2[T]) Count(query bson.M) (total int, err error) {
ctx, cancel := svc.c.Context()
defer cancel()
queryData, err := json.Marshal(query)
if err != nil {
return 0, err
}
res, err := svc.c.ModelBaseServiceV2Client.Count(ctx, &grpc.ModelServiceV2CountRequest{
NodeKey: svc.cfg.GetNodeKey(),
ModelType: svc.modelType,
Query: queryData,
})
if err != nil {
return 0, err
}
return deserialize[int](res)
}
func (svc *ModelServiceV2[T]) GetCol() (col *mongo.Col) {
return nil
}
func (svc *ModelServiceV2[T]) deserializeOne(res *grpc.Response) (result *T, err error) {
r, err := deserialize[T](res)
if err != nil {
return nil, err
}
return &r, err
}
func (svc *ModelServiceV2[T]) deserializeMany(res *grpc.Response) (results []T, err error) {
return deserialize[[]T](res)
}
func deserialize[T any](res *grpc.Response) (result T, err error) {
err = json.Unmarshal(res.Data, &result)
if err != nil {
return result, err
}
return result, nil
}
func NewModelServiceV2[T any]() *ModelServiceV2[T] {
mu.Lock()
defer mu.Unlock()
var v T
t := reflect.TypeOf(v)
typeName := t.Name()
if _, exists := onceMap[typeName]; !exists {
onceMap[typeName] = &sync.Once{}
}
var instance *ModelServiceV2[T]
c, err := client.GetGrpcClientV2()
if err != nil {
panic(err)
}
if !c.IsStarted() {
err = c.Start()
if err != nil {
panic(err)
}
}
onceMap[typeName].Do(func() {
instance = &ModelServiceV2[T]{
cfg: nodeconfig.GetNodeConfigService(),
c: c,
modelType: typeName,
}
instanceMap[typeName] = instance
})
return instanceMap[typeName].(*ModelServiceV2[T])
}

439
core/models/client/model_service_v2_test.go Normal file
View File

@@ -0,0 +1,439 @@
package client_test
import (
"context"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/grpc/server"
"github.com/crawlab-team/crawlab/core/models/client"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.mongodb.org/mongo-driver/bson"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"testing"
"time"
)
type TestModel models.TestModel
func setupTestDB() {
viper.Set("mongo.db", "testdb")
}
func teardownTestDB() {
db := mongo.GetMongoDb("testdb")
db.Drop(context.Background())
}
func TestModelServiceV2_GetById(t *testing.T) {
setupTestDB()
defer teardownTestDB()
svr, err := server.NewGrpcServerV2()
require.Nil(t, err)
go svr.Start()
defer svr.Stop()
m := TestModel{
Name: "Test Name",
}
modelSvc := service.NewModelServiceV2[TestModel]()
id, err := modelSvc.InsertOne(m)
require.Nil(t, err)
m.SetId(id)
time.Sleep(100 * time.Millisecond)
c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
require.Nil(t, err)
c.Connect()
clientSvc := client.NewModelServiceV2[TestModel]()
res, err := clientSvc.GetById(m.Id)
require.Nil(t, err)
assert.Equal(t, res.Id, m.Id)
assert.Equal(t, res.Name, m.Name)
}
func TestModelServiceV2_GetOne(t *testing.T) {
setupTestDB()
defer teardownTestDB()
svr, err := server.NewGrpcServerV2()
require.Nil(t, err)
go svr.Start()
defer svr.Stop()
m := TestModel{
Name: "Test Name",
}
modelSvc := service.NewModelServiceV2[TestModel]()
id, err := modelSvc.InsertOne(m)
require.Nil(t, err)
m.SetId(id)
time.Sleep(100 * time.Millisecond)
c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
require.Nil(t, err)
c.Connect()
clientSvc := client.NewModelServiceV2[TestModel]()
res, err := clientSvc.GetOne(bson.M{"name": m.Name}, nil)
require.Nil(t, err)
assert.Equal(t, res.Id, m.Id)
assert.Equal(t, res.Name, m.Name)
}
func TestModelServiceV2_GetMany(t *testing.T) {
setupTestDB()
defer teardownTestDB()
svr, err := server.NewGrpcServerV2()
require.Nil(t, err)
go svr.Start()
defer svr.Stop()
m := TestModel{
Name: "Test Name",
}
modelSvc := service.NewModelServiceV2[TestModel]()
id, err := modelSvc.InsertOne(m)
require.Nil(t, err)
m.SetId(id)
time.Sleep(100 * time.Millisecond)
c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
require.Nil(t, err)
c.Connect()
clientSvc := client.NewModelServiceV2[TestModel]()
res, err := clientSvc.GetMany(bson.M{"name": m.Name}, nil)
require.Nil(t, err)
assert.Equal(t, len(res), 1)
assert.Equal(t, res[0].Id, m.Id)
assert.Equal(t, res[0].Name, m.Name)
}
func TestModelServiceV2_DeleteById(t *testing.T) {
setupTestDB()
defer teardownTestDB()
svr, err := server.NewGrpcServerV2()
require.Nil(t, err)
go svr.Start()
defer svr.Stop()
m := TestModel{
Name: "Test Name",
}
modelSvc := service.NewModelServiceV2[TestModel]()
id, err := modelSvc.InsertOne(m)
require.Nil(t, err)
m.SetId(id)
time.Sleep(100 * time.Millisecond)
c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
require.Nil(t, err)
c.Connect()
clientSvc := client.NewModelServiceV2[TestModel]()
err = clientSvc.DeleteById(m.Id)
require.Nil(t, err)
res, err := clientSvc.GetById(m.Id)
assert.NotNil(t, err)
require.Nil(t, res)
}
func TestModelServiceV2_DeleteOne(t *testing.T) {
setupTestDB()
defer teardownTestDB()
svr, err := server.NewGrpcServerV2()
require.Nil(t, err)
go svr.Start()
defer svr.Stop()
m := TestModel{
Name: "Test Name",
}
modelSvc := service.NewModelServiceV2[TestModel]()
id, err := modelSvc.InsertOne(m)
require.Nil(t, err)
m.SetId(id)
time.Sleep(100 * time.Millisecond)
c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
require.Nil(t, err)
c.Connect()
clientSvc := client.NewModelServiceV2[TestModel]()
err = clientSvc.DeleteOne(bson.M{"name": m.Name})
require.Nil(t, err)
res, err := clientSvc.GetOne(bson.M{"name": m.Name}, nil)
assert.NotNil(t, err)
require.Nil(t, res)
}
func TestModelServiceV2_DeleteMany(t *testing.T) {
setupTestDB()
defer teardownTestDB()
svr, err := server.NewGrpcServerV2()
require.Nil(t, err)
go svr.Start()
defer svr.Stop()
m := TestModel{
Name: "Test Name",
}
modelSvc := service.NewModelServiceV2[TestModel]()
id, err := modelSvc.InsertOne(m)
require.Nil(t, err)
m.SetId(id)
time.Sleep(100 * time.Millisecond)
c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
require.Nil(t, err)
c.Connect()
clientSvc := client.NewModelServiceV2[TestModel]()
err = clientSvc.DeleteMany(bson.M{"name": m.Name})
require.Nil(t, err)
res, err := clientSvc.GetMany(bson.M{"name": m.Name}, nil)
require.Nil(t, err)
assert.Equal(t, len(res), 0)
}
func TestModelServiceV2_UpdateById(t *testing.T) {
setupTestDB()
defer teardownTestDB()
svr, err := server.NewGrpcServerV2()
require.Nil(t, err)
go svr.Start()
defer svr.Stop()
m := TestModel{
Name: "Test Name",
}
modelSvc := service.NewModelServiceV2[TestModel]()
id, err := modelSvc.InsertOne(m)
require.Nil(t, err)
m.SetId(id)
time.Sleep(100 * time.Millisecond)
c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
require.Nil(t, err)
c.Connect()
clientSvc := client.NewModelServiceV2[TestModel]()
err = clientSvc.UpdateById(m.Id, bson.M{"$set": bson.M{"name": "New Name"}})
require.Nil(t, err)
res, err := clientSvc.GetById(m.Id)
require.Nil(t, err)
assert.Equal(t, res.Name, "New Name")
}
func TestModelServiceV2_UpdateOne(t *testing.T) {
setupTestDB()
defer teardownTestDB()
svr, err := server.NewGrpcServerV2()
require.Nil(t, err)
go svr.Start()
defer svr.Stop()
m := TestModel{
Name: "Test Name",
}
modelSvc := service.NewModelServiceV2[TestModel]()
id, err := modelSvc.InsertOne(m)
require.Nil(t, err)
m.SetId(id)
time.Sleep(100 * time.Millisecond)
c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
require.Nil(t, err)
c.Connect()
clientSvc := client.NewModelServiceV2[TestModel]()
err = clientSvc.UpdateOne(bson.M{"name": m.Name}, bson.M{"$set": bson.M{"name": "New Name"}})
require.Nil(t, err)
res, err := clientSvc.GetOne(bson.M{"name": "New Name"}, nil)
require.Nil(t, err)
assert.Equal(t, res.Name, "New Name")
}
func TestModelServiceV2_UpdateMany(t *testing.T) {
setupTestDB()
defer teardownTestDB()
svr, err := server.NewGrpcServerV2()
require.Nil(t, err)
go svr.Start()
defer svr.Stop()
m1 := TestModel{
Name: "Test Name",
}
m2 := TestModel{
Name: "Test Name",
}
modelSvc := service.NewModelServiceV2[TestModel]()
_, err = modelSvc.InsertOne(m1)
require.Nil(t, err)
_, err = modelSvc.InsertOne(m2)
require.Nil(t, err)
time.Sleep(100 * time.Millisecond)
c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
require.Nil(t, err)
c.Connect()
clientSvc := client.NewModelServiceV2[TestModel]()
err = clientSvc.UpdateMany(bson.M{"name": "Test Name"}, bson.M{"$set": bson.M{"name": "New Name"}})
require.Nil(t, err)
res, err := clientSvc.GetMany(bson.M{"name": "New Name"}, nil)
require.Nil(t, err)
assert.Equal(t, len(res), 2)
}
func TestModelServiceV2_ReplaceById(t *testing.T) {
setupTestDB()
defer teardownTestDB()
svr, err := server.NewGrpcServerV2()
require.Nil(t, err)
go svr.Start()
defer svr.Stop()
m := TestModel{
Name: "Test Name",
}
modelSvc := service.NewModelServiceV2[TestModel]()
id, err := modelSvc.InsertOne(m)
require.Nil(t, err)
m.SetId(id)
time.Sleep(100 * time.Millisecond)
c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
require.Nil(t, err)
c.Connect()
clientSvc := client.NewModelServiceV2[TestModel]()
m.Name = "New Name"
err = clientSvc.ReplaceById(m.Id, m)
require.Nil(t, err)
res, err := clientSvc.GetById(m.Id)
require.Nil(t, err)
assert.Equal(t, res.Name, "New Name")
}
func TestModelServiceV2_ReplaceOne(t *testing.T) {
setupTestDB()
defer teardownTestDB()
svr, err := server.NewGrpcServerV2()
require.Nil(t, err)
go svr.Start()
defer svr.Stop()
m := TestModel{
Name: "Test Name",
}
modelSvc := service.NewModelServiceV2[TestModel]()
id, err := modelSvc.InsertOne(m)
require.Nil(t, err)
m.SetId(id)
time.Sleep(100 * time.Millisecond)
c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
require.Nil(t, err)
c.Connect()
clientSvc := client.NewModelServiceV2[TestModel]()
m.Name = "New Name"
err = clientSvc.ReplaceOne(bson.M{"name": "Test Name"}, m)
require.Nil(t, err)
res, err := clientSvc.GetOne(bson.M{"name": "New Name"}, nil)
require.Nil(t, err)
assert.Equal(t, res.Name, "New Name")
}
func TestModelServiceV2_InsertOne(t *testing.T) {
setupTestDB()
defer teardownTestDB()
svr, err := server.NewGrpcServerV2()
require.Nil(t, err)
go svr.Start()
defer svr.Stop()
c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
require.Nil(t, err)
c.Connect()
clientSvc := client.NewModelServiceV2[TestModel]()
m := TestModel{
Name: "Test Name",
}
id, err := clientSvc.InsertOne(m)
require.Nil(t, err)
res, err := clientSvc.GetById(id)
require.Nil(t, err)
assert.Equal(t, res.Name, m.Name)
}
func TestModelServiceV2_InsertMany(t *testing.T) {
setupTestDB()
defer teardownTestDB()
svr, err := server.NewGrpcServerV2()
require.Nil(t, err)
go svr.Start()
defer svr.Stop()
c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
require.Nil(t, err)
c.Connect()
clientSvc := client.NewModelServiceV2[TestModel]()
models := []TestModel{
{Name: "Test Name 1"},
{Name: "Test Name 2"},
}
ids, err := clientSvc.InsertMany(models)
require.Nil(t, err)
for i, id := range ids {
res, err := clientSvc.GetById(id)
require.Nil(t, err)
assert.Equal(t, res.Name, models[i].Name)
}
}
func TestModelServiceV2_Count(t *testing.T) {
setupTestDB()
defer teardownTestDB()
svr, err := server.NewGrpcServerV2()
require.Nil(t, err)
go svr.Start()
defer svr.Stop()
modelSvc := service.NewModelServiceV2[TestModel]()
for i := 0; i < 5; i++ {
_, err = modelSvc.InsertOne(TestModel{
Name: "Test Name",
})
require.Nil(t, err)
}
time.Sleep(100 * time.Millisecond)
c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
require.Nil(t, err)
c.Connect()
clientSvc := client.NewModelServiceV2[TestModel]()
count, err := clientSvc.Count(bson.M{})
require.Nil(t, err)
assert.Equal(t, count, 5)
}

70
core/models/client/model_spider_service.go Normal file
View File

@@ -0,0 +1,70 @@
package client
import (
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type SpiderServiceDelegate struct {
interfaces.GrpcClientModelBaseService
}
func (svc *SpiderServiceDelegate) GetSpiderById(id primitive.ObjectID) (s interfaces.Spider, err error) {
res, err := svc.GetById(id)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Spider)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *SpiderServiceDelegate) GetSpider(query bson.M, opts *mongo.FindOptions) (s interfaces.Spider, err error) {
res, err := svc.Get(query, opts)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Spider)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *SpiderServiceDelegate) GetSpiderList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Spider, err error) {
list, err := svc.GetList(query, opts)
if err != nil {
return nil, err
}
for _, item := range list.GetModels() {
s, ok := item.(interfaces.Spider)
if !ok {
return nil, errors.ErrorModelInvalidType
}
res = append(res, s)
}
return res, nil
}
func NewSpiderServiceDelegate() (svc2 interfaces.GrpcClientModelSpiderService, err error) {
var opts []ModelBaseServiceDelegateOption
// apply options
opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdSpider))
// base service
baseSvc, err := NewBaseServiceDelegate(opts...)
if err != nil {
return nil, err
}
// service
svc := &SpiderServiceDelegate{baseSvc}
return svc, nil
}

70
core/models/client/model_task_service.go Normal file
View File

@@ -0,0 +1,70 @@
package client
import (
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type TaskServiceDelegate struct {
interfaces.GrpcClientModelBaseService
}
func (svc *TaskServiceDelegate) GetTaskById(id primitive.ObjectID) (t interfaces.Task, err error) {
res, err := svc.GetById(id)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Task)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *TaskServiceDelegate) GetTask(query bson.M, opts *mongo.FindOptions) (t interfaces.Task, err error) {
res, err := svc.Get(query, opts)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Task)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *TaskServiceDelegate) GetTaskList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Task, err error) {
list, err := svc.GetList(query, opts)
if err != nil {
return nil, err
}
for _, item := range list.GetModels() {
s, ok := item.(interfaces.Task)
if !ok {
return nil, errors.ErrorModelInvalidType
}
res = append(res, s)
}
return res, nil
}
func NewTaskServiceDelegate() (svc2 interfaces.GrpcClientModelTaskService, err error) {
var opts []ModelBaseServiceDelegateOption
// apply options
opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdTask))
// base service
baseSvc, err := NewBaseServiceDelegate(opts...)
if err != nil {
return nil, err
}
// service
svc := &TaskServiceDelegate{baseSvc}
return svc, nil
}

70
core/models/client/model_task_stat_service.go Normal file
View File

@@ -0,0 +1,70 @@
package client
import (
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type TaskStatServiceDelegate struct {
interfaces.GrpcClientModelBaseService
}
func (svc *TaskStatServiceDelegate) GetTaskStatById(id primitive.ObjectID) (t interfaces.TaskStat, err error) {
res, err := svc.GetById(id)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.TaskStat)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *TaskStatServiceDelegate) GetTaskStat(query bson.M, opts *mongo.FindOptions) (t interfaces.TaskStat, err error) {
res, err := svc.Get(query, opts)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.TaskStat)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *TaskStatServiceDelegate) GetTaskStatList(query bson.M, opts *mongo.FindOptions) (res []interfaces.TaskStat, err error) {
list, err := svc.GetList(query, opts)
if err != nil {
return nil, err
}
for _, item := range list.GetModels() {
s, ok := item.(interfaces.TaskStat)
if !ok {
return nil, errors.ErrorModelInvalidType
}
res = append(res, s)
}
return res, nil
}
func NewTaskStatServiceDelegate() (svc2 interfaces.GrpcClientModelTaskStatService, err error) {
var opts []ModelBaseServiceDelegateOption
// apply options
opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdTaskStat))
// base service
baseSvc, err := NewBaseServiceDelegate(opts...)
if err != nil {
return nil, err
}
// service
svc := &TaskStatServiceDelegate{baseSvc}
return svc, nil
}

33
core/models/client/options.go Normal file
View File

@@ -0,0 +1,33 @@
package client
import "github.com/crawlab-team/crawlab/core/interfaces"
type ModelDelegateOption func(delegate interfaces.GrpcClientModelDelegate)
func WithDelegateConfigPath(path string) ModelDelegateOption {
return func(d interfaces.GrpcClientModelDelegate) {
d.SetConfigPath(path)
}
}
type ModelServiceDelegateOption func(delegate interfaces.GrpcClientModelService)
func WithServiceConfigPath(path string) ModelServiceDelegateOption {
return func(d interfaces.GrpcClientModelService) {
d.SetConfigPath(path)
}
}
type ModelBaseServiceDelegateOption func(delegate interfaces.GrpcClientModelBaseService)
func WithBaseServiceModelId(id interfaces.ModelId) ModelBaseServiceDelegateOption {
return func(d interfaces.GrpcClientModelBaseService) {
d.SetModelId(id)
}
}
func WithBaseServiceConfigPath(path string) ModelBaseServiceDelegateOption {
return func(d interfaces.GrpcClientModelBaseService) {
d.SetConfigPath(path)
}
}