feat: added modules

core/models/client/binder_basic.go

@@ -0,0 +1,92 @@
+package client
+
+import (
+	"encoding/json"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	grpc "github.com/crawlab-team/crawlab/grpc"
+	"github.com/crawlab-team/go-trace"
+)
+
+func NewBasicBinder(id interfaces.ModelId, res *grpc.Response) (b interfaces.GrpcModelBinder) {
+	return &BasicBinder{
+		id:  id,
+		res: res,
+	}
+}
+
+type BasicBinder struct {
+	id  interfaces.ModelId
+	res *grpc.Response
+}
+
+func (b *BasicBinder) Bind() (res interfaces.Model, err error) {
+	m := models.NewModelMap()
+
+	switch b.id {
+	case interfaces.ModelIdArtifact:
+		return b.Process(&m.Artifact)
+	case interfaces.ModelIdTag:
+		return b.Process(&m.Tag)
+	case interfaces.ModelIdNode:
+		return b.Process(&m.Node)
+	case interfaces.ModelIdProject:
+		return b.Process(&m.Project)
+	case interfaces.ModelIdSpider:
+		return b.Process(&m.Spider)
+	case interfaces.ModelIdTask:
+		return b.Process(&m.Task)
+	case interfaces.ModelIdJob:
+		return b.Process(&m.Job)
+	case interfaces.ModelIdSchedule:
+		return b.Process(&m.Schedule)
+	case interfaces.ModelIdUser:
+		return b.Process(&m.User)
+	case interfaces.ModelIdSetting:
+		return b.Process(&m.Setting)
+	case interfaces.ModelIdToken:
+		return b.Process(&m.Token)
+	case interfaces.ModelIdVariable:
+		return b.Process(&m.Variable)
+	case interfaces.ModelIdTaskQueue:
+		return b.Process(&m.TaskQueueItem)
+	case interfaces.ModelIdTaskStat:
+		return b.Process(&m.TaskStat)
+	case interfaces.ModelIdSpiderStat:
+		return b.Process(&m.SpiderStat)
+	case interfaces.ModelIdDataSource:
+		return b.Process(&m.DataSource)
+	case interfaces.ModelIdDataCollection:
+		return b.Process(&m.DataCollection)
+	case interfaces.ModelIdResult:
+		return b.Process(&m.Result)
+	case interfaces.ModelIdPassword:
+		return b.Process(&m.Password)
+	case interfaces.ModelIdExtraValue:
+		return b.Process(&m.ExtraValue)
+	case interfaces.ModelIdGit:
+		return b.Process(&m.Git)
+	case interfaces.ModelIdRole:
+		return b.Process(&m.Role)
+	case interfaces.ModelIdUserRole:
+		return b.Process(&m.UserRole)
+	case interfaces.ModelIdPermission:
+		return b.Process(&m.Permission)
+	case interfaces.ModelIdRolePermission:
+		return b.Process(&m.RolePermission)
+	case interfaces.ModelIdEnvironment:
+		return b.Process(&m.Environment)
+	case interfaces.ModelIdDependencySetting:
+		return b.Process(&m.DependencySetting)
+	default:
+		return nil, errors.ErrorModelInvalidModelId
+	}
+}
+
+func (b *BasicBinder) Process(d interfaces.Model) (res interfaces.Model, err error) {
+	if err := json.Unmarshal(b.res.Data, d); err != nil {
+		return nil, trace.TraceError(err)
+	}
+	return d, nil
+}

core/models/client/binder_list.go

@@ -0,0 +1,100 @@
+package client
+
+import (
+	"encoding/json"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	grpc "github.com/crawlab-team/crawlab/grpc"
+	"github.com/crawlab-team/go-trace"
+)
+
+func NewListBinder(id interfaces.ModelId, res *grpc.Response) (b interfaces.GrpcModelListBinder) {
+	return &ListBinder{
+		id:  id,
+		res: res,
+	}
+}
+
+type ListBinder struct {
+	id  interfaces.ModelId
+	res *grpc.Response
+}
+
+func (b *ListBinder) Bind() (l interfaces.List, err error) {
+	m := models.NewModelListMap()
+
+	switch b.id {
+	case interfaces.ModelIdArtifact:
+		return b.Process(&m.Artifacts)
+	case interfaces.ModelIdTag:
+		return b.Process(&m.Tags)
+	case interfaces.ModelIdNode:
+		return b.Process(&m.Nodes)
+	case interfaces.ModelIdProject:
+		return b.Process(&m.Projects)
+	case interfaces.ModelIdSpider:
+		return b.Process(&m.Spiders)
+	case interfaces.ModelIdTask:
+		return b.Process(&m.Tasks)
+	case interfaces.ModelIdJob:
+		return b.Process(&m.Jobs)
+	case interfaces.ModelIdSchedule:
+		return b.Process(&m.Schedules)
+	case interfaces.ModelIdUser:
+		return b.Process(&m.Users)
+	case interfaces.ModelIdSetting:
+		return b.Process(&m.Settings)
+	case interfaces.ModelIdToken:
+		return b.Process(&m.Tokens)
+	case interfaces.ModelIdVariable:
+		return b.Process(&m.Variables)
+	case interfaces.ModelIdTaskQueue:
+		return b.Process(&m.TaskQueueItems)
+	case interfaces.ModelIdTaskStat:
+		return b.Process(&m.TaskStats)
+	case interfaces.ModelIdSpiderStat:
+		return b.Process(&m.SpiderStats)
+	case interfaces.ModelIdDataSource:
+		return b.Process(&m.DataSources)
+	case interfaces.ModelIdDataCollection:
+		return b.Process(&m.DataCollections)
+	case interfaces.ModelIdResult:
+		return b.Process(&m.Results)
+	case interfaces.ModelIdPassword:
+		return b.Process(&m.Passwords)
+	case interfaces.ModelIdExtraValue:
+		return b.Process(&m.ExtraValues)
+	case interfaces.ModelIdGit:
+		return b.Process(&m.Gits)
+	case interfaces.ModelIdRole:
+		return b.Process(&m.Roles)
+	case interfaces.ModelIdUserRole:
+		return b.Process(&m.UserRoles)
+	case interfaces.ModelIdPermission:
+		return b.Process(&m.PermissionList)
+	case interfaces.ModelIdRolePermission:
+		return b.Process(&m.RolePermissionList)
+	case interfaces.ModelIdEnvironment:
+		return b.Process(&m.Environments)
+	case interfaces.ModelIdDependencySetting:
+		return b.Process(&m.DependencySettings)
+	default:
+		return l, errors.ErrorModelInvalidModelId
+	}
+}
+
+func (b *ListBinder) MustBind() (res interface{}) {
+	res, err := b.Bind()
+	if err != nil {
+		panic(err)
+	}
+	return res
+}
+
+func (b *ListBinder) Process(d interface{}) (l interfaces.List, err error) {
+	if err := json.Unmarshal(b.res.Data, d); err != nil {
+		return l, trace.TraceError(err)
+	}
+	return d.(interfaces.List), nil
+}

core/models/client/model_base_service.go

@@ -0,0 +1,251 @@
+package client
+
+import (
+	"encoding/json"
+	"github.com/apex/log"
+	"github.com/cenkalti/backoff/v4"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/utils"
+	grpc "github.com/crawlab-team/crawlab/grpc"
+	"github.com/crawlab-team/go-trace"
+	errors2 "github.com/pkg/errors"
+	"github.com/spf13/viper"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type BaseServiceDelegate struct {
+	// settings
+	cfgPath string
+
+	// internals
+	id interfaces.ModelId
+	c  interfaces.GrpcClient
+}
+
+func (d *BaseServiceDelegate) GetModelId() (id interfaces.ModelId) {
+	return d.id
+}
+
+func (d *BaseServiceDelegate) SetModelId(id interfaces.ModelId) {
+	d.id = id
+}
+
+func (d *BaseServiceDelegate) GetConfigPath() (path string) {
+	return d.cfgPath
+}
+
+func (d *BaseServiceDelegate) SetConfigPath(path string) {
+	d.cfgPath = path
+}
+
+func (d *BaseServiceDelegate) GetById(id primitive.ObjectID) (doc interfaces.Model, err error) {
+	log.Debugf("[BaseServiceDelegate] get by id[%s]", id.Hex())
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id})
+	c := d.getModelBaseServiceClient()
+	if c == nil {
+		return nil, trace.TraceError(errors.ErrorModelNilPointer)
+	}
+	log.Debugf("[BaseServiceDelegate] get by id[%s] req: %v", id.Hex(), req)
+	res, err := c.GetById(ctx, req)
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+	log.Debugf("[BaseServiceDelegate] get by id[%s] res: %v", id.Hex(), res)
+	return NewBasicBinder(d.id, res).Bind()
+}
+
+func (d *BaseServiceDelegate) Get(query bson.M, opts *mongo.FindOptions) (doc interfaces.Model, err error) {
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, FindOptions: opts})
+	res, err := d.getModelBaseServiceClient().Get(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	return NewBasicBinder(d.id, res).Bind()
+}
+
+func (d *BaseServiceDelegate) GetList(query bson.M, opts *mongo.FindOptions) (l interfaces.List, err error) {
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, FindOptions: opts})
+	res, err := d.getModelBaseServiceClient().GetList(ctx, req)
+	if err != nil {
+		return l, err
+	}
+	return NewListBinder(d.id, res).Bind()
+}
+
+func (d *BaseServiceDelegate) DeleteById(id primitive.ObjectID, args ...interface{}) (err error) {
+	u := utils.GetUserFromArgs(args...)
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id, User: u})
+	_, err = d.getModelBaseServiceClient().DeleteById(ctx, req)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *BaseServiceDelegate) Delete(query bson.M, args ...interface{}) (err error) {
+	u := utils.GetUserFromArgs(args...)
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u})
+	_, err = d.getModelBaseServiceClient().Delete(ctx, req)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *BaseServiceDelegate) DeleteList(query bson.M, args ...interface{}) (err error) {
+	u := utils.GetUserFromArgs(args...)
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u})
+	_, err = d.getModelBaseServiceClient().DeleteList(ctx, req)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *BaseServiceDelegate) ForceDeleteList(query bson.M, args ...interface{}) (err error) {
+	u := utils.GetUserFromArgs(args...)
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u})
+	_, err = d.getModelBaseServiceClient().ForceDeleteList(ctx, req)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *BaseServiceDelegate) UpdateById(id primitive.ObjectID, update bson.M, args ...interface{}) (err error) {
+	u := utils.GetUserFromArgs(args...)
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id, Update: update, User: u})
+	_, err = d.getModelBaseServiceClient().UpdateById(ctx, req)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *BaseServiceDelegate) Update(query bson.M, update bson.M, fields []string, args ...interface{}) (err error) {
+	u := utils.GetUserFromArgs(args...)
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, Update: update, Fields: fields, User: u})
+	_, err = d.getModelBaseServiceClient().Update(ctx, req)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *BaseServiceDelegate) UpdateDoc(query bson.M, doc interfaces.Model, fields []string, args ...interface{}) (err error) {
+	u := utils.GetUserFromArgs(args...)
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, Doc: doc, Fields: fields, User: u})
+	_, err = d.getModelBaseServiceClient().UpdateDoc(ctx, req)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *BaseServiceDelegate) Insert(u interfaces.User, docs ...interface{}) (err error) {
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Docs: docs, User: u})
+	_, err = d.getModelBaseServiceClient().Insert(ctx, req)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *BaseServiceDelegate) Count(query bson.M) (total int, err error) {
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query})
+	res, err := d.getModelBaseServiceClient().Count(ctx, req)
+	if err != nil {
+		return total, err
+	}
+	if err := json.Unmarshal(res.Data, &total); err != nil {
+		return total, err
+	}
+	return total, nil
+}
+
+func (d *BaseServiceDelegate) newRequest(params interfaces.GrpcBaseServiceParams) (req *grpc.Request, err error) {
+	return d.c.NewModelBaseServiceRequest(d.id, params)
+}
+
+func (d *BaseServiceDelegate) mustNewRequest(params *entity.GrpcBaseServiceParams) (req *grpc.Request) {
+	req, err := d.newRequest(params)
+	if err != nil {
+		panic(err)
+	}
+	return req
+}
+
+func (d *BaseServiceDelegate) getModelBaseServiceClient() (c grpc.ModelBaseServiceClient) {
+	if err := backoff.Retry(func() (err error) {
+		c = d.c.GetModelBaseServiceClient()
+		if c == nil {
+			err = errors2.New("unable to get model base service client")
+			log.Debugf("[BaseServiceDelegate] err: %v", err)
+			return err
+		}
+		return nil
+	}, backoff.NewConstantBackOff(1*time.Second)); err != nil {
+		trace.PrintError(err)
+	}
+	return c
+}
+
+func NewBaseServiceDelegate(opts ...ModelBaseServiceDelegateOption) (svc2 interfaces.GrpcClientModelBaseService, err error) {
+	// base service
+	svc := &BaseServiceDelegate{}
+
+	// apply options
+	for _, opt := range opts {
+		opt(svc)
+	}
+
+	// config path
+	if viper.GetString("config.path") != "" {
+		svc.cfgPath = viper.GetString("config.path")
+	}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(client interfaces.GrpcClient) {
+		svc.c = client
+	}); err != nil {
+		return nil, err
+	}
+
+	return svc, nil
+}
+
+func ProvideBaseServiceDelegate(id interfaces.ModelId) func() (svc interfaces.GrpcClientModelBaseService, err error) {
+	return func() (svc interfaces.GrpcClientModelBaseService, err error) {
+		return NewBaseServiceDelegate(WithBaseServiceModelId(id))
+	}
+}

core/models/client/model_delegate.go

@@ -0,0 +1,329 @@
+package client
+
+import (
+	"encoding/json"
+	config2 "github.com/crawlab-team/crawlab/core/config"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/grpc/client"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/utils"
+	grpc "github.com/crawlab-team/crawlab/grpc"
+	"github.com/crawlab-team/go-trace"
+	"github.com/spf13/viper"
+)
+
+func NewModelDelegate(doc interfaces.Model, opts ...ModelDelegateOption) interfaces.GrpcClientModelDelegate {
+	switch doc.(type) {
+	case *models.Artifact:
+		return newModelDelegate(interfaces.ModelIdArtifact, doc, opts...)
+	case *models.Tag:
+		return newModelDelegate(interfaces.ModelIdTag, doc, opts...)
+	case *models.Node:
+		return newModelDelegate(interfaces.ModelIdNode, doc, opts...)
+	case *models.Project:
+		return newModelDelegate(interfaces.ModelIdProject, doc, opts...)
+	case *models.Spider:
+		return newModelDelegate(interfaces.ModelIdSpider, doc, opts...)
+	case *models.Task:
+		return newModelDelegate(interfaces.ModelIdTask, doc, opts...)
+	case *models.Job:
+		return newModelDelegate(interfaces.ModelIdJob, doc, opts...)
+	case *models.Schedule:
+		return newModelDelegate(interfaces.ModelIdSchedule, doc, opts...)
+	case *models.User:
+		return newModelDelegate(interfaces.ModelIdUser, doc, opts...)
+	case *models.Setting:
+		return newModelDelegate(interfaces.ModelIdSetting, doc, opts...)
+	case *models.Token:
+		return newModelDelegate(interfaces.ModelIdToken, doc, opts...)
+	case *models.Variable:
+		return newModelDelegate(interfaces.ModelIdVariable, doc, opts...)
+	case *models.TaskQueueItem:
+		return newModelDelegate(interfaces.ModelIdTaskQueue, doc, opts...)
+	case *models.TaskStat:
+		return newModelDelegate(interfaces.ModelIdTaskStat, doc, opts...)
+	case *models.SpiderStat:
+		return newModelDelegate(interfaces.ModelIdSpiderStat, doc, opts...)
+	case *models.DataSource:
+		return newModelDelegate(interfaces.ModelIdDataSource, doc, opts...)
+	case *models.DataCollection:
+		return newModelDelegate(interfaces.ModelIdDataCollection, doc, opts...)
+	case *models.Result:
+		return newModelDelegate(interfaces.ModelIdResult, doc, opts...)
+	case *models.Password:
+		return newModelDelegate(interfaces.ModelIdPassword, doc, opts...)
+	case *models.ExtraValue:
+		return newModelDelegate(interfaces.ModelIdExtraValue, doc, opts...)
+	case *models.Git:
+		return newModelDelegate(interfaces.ModelIdGit, doc, opts...)
+	case *models.UserRole:
+		return newModelDelegate(interfaces.ModelIdUserRole, doc, opts...)
+	case *models.Permission:
+		return newModelDelegate(interfaces.ModelIdPermission, doc, opts...)
+	case *models.RolePermission:
+		return newModelDelegate(interfaces.ModelIdRolePermission, doc, opts...)
+	case *models.Environment:
+		return newModelDelegate(interfaces.ModelIdEnvironment, doc, opts...)
+	case *models.DependencySetting:
+		return newModelDelegate(interfaces.ModelIdDependencySetting, doc, opts...)
+	default:
+		_ = trace.TraceError(errors.ErrorModelInvalidType)
+		return nil
+	}
+}
+
+func newModelDelegate(id interfaces.ModelId, doc interfaces.Model, opts ...ModelDelegateOption) interfaces.GrpcClientModelDelegate {
+	var err error
+
+	// collection name
+	colName := models.GetModelColName(id)
+
+	// model delegate
+	d := &ModelDelegate{
+		id:      id,
+		colName: colName,
+		doc:     doc,
+		cfgPath: config2.GetConfigPath(),
+		a: &models.Artifact{
+			Col: colName,
+		},
+	}
+
+	// config path
+	if viper.GetString("config.path") != "" {
+		d.cfgPath = viper.GetString("config.path")
+	}
+
+	// apply options
+	for _, opt := range opts {
+		opt(d)
+	}
+
+	// grpc client
+	d.c, err = client.GetClient()
+	if err != nil {
+		trace.PrintError(errors.ErrorModelInvalidType)
+		return nil
+	}
+	if !d.c.IsStarted() {
+		if err := d.c.Start(); err != nil {
+			trace.PrintError(err)
+			return nil
+		}
+	} else if d.c.IsClosed() {
+		if err := d.c.Restart(); err != nil {
+			trace.PrintError(err)
+			return nil
+		}
+	}
+
+	return d
+}
+
+type ModelDelegate struct {
+	// settings
+	cfgPath string
+
+	// internals
+	id      interfaces.ModelId
+	colName string
+	c       interfaces.GrpcClient
+	doc     interfaces.Model
+	a       interfaces.ModelArtifact
+}
+
+func (d *ModelDelegate) Add() (err error) {
+	return d.do(interfaces.ModelDelegateMethodAdd)
+}
+
+func (d *ModelDelegate) Save() (err error) {
+	return d.do(interfaces.ModelDelegateMethodSave)
+}
+
+func (d *ModelDelegate) Delete() (err error) {
+	return d.do(interfaces.ModelDelegateMethodDelete)
+}
+
+func (d *ModelDelegate) GetArtifact() (res interfaces.ModelArtifact, err error) {
+	if err := d.do(interfaces.ModelDelegateMethodGetArtifact); err != nil {
+		return nil, err
+	}
+	return d.a, nil
+}
+
+func (d *ModelDelegate) GetModel() (res interfaces.Model) {
+	return d.doc
+}
+
+func (d *ModelDelegate) Refresh() (err error) {
+	return d.refresh()
+}
+
+func (d *ModelDelegate) GetConfigPath() (path string) {
+	return d.cfgPath
+}
+
+func (d *ModelDelegate) SetConfigPath(path string) {
+	d.cfgPath = path
+}
+
+func (d *ModelDelegate) Close() (err error) {
+	return d.c.Stop()
+}
+
+func (d *ModelDelegate) ToBytes(m interface{}) (bytes []byte, err error) {
+	if m != nil {
+		return utils.JsonToBytes(m)
+	}
+	return json.Marshal(d.doc)
+}
+
+func (d *ModelDelegate) do(method interfaces.ModelDelegateMethod) (err error) {
+	switch method {
+	case interfaces.ModelDelegateMethodAdd:
+		err = d.add()
+	case interfaces.ModelDelegateMethodSave:
+		err = d.save()
+	case interfaces.ModelDelegateMethodDelete:
+		err = d.delete()
+	case interfaces.ModelDelegateMethodGetArtifact, interfaces.ModelDelegateMethodRefresh:
+		return d.refresh()
+	default:
+		return trace.TraceError(errors.ErrorModelInvalidType)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *ModelDelegate) add() (err error) {
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodAdd)
+	res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
+		ModelId: d.id,
+		Method:  method,
+		Data:    d.mustGetData(),
+	}))
+	if err != nil {
+		return trace.TraceError(err)
+	}
+	if err := d.deserialize(res, method); err != nil {
+		return err
+	}
+	return d.refreshArtifact()
+}
+
+func (d *ModelDelegate) save() (err error) {
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodSave)
+	res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
+		ModelId: d.id,
+		Method:  method,
+		Data:    d.mustGetData(),
+	}))
+	if err != nil {
+		return trace.TraceError(err)
+	}
+	if err := d.deserialize(res, method); err != nil {
+		return err
+	}
+	return d.refreshArtifact()
+}
+
+func (d *ModelDelegate) delete() (err error) {
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodDelete)
+	res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
+		ModelId: d.id,
+		Method:  method,
+		Data:    d.mustGetData(),
+	}))
+	if err != nil {
+		return trace.TraceError(err)
+	}
+	if err := d.deserialize(res, method); err != nil {
+		return err
+	}
+	return d.refreshArtifact()
+}
+
+func (d *ModelDelegate) refresh() (err error) {
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodRefresh)
+	res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
+		ModelId: d.id,
+		Method:  method,
+		Data:    d.mustGetData(),
+	}))
+	if err != nil {
+		return trace.TraceError(err)
+	}
+	if err := d.deserialize(res, method); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *ModelDelegate) refreshArtifact() (err error) {
+	_, err = d.getArtifact()
+	return err
+}
+
+func (d *ModelDelegate) getArtifact() (res2 interfaces.ModelArtifact, err error) {
+	ctx, cancel := d.c.Context()
+	defer cancel()
+	method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodGetArtifact)
+	res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
+		ModelId: d.id,
+		Method:  method,
+		Data:    d.mustGetData(),
+	}))
+	if err != nil {
+		return nil, err
+	}
+	if err := d.deserialize(res, method); err != nil {
+		return nil, err
+	}
+	return d.a, nil
+}
+
+func (d *ModelDelegate) mustGetData() (data []byte) {
+	data, err := d.getData()
+	if err != nil {
+		panic(err)
+	}
+	return data
+}
+
+func (d *ModelDelegate) getData() (data []byte, err error) {
+	return json.Marshal(d.doc)
+}
+
+func (d *ModelDelegate) deserialize(res *grpc.Response, method interfaces.ModelDelegateMethod) (err error) {
+	if method == interfaces.ModelDelegateMethodGetArtifact {
+		res, err := NewBasicBinder(interfaces.ModelIdArtifact, res).Bind()
+		if err != nil {
+			return err
+		}
+		a, ok := res.(interfaces.ModelArtifact)
+		if !ok {
+			return trace.TraceError(errors.ErrorModelInvalidType)
+		}
+		d.a = a
+	} else {
+		d.doc, err = NewBasicBinder(d.id, res).Bind()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

core/models/client/model_environment_service.go

@@ -0,0 +1,70 @@
+package client
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type EnvironmentServiceDelegate struct {
+	interfaces.GrpcClientModelBaseService
+}
+
+func (svc *EnvironmentServiceDelegate) GetEnvironmentById(id primitive.ObjectID) (e interfaces.Environment, err error) {
+	res, err := svc.GetById(id)
+	if err != nil {
+		return nil, err
+	}
+	s, ok := res.(interfaces.Environment)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return s, nil
+}
+
+func (svc *EnvironmentServiceDelegate) GetEnvironment(query bson.M, opts *mongo.FindOptions) (e interfaces.Environment, err error) {
+	res, err := svc.Get(query, opts)
+	if err != nil {
+		return nil, err
+	}
+	s, ok := res.(interfaces.Environment)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return s, nil
+}
+
+func (svc *EnvironmentServiceDelegate) GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Environment, err error) {
+	list, err := svc.GetList(query, opts)
+	if err != nil {
+		return nil, err
+	}
+	for _, item := range list.GetModels() {
+		s, ok := item.(interfaces.Environment)
+		if !ok {
+			return nil, errors.ErrorModelInvalidType
+		}
+		res = append(res, s)
+	}
+	return res, nil
+}
+
+func NewEnvironmentServiceDelegate() (svc2 interfaces.GrpcClientModelEnvironmentService, err error) {
+	var opts []ModelBaseServiceDelegateOption
+
+	// apply options
+	opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdEnvironment))
+
+	// base service
+	baseSvc, err := NewBaseServiceDelegate(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	// service
+	svc := &EnvironmentServiceDelegate{baseSvc}
+
+	return svc, nil
+}

core/models/client/model_node_delegate.go

@@ -0,0 +1,37 @@
+package client
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"time"
+)
+
+type ModelNodeDelegate struct {
+	n interfaces.Node
+	interfaces.GrpcClientModelDelegate
+}
+
+func (d *ModelNodeDelegate) UpdateStatus(active bool, activeTs *time.Time, status string) (err error) {
+	d.n.SetActive(active)
+	if activeTs != nil {
+		d.n.SetActiveTs(*activeTs)
+	}
+	d.n.SetStatus(status)
+	return d.Save()
+}
+
+func (d *ModelNodeDelegate) UpdateStatusOnline() (err error) {
+	now := time.Now()
+	return d.UpdateStatus(true, &now, constants.NodeStatusOnline)
+}
+
+func (d *ModelNodeDelegate) UpdateStatusOffline() (err error) {
+	return d.UpdateStatus(false, nil, constants.NodeStatusOffline)
+}
+
+func NewModelNodeDelegate(n interfaces.Node) interfaces.ModelNodeDelegate {
+	return &ModelNodeDelegate{
+		n:                       n,
+		GrpcClientModelDelegate: NewModelDelegate(n),
+	}
+}

core/models/client/model_node_service.go

@@ -0,0 +1,74 @@
+package client
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type NodeServiceDelegate struct {
+	interfaces.GrpcClientModelBaseService
+}
+
+func (svc *NodeServiceDelegate) GetNodeById(id primitive.ObjectID) (n interfaces.Node, err error) {
+	res, err := svc.GetById(id)
+	if err != nil {
+		return nil, err
+	}
+	s, ok := res.(interfaces.Node)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return s, nil
+}
+
+func (svc *NodeServiceDelegate) GetNode(query bson.M, opts *mongo.FindOptions) (n interfaces.Node, err error) {
+	res, err := svc.Get(query, opts)
+	if err != nil {
+		return nil, err
+	}
+	s, ok := res.(interfaces.Node)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return s, nil
+}
+
+func (svc *NodeServiceDelegate) GetNodeByKey(key string) (n interfaces.Node, err error) {
+	return svc.GetNode(bson.M{"key": key}, nil)
+}
+
+func (svc *NodeServiceDelegate) GetNodeList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Node, err error) {
+	list, err := svc.GetList(query, opts)
+	if err != nil {
+		return nil, err
+	}
+	for _, item := range list.GetModels() {
+		s, ok := item.(interfaces.Node)
+		if !ok {
+			return nil, errors.ErrorModelInvalidType
+		}
+		res = append(res, s)
+	}
+	return res, nil
+}
+
+func NewNodeServiceDelegate() (svc2 interfaces.GrpcClientModelNodeService, err error) {
+	var opts []ModelBaseServiceDelegateOption
+
+	// apply options
+	opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdNode))
+
+	// base service
+	baseSvc, err := NewBaseServiceDelegate(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	// service
+	svc := &NodeServiceDelegate{baseSvc}
+
+	return svc, nil
+}

core/models/client/model_service.go

@@ -0,0 +1,48 @@
+package client
+
+import (
+	config2 "github.com/crawlab-team/crawlab/core/config"
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+)
+
+type ServiceDelegate struct {
+	// settings
+	cfgPath string
+
+	// internals
+	c interfaces.GrpcClient
+}
+
+func (d *ServiceDelegate) GetConfigPath() string {
+	return d.cfgPath
+}
+
+func (d *ServiceDelegate) SetConfigPath(path string) {
+	d.cfgPath = path
+}
+
+func (d *ServiceDelegate) NewBaseServiceDelegate(id interfaces.ModelId) (svc interfaces.GrpcClientModelBaseService, err error) {
+	var opts []ModelBaseServiceDelegateOption
+	opts = append(opts, WithBaseServiceModelId(id))
+	if d.cfgPath != "" {
+		opts = append(opts, WithBaseServiceConfigPath(d.cfgPath))
+	}
+	return NewBaseServiceDelegate(opts...)
+}
+
+func NewServiceDelegate() (svc2 interfaces.GrpcClientModelService, err error) {
+	// service delegate
+	svc := &ServiceDelegate{
+		cfgPath: config2.GetConfigPath(),
+	}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(client interfaces.GrpcClient) {
+		svc.c = client
+	}); err != nil {
+		return nil, err
+	}
+
+	return svc, nil
+}

core/models/client/model_service_v2.go

@@ -0,0 +1,360 @@
+package client
+
+import (
+	"encoding/json"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/grpc/client"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
+	grpc "github.com/crawlab-team/crawlab/grpc"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"reflect"
+	"sync"
+)
+
+var (
+	instanceMap = make(map[string]interface{})
+	onceMap     = make(map[string]*sync.Once)
+	mu          sync.Mutex
+)
+
+type ModelServiceV2[T any] struct {
+	cfg       interfaces.NodeConfigService
+	c         *client.GrpcClientV2
+	modelType string
+}
+
+func (svc *ModelServiceV2[T]) GetById(id primitive.ObjectID) (model *T, err error) {
+	ctx, cancel := svc.c.Context()
+	defer cancel()
+	res, err := svc.c.ModelBaseServiceV2Client.GetById(ctx, &grpc.ModelServiceV2GetByIdRequest{
+		NodeKey:   svc.cfg.GetNodeKey(),
+		ModelType: svc.modelType,
+		Id:        id.Hex(),
+	})
+	if err != nil {
+		return nil, err
+	}
+	return svc.deserializeOne(res)
+}
+
+func (svc *ModelServiceV2[T]) GetOne(query bson.M, options *mongo.FindOptions) (model *T, err error) {
+	ctx, cancel := svc.c.Context()
+	defer cancel()
+	queryData, err := json.Marshal(query)
+	if err != nil {
+		return nil, err
+	}
+	findOptionsData, err := json.Marshal(options)
+	if err != nil {
+		return nil, err
+	}
+	res, err := svc.c.ModelBaseServiceV2Client.GetOne(ctx, &grpc.ModelServiceV2GetOneRequest{
+		NodeKey:     svc.cfg.GetNodeKey(),
+		ModelType:   svc.modelType,
+		Query:       queryData,
+		FindOptions: findOptionsData,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return svc.deserializeOne(res)
+}
+
+func (svc *ModelServiceV2[T]) GetMany(query bson.M, options *mongo.FindOptions) (models []T, err error) {
+	ctx, cancel := svc.c.Context()
+	defer cancel()
+	queryData, err := json.Marshal(query)
+	if err != nil {
+		return nil, err
+	}
+	findOptionsData, err := json.Marshal(options)
+	if err != nil {
+		return nil, err
+	}
+	res, err := svc.c.ModelBaseServiceV2Client.GetMany(ctx, &grpc.ModelServiceV2GetManyRequest{
+		NodeKey:     svc.cfg.GetNodeKey(),
+		ModelType:   svc.modelType,
+		Query:       queryData,
+		FindOptions: findOptionsData,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return svc.deserializeMany(res)
+}
+
+func (svc *ModelServiceV2[T]) DeleteById(id primitive.ObjectID) (err error) {
+	ctx, cancel := svc.c.Context()
+	defer cancel()
+	_, err = svc.c.ModelBaseServiceV2Client.DeleteById(ctx, &grpc.ModelServiceV2DeleteByIdRequest{
+		NodeKey:   svc.cfg.GetNodeKey(),
+		ModelType: svc.modelType,
+		Id:        id.Hex(),
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (svc *ModelServiceV2[T]) DeleteOne(query bson.M) (err error) {
+	ctx, cancel := svc.c.Context()
+	defer cancel()
+	queryData, err := json.Marshal(query)
+	if err != nil {
+		return err
+	}
+	_, err = svc.c.ModelBaseServiceV2Client.DeleteOne(ctx, &grpc.ModelServiceV2DeleteOneRequest{
+		NodeKey:   svc.cfg.GetNodeKey(),
+		ModelType: svc.modelType,
+		Query:     queryData,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (svc *ModelServiceV2[T]) DeleteMany(query bson.M) (err error) {
+	ctx, cancel := svc.c.Context()
+	defer cancel()
+	queryData, err := json.Marshal(query)
+	if err != nil {
+		return err
+	}
+	_, err = svc.c.ModelBaseServiceV2Client.DeleteMany(ctx, &grpc.ModelServiceV2DeleteManyRequest{
+		NodeKey:   svc.cfg.GetNodeKey(),
+		ModelType: svc.modelType,
+		Query:     queryData,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (svc *ModelServiceV2[T]) UpdateById(id primitive.ObjectID, update bson.M) (err error) {
+	ctx, cancel := svc.c.Context()
+	defer cancel()
+	updateData, err := json.Marshal(update)
+	if err != nil {
+		return err
+	}
+	_, err = svc.c.ModelBaseServiceV2Client.UpdateById(ctx, &grpc.ModelServiceV2UpdateByIdRequest{
+		NodeKey:   svc.cfg.GetNodeKey(),
+		ModelType: svc.modelType,
+		Id:        id.Hex(),
+		Update:    updateData,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (svc *ModelServiceV2[T]) UpdateOne(query bson.M, update bson.M) (err error) {
+	ctx, cancel := svc.c.Context()
+	defer cancel()
+	queryData, err := json.Marshal(query)
+	if err != nil {
+		return err
+	}
+	updateData, err := json.Marshal(update)
+	if err != nil {
+		return err
+	}
+	_, err = svc.c.ModelBaseServiceV2Client.UpdateOne(ctx, &grpc.ModelServiceV2UpdateOneRequest{
+		NodeKey:   svc.cfg.GetNodeKey(),
+		ModelType: svc.modelType,
+		Query:     queryData,
+		Update:    updateData,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (svc *ModelServiceV2[T]) UpdateMany(query bson.M, update bson.M) (err error) {
+	ctx, cancel := svc.c.Context()
+	defer cancel()
+	queryData, err := json.Marshal(query)
+	if err != nil {
+		return err
+	}
+	updateData, err := json.Marshal(update)
+	if err != nil {
+		return err
+	}
+	_, err = svc.c.ModelBaseServiceV2Client.UpdateMany(ctx, &grpc.ModelServiceV2UpdateManyRequest{
+		NodeKey:   svc.cfg.GetNodeKey(),
+		ModelType: svc.modelType,
+		Query:     queryData,
+		Update:    updateData,
+	})
+	return nil
+}
+
+func (svc *ModelServiceV2[T]) ReplaceById(id primitive.ObjectID, model T) (err error) {
+	ctx, cancel := svc.c.Context()
+	defer cancel()
+	modelData, err := json.Marshal(model)
+	if err != nil {
+		return err
+	}
+	_, err = svc.c.ModelBaseServiceV2Client.ReplaceById(ctx, &grpc.ModelServiceV2ReplaceByIdRequest{
+		NodeKey:   svc.cfg.GetNodeKey(),
+		ModelType: svc.modelType,
+		Id:        id.Hex(),
+		Model:     modelData,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (svc *ModelServiceV2[T]) ReplaceOne(query bson.M, model T) (err error) {
+	ctx, cancel := svc.c.Context()
+	defer cancel()
+	queryData, err := json.Marshal(query)
+	if err != nil {
+		return err
+	}
+	modelData, err := json.Marshal(model)
+	if err != nil {
+		return err
+	}
+	_, err = svc.c.ModelBaseServiceV2Client.ReplaceOne(ctx, &grpc.ModelServiceV2ReplaceOneRequest{
+		NodeKey:   svc.cfg.GetNodeKey(),
+		ModelType: svc.modelType,
+		Query:     queryData,
+		Model:     modelData,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (svc *ModelServiceV2[T]) InsertOne(model T) (id primitive.ObjectID, err error) {
+	ctx, cancel := svc.c.Context()
+	defer cancel()
+	modelData, err := json.Marshal(model)
+	if err != nil {
+		return primitive.NilObjectID, err
+	}
+	res, err := svc.c.ModelBaseServiceV2Client.InsertOne(ctx, &grpc.ModelServiceV2InsertOneRequest{
+		NodeKey:   svc.cfg.GetNodeKey(),
+		ModelType: svc.modelType,
+		Model:     modelData,
+	})
+	if err != nil {
+		return primitive.NilObjectID, err
+	}
+	return deserialize[primitive.ObjectID](res)
+	//idStr, err := deserialize[string](res)
+	//if err != nil {
+	//	return primitive.NilObjectID, err
+	//}
+	//return primitive.ObjectIDFromHex(idStr)
+}
+
+func (svc *ModelServiceV2[T]) InsertMany(models []T) (ids []primitive.ObjectID, err error) {
+	ctx, cancel := svc.c.Context()
+	defer cancel()
+	modelsData, err := json.Marshal(models)
+	if err != nil {
+		return nil, err
+	}
+	res, err := svc.c.ModelBaseServiceV2Client.InsertMany(ctx, &grpc.ModelServiceV2InsertManyRequest{
+		NodeKey:   svc.cfg.GetNodeKey(),
+		ModelType: svc.modelType,
+		Models:    modelsData,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return deserialize[[]primitive.ObjectID](res)
+}
+
+func (svc *ModelServiceV2[T]) Count(query bson.M) (total int, err error) {
+	ctx, cancel := svc.c.Context()
+	defer cancel()
+	queryData, err := json.Marshal(query)
+	if err != nil {
+		return 0, err
+	}
+	res, err := svc.c.ModelBaseServiceV2Client.Count(ctx, &grpc.ModelServiceV2CountRequest{
+		NodeKey:   svc.cfg.GetNodeKey(),
+		ModelType: svc.modelType,
+		Query:     queryData,
+	})
+	if err != nil {
+		return 0, err
+	}
+	return deserialize[int](res)
+}
+
+func (svc *ModelServiceV2[T]) GetCol() (col *mongo.Col) {
+	return nil
+}
+
+func (svc *ModelServiceV2[T]) deserializeOne(res *grpc.Response) (result *T, err error) {
+	r, err := deserialize[T](res)
+	if err != nil {
+		return nil, err
+	}
+	return &r, err
+}
+
+func (svc *ModelServiceV2[T]) deserializeMany(res *grpc.Response) (results []T, err error) {
+	return deserialize[[]T](res)
+}
+
+func deserialize[T any](res *grpc.Response) (result T, err error) {
+	err = json.Unmarshal(res.Data, &result)
+	if err != nil {
+		return result, err
+	}
+	return result, nil
+}
+
+func NewModelServiceV2[T any]() *ModelServiceV2[T] {
+	mu.Lock()
+	defer mu.Unlock()
+
+	var v T
+	t := reflect.TypeOf(v)
+	typeName := t.Name()
+
+	if _, exists := onceMap[typeName]; !exists {
+		onceMap[typeName] = &sync.Once{}
+	}
+
+	var instance *ModelServiceV2[T]
+
+	c, err := client.GetGrpcClientV2()
+	if err != nil {
+		panic(err)
+	}
+	if !c.IsStarted() {
+		err = c.Start()
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	onceMap[typeName].Do(func() {
+		instance = &ModelServiceV2[T]{
+			cfg:       nodeconfig.GetNodeConfigService(),
+			c:         c,
+			modelType: typeName,
+		}
+		instanceMap[typeName] = instance
+	})
+
+	return instanceMap[typeName].(*ModelServiceV2[T])
+}

core/models/client/model_service_v2_test.go

@@ -0,0 +1,439 @@
+package client_test
+
+import (
+	"context"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/grpc/server"
+	"github.com/crawlab-team/crawlab/core/models/client"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"testing"
+	"time"
+)
+
+type TestModel models.TestModel
+
+func setupTestDB() {
+	viper.Set("mongo.db", "testdb")
+}
+
+func teardownTestDB() {
+	db := mongo.GetMongoDb("testdb")
+	db.Drop(context.Background())
+}
+
+func TestModelServiceV2_GetById(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+	svr, err := server.NewGrpcServerV2()
+	require.Nil(t, err)
+	go svr.Start()
+	defer svr.Stop()
+
+	m := TestModel{
+		Name: "Test Name",
+	}
+	modelSvc := service.NewModelServiceV2[TestModel]()
+	id, err := modelSvc.InsertOne(m)
+	require.Nil(t, err)
+	m.SetId(id)
+	time.Sleep(100 * time.Millisecond)
+
+	c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.Nil(t, err)
+	c.Connect()
+
+	clientSvc := client.NewModelServiceV2[TestModel]()
+	res, err := clientSvc.GetById(m.Id)
+	require.Nil(t, err)
+	assert.Equal(t, res.Id, m.Id)
+	assert.Equal(t, res.Name, m.Name)
+}
+
+func TestModelServiceV2_GetOne(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+	svr, err := server.NewGrpcServerV2()
+	require.Nil(t, err)
+	go svr.Start()
+	defer svr.Stop()
+
+	m := TestModel{
+		Name: "Test Name",
+	}
+	modelSvc := service.NewModelServiceV2[TestModel]()
+	id, err := modelSvc.InsertOne(m)
+	require.Nil(t, err)
+	m.SetId(id)
+	time.Sleep(100 * time.Millisecond)
+
+	c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.Nil(t, err)
+	c.Connect()
+
+	clientSvc := client.NewModelServiceV2[TestModel]()
+	res, err := clientSvc.GetOne(bson.M{"name": m.Name}, nil)
+	require.Nil(t, err)
+	assert.Equal(t, res.Id, m.Id)
+	assert.Equal(t, res.Name, m.Name)
+}
+
+func TestModelServiceV2_GetMany(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+	svr, err := server.NewGrpcServerV2()
+	require.Nil(t, err)
+	go svr.Start()
+	defer svr.Stop()
+
+	m := TestModel{
+		Name: "Test Name",
+	}
+	modelSvc := service.NewModelServiceV2[TestModel]()
+	id, err := modelSvc.InsertOne(m)
+	require.Nil(t, err)
+	m.SetId(id)
+	time.Sleep(100 * time.Millisecond)
+
+	c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.Nil(t, err)
+	c.Connect()
+
+	clientSvc := client.NewModelServiceV2[TestModel]()
+	res, err := clientSvc.GetMany(bson.M{"name": m.Name}, nil)
+	require.Nil(t, err)
+	assert.Equal(t, len(res), 1)
+	assert.Equal(t, res[0].Id, m.Id)
+	assert.Equal(t, res[0].Name, m.Name)
+}
+
+func TestModelServiceV2_DeleteById(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+	svr, err := server.NewGrpcServerV2()
+	require.Nil(t, err)
+	go svr.Start()
+	defer svr.Stop()
+
+	m := TestModel{
+		Name: "Test Name",
+	}
+	modelSvc := service.NewModelServiceV2[TestModel]()
+	id, err := modelSvc.InsertOne(m)
+	require.Nil(t, err)
+	m.SetId(id)
+	time.Sleep(100 * time.Millisecond)
+
+	c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.Nil(t, err)
+	c.Connect()
+
+	clientSvc := client.NewModelServiceV2[TestModel]()
+	err = clientSvc.DeleteById(m.Id)
+	require.Nil(t, err)
+
+	res, err := clientSvc.GetById(m.Id)
+	assert.NotNil(t, err)
+	require.Nil(t, res)
+}
+
+func TestModelServiceV2_DeleteOne(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+	svr, err := server.NewGrpcServerV2()
+	require.Nil(t, err)
+	go svr.Start()
+	defer svr.Stop()
+
+	m := TestModel{
+		Name: "Test Name",
+	}
+	modelSvc := service.NewModelServiceV2[TestModel]()
+	id, err := modelSvc.InsertOne(m)
+	require.Nil(t, err)
+	m.SetId(id)
+	time.Sleep(100 * time.Millisecond)
+
+	c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.Nil(t, err)
+	c.Connect()
+
+	clientSvc := client.NewModelServiceV2[TestModel]()
+	err = clientSvc.DeleteOne(bson.M{"name": m.Name})
+	require.Nil(t, err)
+
+	res, err := clientSvc.GetOne(bson.M{"name": m.Name}, nil)
+	assert.NotNil(t, err)
+	require.Nil(t, res)
+}
+
+func TestModelServiceV2_DeleteMany(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+	svr, err := server.NewGrpcServerV2()
+	require.Nil(t, err)
+	go svr.Start()
+	defer svr.Stop()
+
+	m := TestModel{
+		Name: "Test Name",
+	}
+	modelSvc := service.NewModelServiceV2[TestModel]()
+	id, err := modelSvc.InsertOne(m)
+	require.Nil(t, err)
+	m.SetId(id)
+	time.Sleep(100 * time.Millisecond)
+
+	c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.Nil(t, err)
+	c.Connect()
+
+	clientSvc := client.NewModelServiceV2[TestModel]()
+	err = clientSvc.DeleteMany(bson.M{"name": m.Name})
+	require.Nil(t, err)
+
+	res, err := clientSvc.GetMany(bson.M{"name": m.Name}, nil)
+	require.Nil(t, err)
+	assert.Equal(t, len(res), 0)
+}
+
+func TestModelServiceV2_UpdateById(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+	svr, err := server.NewGrpcServerV2()
+	require.Nil(t, err)
+	go svr.Start()
+	defer svr.Stop()
+
+	m := TestModel{
+		Name: "Test Name",
+	}
+	modelSvc := service.NewModelServiceV2[TestModel]()
+	id, err := modelSvc.InsertOne(m)
+	require.Nil(t, err)
+	m.SetId(id)
+	time.Sleep(100 * time.Millisecond)
+
+	c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.Nil(t, err)
+	c.Connect()
+
+	clientSvc := client.NewModelServiceV2[TestModel]()
+	err = clientSvc.UpdateById(m.Id, bson.M{"$set": bson.M{"name": "New Name"}})
+	require.Nil(t, err)
+
+	res, err := clientSvc.GetById(m.Id)
+	require.Nil(t, err)
+	assert.Equal(t, res.Name, "New Name")
+}
+
+func TestModelServiceV2_UpdateOne(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+	svr, err := server.NewGrpcServerV2()
+	require.Nil(t, err)
+	go svr.Start()
+	defer svr.Stop()
+
+	m := TestModel{
+		Name: "Test Name",
+	}
+	modelSvc := service.NewModelServiceV2[TestModel]()
+	id, err := modelSvc.InsertOne(m)
+	require.Nil(t, err)
+	m.SetId(id)
+	time.Sleep(100 * time.Millisecond)
+
+	c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.Nil(t, err)
+	c.Connect()
+
+	clientSvc := client.NewModelServiceV2[TestModel]()
+	err = clientSvc.UpdateOne(bson.M{"name": m.Name}, bson.M{"$set": bson.M{"name": "New Name"}})
+	require.Nil(t, err)
+
+	res, err := clientSvc.GetOne(bson.M{"name": "New Name"}, nil)
+	require.Nil(t, err)
+	assert.Equal(t, res.Name, "New Name")
+}
+
+func TestModelServiceV2_UpdateMany(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+	svr, err := server.NewGrpcServerV2()
+	require.Nil(t, err)
+	go svr.Start()
+	defer svr.Stop()
+
+	m1 := TestModel{
+		Name: "Test Name",
+	}
+	m2 := TestModel{
+		Name: "Test Name",
+	}
+	modelSvc := service.NewModelServiceV2[TestModel]()
+	_, err = modelSvc.InsertOne(m1)
+	require.Nil(t, err)
+	_, err = modelSvc.InsertOne(m2)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.Nil(t, err)
+	c.Connect()
+
+	clientSvc := client.NewModelServiceV2[TestModel]()
+	err = clientSvc.UpdateMany(bson.M{"name": "Test Name"}, bson.M{"$set": bson.M{"name": "New Name"}})
+	require.Nil(t, err)
+
+	res, err := clientSvc.GetMany(bson.M{"name": "New Name"}, nil)
+	require.Nil(t, err)
+	assert.Equal(t, len(res), 2)
+}
+
+func TestModelServiceV2_ReplaceById(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+	svr, err := server.NewGrpcServerV2()
+	require.Nil(t, err)
+	go svr.Start()
+	defer svr.Stop()
+
+	m := TestModel{
+		Name: "Test Name",
+	}
+	modelSvc := service.NewModelServiceV2[TestModel]()
+	id, err := modelSvc.InsertOne(m)
+	require.Nil(t, err)
+	m.SetId(id)
+	time.Sleep(100 * time.Millisecond)
+
+	c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.Nil(t, err)
+	c.Connect()
+
+	clientSvc := client.NewModelServiceV2[TestModel]()
+	m.Name = "New Name"
+	err = clientSvc.ReplaceById(m.Id, m)
+	require.Nil(t, err)
+
+	res, err := clientSvc.GetById(m.Id)
+	require.Nil(t, err)
+	assert.Equal(t, res.Name, "New Name")
+}
+
+func TestModelServiceV2_ReplaceOne(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+	svr, err := server.NewGrpcServerV2()
+	require.Nil(t, err)
+	go svr.Start()
+	defer svr.Stop()
+
+	m := TestModel{
+		Name: "Test Name",
+	}
+	modelSvc := service.NewModelServiceV2[TestModel]()
+	id, err := modelSvc.InsertOne(m)
+	require.Nil(t, err)
+	m.SetId(id)
+	time.Sleep(100 * time.Millisecond)
+
+	c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.Nil(t, err)
+	c.Connect()
+
+	clientSvc := client.NewModelServiceV2[TestModel]()
+	m.Name = "New Name"
+	err = clientSvc.ReplaceOne(bson.M{"name": "Test Name"}, m)
+	require.Nil(t, err)
+
+	res, err := clientSvc.GetOne(bson.M{"name": "New Name"}, nil)
+	require.Nil(t, err)
+	assert.Equal(t, res.Name, "New Name")
+}
+
+func TestModelServiceV2_InsertOne(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+	svr, err := server.NewGrpcServerV2()
+	require.Nil(t, err)
+	go svr.Start()
+	defer svr.Stop()
+
+	c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.Nil(t, err)
+	c.Connect()
+
+	clientSvc := client.NewModelServiceV2[TestModel]()
+	m := TestModel{
+		Name: "Test Name",
+	}
+	id, err := clientSvc.InsertOne(m)
+	require.Nil(t, err)
+
+	res, err := clientSvc.GetById(id)
+	require.Nil(t, err)
+	assert.Equal(t, res.Name, m.Name)
+}
+
+func TestModelServiceV2_InsertMany(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+	svr, err := server.NewGrpcServerV2()
+	require.Nil(t, err)
+	go svr.Start()
+	defer svr.Stop()
+
+	c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.Nil(t, err)
+	c.Connect()
+
+	clientSvc := client.NewModelServiceV2[TestModel]()
+	models := []TestModel{
+		{Name: "Test Name 1"},
+		{Name: "Test Name 2"},
+	}
+	ids, err := clientSvc.InsertMany(models)
+	require.Nil(t, err)
+
+	for i, id := range ids {
+		res, err := clientSvc.GetById(id)
+		require.Nil(t, err)
+		assert.Equal(t, res.Name, models[i].Name)
+	}
+}
+
+func TestModelServiceV2_Count(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+	svr, err := server.NewGrpcServerV2()
+	require.Nil(t, err)
+	go svr.Start()
+	defer svr.Stop()
+
+	modelSvc := service.NewModelServiceV2[TestModel]()
+	for i := 0; i < 5; i++ {
+		_, err = modelSvc.InsertOne(TestModel{
+			Name: "Test Name",
+		})
+		require.Nil(t, err)
+	}
+	time.Sleep(100 * time.Millisecond)
+
+	c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.Nil(t, err)
+	c.Connect()
+
+	clientSvc := client.NewModelServiceV2[TestModel]()
+	count, err := clientSvc.Count(bson.M{})
+	require.Nil(t, err)
+
+	assert.Equal(t, count, 5)
+}

core/models/client/model_spider_service.go

@@ -0,0 +1,70 @@
+package client
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type SpiderServiceDelegate struct {
+	interfaces.GrpcClientModelBaseService
+}
+
+func (svc *SpiderServiceDelegate) GetSpiderById(id primitive.ObjectID) (s interfaces.Spider, err error) {
+	res, err := svc.GetById(id)
+	if err != nil {
+		return nil, err
+	}
+	s, ok := res.(interfaces.Spider)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return s, nil
+}
+
+func (svc *SpiderServiceDelegate) GetSpider(query bson.M, opts *mongo.FindOptions) (s interfaces.Spider, err error) {
+	res, err := svc.Get(query, opts)
+	if err != nil {
+		return nil, err
+	}
+	s, ok := res.(interfaces.Spider)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return s, nil
+}
+
+func (svc *SpiderServiceDelegate) GetSpiderList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Spider, err error) {
+	list, err := svc.GetList(query, opts)
+	if err != nil {
+		return nil, err
+	}
+	for _, item := range list.GetModels() {
+		s, ok := item.(interfaces.Spider)
+		if !ok {
+			return nil, errors.ErrorModelInvalidType
+		}
+		res = append(res, s)
+	}
+	return res, nil
+}
+
+func NewSpiderServiceDelegate() (svc2 interfaces.GrpcClientModelSpiderService, err error) {
+	var opts []ModelBaseServiceDelegateOption
+
+	// apply options
+	opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdSpider))
+
+	// base service
+	baseSvc, err := NewBaseServiceDelegate(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	// service
+	svc := &SpiderServiceDelegate{baseSvc}
+
+	return svc, nil
+}

core/models/client/model_task_service.go

@@ -0,0 +1,70 @@
+package client
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type TaskServiceDelegate struct {
+	interfaces.GrpcClientModelBaseService
+}
+
+func (svc *TaskServiceDelegate) GetTaskById(id primitive.ObjectID) (t interfaces.Task, err error) {
+	res, err := svc.GetById(id)
+	if err != nil {
+		return nil, err
+	}
+	s, ok := res.(interfaces.Task)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return s, nil
+}
+
+func (svc *TaskServiceDelegate) GetTask(query bson.M, opts *mongo.FindOptions) (t interfaces.Task, err error) {
+	res, err := svc.Get(query, opts)
+	if err != nil {
+		return nil, err
+	}
+	s, ok := res.(interfaces.Task)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return s, nil
+}
+
+func (svc *TaskServiceDelegate) GetTaskList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Task, err error) {
+	list, err := svc.GetList(query, opts)
+	if err != nil {
+		return nil, err
+	}
+	for _, item := range list.GetModels() {
+		s, ok := item.(interfaces.Task)
+		if !ok {
+			return nil, errors.ErrorModelInvalidType
+		}
+		res = append(res, s)
+	}
+	return res, nil
+}
+
+func NewTaskServiceDelegate() (svc2 interfaces.GrpcClientModelTaskService, err error) {
+	var opts []ModelBaseServiceDelegateOption
+
+	// apply options
+	opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdTask))
+
+	// base service
+	baseSvc, err := NewBaseServiceDelegate(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	// service
+	svc := &TaskServiceDelegate{baseSvc}
+
+	return svc, nil
+}

core/models/client/model_task_stat_service.go

@@ -0,0 +1,70 @@
+package client
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type TaskStatServiceDelegate struct {
+	interfaces.GrpcClientModelBaseService
+}
+
+func (svc *TaskStatServiceDelegate) GetTaskStatById(id primitive.ObjectID) (t interfaces.TaskStat, err error) {
+	res, err := svc.GetById(id)
+	if err != nil {
+		return nil, err
+	}
+	s, ok := res.(interfaces.TaskStat)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return s, nil
+}
+
+func (svc *TaskStatServiceDelegate) GetTaskStat(query bson.M, opts *mongo.FindOptions) (t interfaces.TaskStat, err error) {
+	res, err := svc.Get(query, opts)
+	if err != nil {
+		return nil, err
+	}
+	s, ok := res.(interfaces.TaskStat)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return s, nil
+}
+
+func (svc *TaskStatServiceDelegate) GetTaskStatList(query bson.M, opts *mongo.FindOptions) (res []interfaces.TaskStat, err error) {
+	list, err := svc.GetList(query, opts)
+	if err != nil {
+		return nil, err
+	}
+	for _, item := range list.GetModels() {
+		s, ok := item.(interfaces.TaskStat)
+		if !ok {
+			return nil, errors.ErrorModelInvalidType
+		}
+		res = append(res, s)
+	}
+	return res, nil
+}
+
+func NewTaskStatServiceDelegate() (svc2 interfaces.GrpcClientModelTaskStatService, err error) {
+	var opts []ModelBaseServiceDelegateOption
+
+	// apply options
+	opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdTaskStat))
+
+	// base service
+	baseSvc, err := NewBaseServiceDelegate(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	// service
+	svc := &TaskStatServiceDelegate{baseSvc}
+
+	return svc, nil
+}

core/models/client/options.go

@@ -0,0 +1,33 @@
+package client
+
+import "github.com/crawlab-team/crawlab/core/interfaces"
+
+type ModelDelegateOption func(delegate interfaces.GrpcClientModelDelegate)
+
+func WithDelegateConfigPath(path string) ModelDelegateOption {
+	return func(d interfaces.GrpcClientModelDelegate) {
+		d.SetConfigPath(path)
+	}
+}
+
+type ModelServiceDelegateOption func(delegate interfaces.GrpcClientModelService)
+
+func WithServiceConfigPath(path string) ModelServiceDelegateOption {
+	return func(d interfaces.GrpcClientModelService) {
+		d.SetConfigPath(path)
+	}
+}
+
+type ModelBaseServiceDelegateOption func(delegate interfaces.GrpcClientModelBaseService)
+
+func WithBaseServiceModelId(id interfaces.ModelId) ModelBaseServiceDelegateOption {
+	return func(d interfaces.GrpcClientModelBaseService) {
+		d.SetModelId(id)
+	}
+}
+
+func WithBaseServiceConfigPath(path string) ModelBaseServiceDelegateOption {
+	return func(d interfaces.GrpcClientModelBaseService) {
+		d.SetConfigPath(path)
+	}
+}

core/models/common/index_service.go

@@ -0,0 +1,145 @@
+package common
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func CreateIndexes() {
+	// artifacts
+	mongo.GetMongoCol(interfaces.ModelColNameArtifact).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"_col": 1}},
+		{Keys: bson.M{"_del": 1}},
+		{Keys: bson.M{"_tid": 1}},
+	})
+
+	// tags
+	mongo.GetMongoCol(interfaces.ModelColNameTag).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"col": 1}},
+		{Keys: bson.M{"name": 1}},
+	})
+
+	// nodes
+	mongo.GetMongoCol(interfaces.ModelColNameNode).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"key": 1}},       // key
+		{Keys: bson.M{"name": 1}},      // name
+		{Keys: bson.M{"is_master": 1}}, // is_master
+		{Keys: bson.M{"status": 1}},    // status
+		{Keys: bson.M{"enabled": 1}},   // enabled
+		{Keys: bson.M{"active": 1}},    // active
+	})
+
+	// projects
+	mongo.GetMongoCol(interfaces.ModelColNameNode).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"name": 1}},
+	})
+
+	// spiders
+	mongo.GetMongoCol(interfaces.ModelColNameSpider).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"name": 1}},
+		{Keys: bson.M{"type": 1}},
+		{Keys: bson.M{"col_id": 1}},
+		{Keys: bson.M{"project_id": 1}},
+	})
+
+	// tasks
+	mongo.GetMongoCol(interfaces.ModelColNameTask).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"spider_id": 1}},
+		{Keys: bson.M{"status": 1}},
+		{Keys: bson.M{"node_id": 1}},
+		{Keys: bson.M{"schedule_id": 1}},
+		{Keys: bson.M{"type": 1}},
+		{Keys: bson.M{"mode": 1}},
+		{Keys: bson.M{"priority": 1}},
+		{Keys: bson.M{"parent_id": 1}},
+		{Keys: bson.M{"has_sub": 1}},
+		{Keys: bson.M{"create_ts": -1}},
+	})
+
+	// task stats
+	mongo.GetMongoCol(interfaces.ModelColNameTaskStat).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"create_ts": 1}},
+	})
+
+	// schedules
+	mongo.GetMongoCol(interfaces.ModelColNameSchedule).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"name": 1}},
+		{Keys: bson.M{"spider_id": 1}},
+		{Keys: bson.M{"enabled": 1}},
+	})
+
+	// users
+	mongo.GetMongoCol(interfaces.ModelColNameUser).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"username": 1}},
+		{Keys: bson.M{"role": 1}},
+		{Keys: bson.M{"email": 1}},
+	})
+
+	// settings
+	mongo.GetMongoCol(interfaces.ModelColNameSetting).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"key": 1}},
+	})
+
+	// tokens
+	mongo.GetMongoCol(interfaces.ModelColNameToken).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"name": 1}},
+	})
+
+	// variables
+	mongo.GetMongoCol(interfaces.ModelColNameVariable).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"key": 1}},
+	})
+
+	// data sources
+	mongo.GetMongoCol(interfaces.ModelColNameDataSource).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"name": 1}},
+	})
+
+	// data collections
+	mongo.GetMongoCol(interfaces.ModelColNameDataCollection).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"name": 1}},
+	})
+
+	// extra values
+	mongo.GetMongoCol(interfaces.ModelColNameExtraValues).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"oid": 1}},
+		{Keys: bson.M{"m": 1}},
+		{Keys: bson.M{"t": 1}},
+		{Keys: bson.M{"m": 1, "t": 1}},
+		{Keys: bson.M{"oid": 1, "m": 1, "t": 1}},
+	})
+
+	// roles
+	mongo.GetMongoCol(interfaces.ModelColNameRole).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)},
+	})
+
+	// user role relations
+	mongo.GetMongoCol(interfaces.ModelColNameUserRole).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.D{{"user_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)},
+		{Keys: bson.D{{"role_id", 1}, {"user_id", 1}}, Options: options.Index().SetUnique(true)},
+	})
+
+	// permissions
+	mongo.GetMongoCol(interfaces.ModelColNamePermission).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)},
+	})
+
+	// role permission relations
+	mongo.GetMongoCol(interfaces.ModelColNameRolePermission).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.D{{"role_id", 1}, {"permission_id", 1}}, Options: options.Index().SetUnique(true)},
+		{Keys: bson.D{{"permission_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)},
+	})
+
+	// cache
+	mongo.GetMongoCol(constants.CacheColName).MustCreateIndexes([]mongo2.IndexModel{
+		{
+			Keys:    bson.M{constants.CacheColTime: 1},
+			Options: options.Index().SetExpireAfterSeconds(3600 * 24),
+		},
+	})
+}

core/models/config_spider/common.go

@@ -0,0 +1,24 @@
+package config_spider
+
+import "github.com/crawlab-team/crawlab/core/entity"
+
+func GetAllFields(data entity.ConfigSpiderData) []entity.Field {
+	var fields []entity.Field
+	for _, stage := range data.Stages {
+		fields = append(fields, stage.Fields...)
+	}
+	return fields
+}
+
+func GetStartStageName(data entity.ConfigSpiderData) string {
+	// 如果 start_stage 设置了且在 stages 里，则返回
+	if data.StartStage != "" {
+		return data.StartStage
+	}
+
+	// 否则返回第一个 stage
+	for _, stage := range data.Stages {
+		return stage.Name
+	}
+	return ""
+}

core/models/config_spider/scrapy.go

@@ -0,0 +1,263 @@
+package config_spider
+
+//import (
+//	"errors"
+//	"fmt"
+//	"github.com/crawlab-team/crawlab/core/constants"
+//	"github.com/crawlab-team/crawlab/core/entity"
+//	"github.com/crawlab-team/crawlab/core/models"
+//	"github.com/crawlab-team/crawlab/core/utils"
+//	"path/filepath"
+//)
+//
+//type ScrapyGenerator struct {
+//	Spider     model.Spider
+//	ConfigData entity.ConfigSpiderData
+//}
+//
+//// 生成爬虫文件
+//func (g ScrapyGenerator) Generate() error {
+//	// 生成 items.py
+//	if err := g.ProcessItems(); err != nil {
+//		return err
+//	}
+//
+//	// 生成 spider.py
+//	if err := g.ProcessSpider(); err != nil {
+//		return err
+//	}
+//	return nil
+//}
+//
+//// 生成 items.py
+//func (g ScrapyGenerator) ProcessItems() error {
+//	// 待处理文件名
+//	src := g.Spider.Src
+//	filePath := filepath.Join(src, "config_spider", "items.py")
+//
+//	// 获取所有字段
+//	fields := g.GetAllFields()
+//
+//	// 字段名列表（包含默认字段名）
+//	fieldNames := []string{
+//		"_id",
+//		"task_id",
+//		"ts",
+//	}
+//
+//	// 加入字段
+//	for _, field := range fields {
+//		fieldNames = append(fieldNames, field.Name)
+//	}
+//
+//	// 将字段名转化为python代码
+//	str := ""
+//	for _, fieldName := range fieldNames {
+//		line := g.PadCode(fmt.Sprintf("%s = scrapy.Field()", fieldName), 1)
+//		str += line
+//	}
+//
+//	// 将占位符替换为代码
+//	if err := utils.SetFileVariable(filePath, constants.AnchorItems, str); err != nil {
+//		return err
+//	}
+//
+//	return nil
+//}
+//
+//// 生成 spider.py
+//func (g ScrapyGenerator) ProcessSpider() error {
+//	// 待处理文件名
+//	src := g.Spider.Src
+//	filePath := filepath.Join(src, "config_spider", "spiders", "spider.py")
+//
+//	// 替换 start_stage
+//	if err := utils.SetFileVariable(filePath, constants.AnchorStartStage, "parse_"+GetStartStageName(g.ConfigData)); err != nil {
+//		return err
+//	}
+//
+//	// 替换 start_url
+//	if err := utils.SetFileVariable(filePath, constants.AnchorStartUrl, g.ConfigData.StartUrl); err != nil {
+//		return err
+//	}
+//
+//	// 替换 parsers
+//	strParser := ""
+//	for _, stage := range g.ConfigData.Stages {
+//		stageName := stage.Name
+//		stageStr := g.GetParserString(stageName, stage)
+//		strParser += stageStr
+//	}
+//	if err := utils.SetFileVariable(filePath, constants.AnchorParsers, strParser); err != nil {
+//		return err
+//	}
+//
+//	return nil
+//}
+//
+//func (g ScrapyGenerator) GetParserString(stageName string, stage entity.Stage) string {
+//	// 构造函数定义行
+//	strDef := g.PadCode(fmt.Sprintf("def parse_%s(self, response):", stageName), 1)
+//
+//	strParse := ""
+//	if stage.IsList {
+//		// 列表逻辑
+//		strParse = g.GetListParserString(stageName, stage)
+//	} else {
+//		// 非列表逻辑
+//		strParse = g.GetNonListParserString(stageName, stage)
+//	}
+//
+//	// 构造
+//	str := fmt.Sprintf(`%s%s`, strDef, strParse)
+//
+//	return str
+//}
+//
+//func (g ScrapyGenerator) PadCode(str string, num int) string {
+//	res := ""
+//	for i := 0; i < num; i++ {
+//		res += "    "
+//	}
+//	res += str
+//	res += "\n"
+//	return res
+//}
+//
+//func (g ScrapyGenerator) GetNonListParserString(stageName string, stage entity.Stage) string {
+//	str := ""
+//
+//	// 获取或构造item
+//	str += g.PadCode("item = Item() if response.meta.get('item') is None else response.meta.get('item')", 2)
+//
+//	// 遍历字段列表
+//	for _, f := range stage.Fields {
+//		line := fmt.Sprintf(`item['%s'] = response.%s.extract_first()`, f.Name, g.GetExtractStringFromField(f))
+//		line = g.PadCode(line, 2)
+//		str += line
+//	}
+//
+//	// next stage 字段
+//	if f, err := g.GetNextStageField(stage); err == nil {
+//		// 如果找到 next stage 字段，进行下一个回调
+//		str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url="get_real_url(response, item['%s'])", callback=self.parse_%s, meta={'item': item})`, f.Name, f.NextStage), 2)
+//	} else {
+//		// 如果没找到 next stage 字段，返回 item
+//		str += g.PadCode(fmt.Sprintf(`yield item`), 2)
+//	}
+//
+//	// 加入末尾换行
+//	str += g.PadCode("", 0)
+//
+//	return str
+//}
+//
+//func (g ScrapyGenerator) GetListParserString(stageName string, stage entity.Stage) string {
+//	str := ""
+//
+//	// 获取前一个 stage 的 item
+//	str += g.PadCode(`prev_item = response.meta.get('item')`, 2)
+//
+//	// for 循环遍历列表
+//	str += g.PadCode(fmt.Sprintf(`for elem in response.%s:`, g.GetListString(stage)), 2)
+//
+//	// 构造item
+//	str += g.PadCode(`item = Item()`, 3)
+//
+//	// 遍历字段列表
+//	for _, f := range stage.Fields {
+//		line := fmt.Sprintf(`item['%s'] = elem.%s.extract_first()`, f.Name, g.GetExtractStringFromField(f))
+//		line = g.PadCode(line, 3)
+//		str += line
+//	}
+//
+//	// 把前一个 stage 的 item 值赋给当前 item
+//	str += g.PadCode(`if prev_item is not None:`, 3)
+//	str += g.PadCode(`for key, value in prev_item.items():`, 4)
+//	str += g.PadCode(`item[key] = value`, 5)
+//
+//	// next stage 字段
+//	if f, err := g.GetNextStageField(stage); err == nil {
+//		// 如果 url 为空，则不进入下一个 stage
+//		str += g.PadCode(fmt.Sprintf(`if not item['%s']:`, f.Name), 3)
+//		str += g.PadCode(`continue`, 4)
+//
+//		// 如果找到 next stage 字段，进行下一个回调
+//		str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, item['%s']), callback=self.parse_%s, meta={'item': item})`, f.Name, f.NextStage), 3)
+//	} else {
+//		// 如果没找到 next stage 字段，返回 item
+//		str += g.PadCode(fmt.Sprintf(`yield item`), 3)
+//	}
+//
+//	// 分页
+//	if stage.PageCss != "" || stage.PageXpath != "" {
+//		str += g.PadCode(fmt.Sprintf(`next_url = response.%s.extract_first()`, g.GetExtractStringFromStage(stage)), 2)
+//		str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, next_url), callback=self.parse_%s, meta={'item': prev_item})`, stageName), 2)
+//	}
+//
+//	// 加入末尾换行
+//	str += g.PadCode("", 0)
+//
+//	return str
+//}
+//
+//// 获取所有字段
+//func (g ScrapyGenerator) GetAllFields() []entity.Field {
+//	return GetAllFields(g.ConfigData)
+//}
+//
+//// 获取包含 next stage 的字段
+//func (g ScrapyGenerator) GetNextStageField(stage entity.Stage) (entity.Field, error) {
+//	for _, field := range stage.Fields {
+//		if field.NextStage != "" {
+//			return field, nil
+//		}
+//	}
+//	return entity.Field{}, errors.New("cannot find next stage field")
+//}
+//
+//func (g ScrapyGenerator) GetExtractStringFromField(f entity.Field) string {
+//	if f.Css != "" {
+//		// 如果为CSS
+//		if f.Attr == "" {
+//			// 文本
+//			return fmt.Sprintf(`css('%s::text')`, f.Css)
+//		} else {
+//			// 属性
+//			return fmt.Sprintf(`css('%s::attr("%s")')`, f.Css, f.Attr)
+//		}
+//	} else {
+//		// 如果为XPath
+//		if f.Attr == "" {
+//			// 文本
+//			return fmt.Sprintf(`xpath('string(%s)')`, f.Xpath)
+//		} else {
+//			// 属性
+//			return fmt.Sprintf(`xpath('%s/@%s')`, f.Xpath, f.Attr)
+//		}
+//	}
+//}
+//
+//func (g ScrapyGenerator) GetExtractStringFromStage(stage entity.Stage) string {
+//	// 分页元素属性，默认为 href
+//	pageAttr := "href"
+//	if stage.PageAttr != "" {
+//		pageAttr = stage.PageAttr
+//	}
+//
+//	if stage.PageCss != "" {
+//		// 如果为CSS
+//		return fmt.Sprintf(`css('%s::attr("%s")')`, stage.PageCss, pageAttr)
+//	} else {
+//		// 如果为XPath
+//		return fmt.Sprintf(`xpath('%s/@%s')`, stage.PageXpath, pageAttr)
+//	}
+//}
+//
+//func (g ScrapyGenerator) GetListString(stage entity.Stage) string {
+//	if stage.ListCss != "" {
+//		return fmt.Sprintf(`css('%s')`, stage.ListCss)
+//	} else {
+//		return fmt.Sprintf(`xpath('%s')`, stage.ListXpath)
+//	}
+//}

core/models/delegate/base_test.go

@@ -0,0 +1,25 @@
+package delegate_test
+
+import (
+	"context"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"go.mongodb.org/mongo-driver/bson"
+	"testing"
+	"time"
+)
+
+func SetupTest(t *testing.T) {
+	CleanupTest()
+	t.Cleanup(CleanupTest)
+}
+
+func CleanupTest() {
+	db := mongo.GetMongoDb("")
+	names, _ := db.ListCollectionNames(context.Background(), bson.M{})
+	for _, n := range names {
+		_, _ = db.Collection(n).DeleteMany(context.Background(), bson.M{})
+	}
+
+	// avoid caching
+	time.Sleep(200 * time.Millisecond)
+}

core/models/delegate/model.go

@@ -0,0 +1,362 @@
+package delegate
+
+import (
+	"encoding/json"
+	"github.com/crawlab-team/crawlab-db/errors"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	errors2 "github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/event"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"reflect"
+	"time"
+)
+
+func NewModelDelegate(doc interfaces.Model, args ...interface{}) interfaces.ModelDelegate {
+	switch doc.(type) {
+	case *models.Artifact:
+		return newModelDelegate(interfaces.ModelIdArtifact, doc, args...)
+	case *models.Tag:
+		return newModelDelegate(interfaces.ModelIdTag, doc, args...)
+	case *models.Node:
+		return newModelDelegate(interfaces.ModelIdNode, doc, args...)
+	case *models.Project:
+		return newModelDelegate(interfaces.ModelIdProject, doc, args...)
+	case *models.Spider:
+		return newModelDelegate(interfaces.ModelIdSpider, doc, args...)
+	case *models.Task:
+		return newModelDelegate(interfaces.ModelIdTask, doc, args...)
+	case *models.Job:
+		return newModelDelegate(interfaces.ModelIdJob, doc, args...)
+	case *models.Schedule:
+		return newModelDelegate(interfaces.ModelIdSchedule, doc, args...)
+	case *models.User:
+		return newModelDelegate(interfaces.ModelIdUser, doc, args...)
+	case *models.Setting:
+		return newModelDelegate(interfaces.ModelIdSetting, doc, args...)
+	case *models.Token:
+		return newModelDelegate(interfaces.ModelIdToken, doc, args...)
+	case *models.Variable:
+		return newModelDelegate(interfaces.ModelIdVariable, doc, args...)
+	case *models.TaskQueueItem:
+		return newModelDelegate(interfaces.ModelIdTaskQueue, doc, args...)
+	case *models.TaskStat:
+		return newModelDelegate(interfaces.ModelIdTaskStat, doc, args...)
+	case *models.SpiderStat:
+		return newModelDelegate(interfaces.ModelIdSpiderStat, doc, args...)
+	case *models.DataSource:
+		return newModelDelegate(interfaces.ModelIdDataSource, doc, args...)
+	case *models.DataCollection:
+		return newModelDelegate(interfaces.ModelIdDataCollection, doc, args...)
+	case *models.Result:
+		return newModelDelegate(interfaces.ModelIdResult, doc, args...)
+	case *models.Password:
+		return newModelDelegate(interfaces.ModelIdPassword, doc, args...)
+	case *models.ExtraValue:
+		return newModelDelegate(interfaces.ModelIdExtraValue, doc, args...)
+	case *models.Git:
+		return newModelDelegate(interfaces.ModelIdGit, doc, args...)
+	case *models.Role:
+		return newModelDelegate(interfaces.ModelIdRole, doc, args...)
+	case *models.UserRole:
+		return newModelDelegate(interfaces.ModelIdUserRole, doc, args...)
+	case *models.Permission:
+		return newModelDelegate(interfaces.ModelIdPermission, doc, args...)
+	case *models.RolePermission:
+		return newModelDelegate(interfaces.ModelIdRolePermission, doc, args...)
+	case *models.Environment:
+		return newModelDelegate(interfaces.ModelIdEnvironment, doc, args...)
+	case *models.DependencySetting:
+		return newModelDelegate(interfaces.ModelIdDependencySetting, doc, args...)
+	default:
+		_ = trace.TraceError(errors2.ErrorModelInvalidType)
+		return nil
+	}
+}
+
+func newModelDelegate(id interfaces.ModelId, doc interfaces.Model, args ...interface{}) interfaces.ModelDelegate {
+	// user
+	u := utils.GetUserFromArgs(args...)
+
+	// collection name
+	colName := models.GetModelColName(id)
+
+	// model delegate
+	d := &ModelDelegate{
+		id:      id,
+		colName: colName,
+		doc:     doc,
+		a: &models.Artifact{
+			Col: colName,
+		},
+		u: u,
+	}
+
+	return d
+}
+
+type ModelDelegate struct {
+	id      interfaces.ModelId
+	colName string
+	doc     interfaces.Model         // doc to delegate
+	cd      bson.M                   // current doc
+	od      bson.M                   // original doc
+	a       interfaces.ModelArtifact // artifact
+	u       interfaces.User          // user
+}
+
+// Add model
+func (d *ModelDelegate) Add() (err error) {
+	return d.do(interfaces.ModelDelegateMethodAdd)
+}
+
+// Save model
+func (d *ModelDelegate) Save() (err error) {
+	return d.do(interfaces.ModelDelegateMethodSave)
+}
+
+// Delete model
+func (d *ModelDelegate) Delete() (err error) {
+	return d.do(interfaces.ModelDelegateMethodDelete)
+}
+
+// GetArtifact refresh artifact and return it
+func (d *ModelDelegate) GetArtifact() (res interfaces.ModelArtifact, err error) {
+	if err := d.do(interfaces.ModelDelegateMethodGetArtifact); err != nil {
+		return nil, err
+	}
+	return d.a, nil
+}
+
+// Refresh model
+func (d *ModelDelegate) Refresh() (err error) {
+	return d.refresh()
+}
+
+// GetModel return model
+func (d *ModelDelegate) GetModel() (res interfaces.Model) {
+	return d.doc
+}
+
+func (d *ModelDelegate) ToBytes(m interface{}) (bytes []byte, err error) {
+	if m != nil {
+		return utils.JsonToBytes(m)
+	}
+	return json.Marshal(d.doc)
+}
+
+// do action given the model delegate method
+func (d *ModelDelegate) do(method interfaces.ModelDelegateMethod) (err error) {
+	switch method {
+	case interfaces.ModelDelegateMethodAdd:
+		err = d.add()
+	case interfaces.ModelDelegateMethodSave:
+		err = d.save()
+	case interfaces.ModelDelegateMethodDelete:
+		err = d.delete()
+	case interfaces.ModelDelegateMethodGetArtifact, interfaces.ModelDelegateMethodRefresh:
+		err = d.refresh()
+	default:
+		return trace.TraceError(errors2.ErrorModelInvalidType)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	// trigger event
+	eventName := GetEventName(d, method)
+	go event.SendEvent(eventName, d.doc)
+
+	return nil
+}
+
+// add model
+func (d *ModelDelegate) add() (err error) {
+	if d.doc == nil {
+		return trace.TraceError(errors.ErrMissingValue)
+	}
+	if d.doc.GetId().IsZero() {
+		d.doc.SetId(primitive.NewObjectID())
+	}
+	col := mongo.GetMongoCol(d.colName)
+	if _, err = col.Insert(d.doc); err != nil {
+		return trace.TraceError(err)
+	}
+	if err := d.upsertArtifact(); err != nil {
+		return trace.TraceError(err)
+	}
+	return d.refresh()
+}
+
+// save model
+func (d *ModelDelegate) save() (err error) {
+	// validate
+	if d.doc == nil || d.doc.GetId().IsZero() {
+		return trace.TraceError(errors.ErrMissingValue)
+	}
+
+	// collection
+	col := mongo.GetMongoCol(d.colName)
+
+	// current doc
+	docData, err := bson.Marshal(d.doc)
+	if err != nil {
+		trace.PrintError(err)
+	} else {
+		if err := bson.Unmarshal(docData, &d.cd); err != nil {
+			trace.PrintError(err)
+		}
+	}
+
+	// original doc
+	if err := col.FindId(d.doc.GetId()).One(&d.od); err != nil {
+		trace.PrintError(err)
+	}
+
+	// replace
+	if err := col.ReplaceId(d.doc.GetId(), d.doc); err != nil {
+		return trace.TraceError(err)
+	}
+
+	// upsert artifact
+	if err := d.upsertArtifact(); err != nil {
+		return trace.TraceError(err)
+	}
+
+	return d.refresh()
+}
+
+// delete model
+func (d *ModelDelegate) delete() (err error) {
+	if d.doc.GetId().IsZero() {
+		return trace.TraceError(errors2.ErrorModelMissingId)
+	}
+	col := mongo.GetMongoCol(d.colName)
+	if err := col.FindId(d.doc.GetId()).One(d.doc); err != nil {
+		return trace.TraceError(err)
+	}
+	if err := col.DeleteId(d.doc.GetId()); err != nil {
+		return trace.TraceError(err)
+	}
+	return d.deleteArtifact()
+}
+
+// refresh model and artifact
+func (d *ModelDelegate) refresh() (err error) {
+	if d.doc.GetId().IsZero() {
+		return trace.TraceError(errors2.ErrorModelMissingId)
+	}
+	col := mongo.GetMongoCol(d.colName)
+	fr := col.FindId(d.doc.GetId())
+	if err := fr.One(d.doc); err != nil {
+		return trace.TraceError(err)
+	}
+	return d.refreshArtifact()
+}
+
+// refresh artifact
+func (d *ModelDelegate) refreshArtifact() (err error) {
+	if d.doc.GetId().IsZero() {
+		return trace.TraceError(errors2.ErrorModelMissingId)
+	}
+	col := mongo.GetMongoCol(interfaces.ModelColNameArtifact)
+	if err := col.FindId(d.doc.GetId()).One(d.a); err != nil {
+		return trace.TraceError(err)
+	}
+	return nil
+}
+
+// upsertArtifact
+func (d *ModelDelegate) upsertArtifact() (err error) {
+	// skip
+	if d._skip() {
+		return nil
+	}
+
+	// validate id
+	if d.doc.GetId().IsZero() {
+		return trace.TraceError(errors.ErrMissingValue)
+	}
+
+	// mongo collection
+	col := mongo.GetMongoCol(interfaces.ModelColNameArtifact)
+
+	// assign id to artifact
+	d.a.SetId(d.doc.GetId())
+
+	// attempt to find artifact
+	if err := col.FindId(d.doc.GetId()).One(d.a); err != nil {
+		if err == mongo2.ErrNoDocuments {
+			// new artifact
+			d.a.GetSys().SetCreateTs(time.Now())
+			d.a.GetSys().SetUpdateTs(time.Now())
+			if d.u != nil && !reflect.ValueOf(d.u).IsZero() {
+				d.a.GetSys().SetCreateUid(d.u.GetId())
+				d.a.GetSys().SetUpdateUid(d.u.GetId())
+			}
+			_, err = col.Insert(d.a)
+			if err != nil {
+				return trace.TraceError(err)
+			}
+			return nil
+		} else {
+			// error
+			return trace.TraceError(err)
+		}
+	}
+
+	// existing artifact
+	d.a.GetSys().SetUpdateTs(time.Now())
+	if d.u != nil {
+		d.a.GetSys().SetUpdateUid(d.u.GetId())
+	}
+
+	// save new artifact
+	return col.ReplaceId(d.a.GetId(), d.a)
+}
+
+// deleteArtifact
+func (d *ModelDelegate) deleteArtifact() (err error) {
+	// skip
+	if d._skip() {
+		return nil
+	}
+
+	if d.doc.GetId().IsZero() {
+		return trace.TraceError(errors.ErrMissingValue)
+	}
+	col := mongo.GetMongoCol(interfaces.ModelColNameArtifact)
+	d.a.SetId(d.doc.GetId())
+	d.a.SetObj(d.doc)
+	d.a.SetDel(true)
+	d.a.GetSys().SetDeleteTs(time.Now())
+	if d.u != nil {
+		d.a.GetSys().SetDeleteUid(d.u.GetId())
+	}
+	return col.ReplaceId(d.doc.GetId(), d.a)
+}
+
+func (d *ModelDelegate) hasChange() (ok bool) {
+	return !utils.BsonMEqual(d.cd, d.od)
+}
+
+func (d *ModelDelegate) _skip() (ok bool) {
+	switch d.id {
+	case
+		interfaces.ModelIdArtifact,
+		interfaces.ModelIdTaskQueue,
+		interfaces.ModelIdTaskStat,
+		interfaces.ModelIdSpiderStat,
+		interfaces.ModelIdResult,
+		interfaces.ModelIdPassword:
+		return true
+	default:
+		return false
+	}
+}

core/models/delegate/model_node.go

@@ -0,0 +1,37 @@
+package delegate
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"time"
+)
+
+type ModelNodeDelegate struct {
+	n interfaces.Node
+	interfaces.ModelDelegate
+}
+
+func (d *ModelNodeDelegate) UpdateStatus(active bool, activeTs *time.Time, status string) (err error) {
+	d.n.SetActive(active)
+	if activeTs != nil {
+		d.n.SetActiveTs(*activeTs)
+	}
+	d.n.SetStatus(status)
+	return d.Save()
+}
+
+func (d *ModelNodeDelegate) UpdateStatusOnline() (err error) {
+	now := time.Now()
+	return d.UpdateStatus(true, &now, constants.NodeStatusOnline)
+}
+
+func (d *ModelNodeDelegate) UpdateStatusOffline() (err error) {
+	return d.UpdateStatus(false, nil, constants.NodeStatusOffline)
+}
+
+func NewModelNodeDelegate(n interfaces.Node) interfaces.ModelNodeDelegate {
+	return &ModelNodeDelegate{
+		n:             n,
+		ModelDelegate: NewModelDelegate(n),
+	}
+}

core/models/delegate/model_node_test.go

@@ -0,0 +1,65 @@
+package delegate_test
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestNode_Add(t *testing.T) {
+	SetupTest(t)
+
+	n := &models2.Node{}
+
+	err := delegate.NewModelDelegate(n).Add()
+	require.Nil(t, err)
+	require.NotNil(t, n.Id)
+
+	// validate artifact
+	a, err := delegate.NewModelDelegate(n).GetArtifact()
+	require.Nil(t, err)
+	require.Equal(t, n.Id, a.GetId())
+	require.NotNil(t, a.GetSys().GetCreateTs())
+	require.NotNil(t, a.GetSys().GetUpdateTs())
+}
+
+func TestNode_Save(t *testing.T) {
+	SetupTest(t)
+
+	n := &models2.Node{}
+
+	err := delegate.NewModelDelegate(n).Add()
+
+	name := "test_node"
+	n.Name = name
+	err = delegate.NewModelDelegate(n).Save()
+	require.Nil(t, err)
+
+	err = mongo.GetMongoCol(interfaces.ModelColNameNode).FindId(n.Id).One(&n)
+	require.Nil(t, err)
+	require.Equal(t, name, n.Name)
+}
+
+func TestNode_Delete(t *testing.T) {
+	SetupTest(t)
+
+	n := &models2.Node{
+		Name: "test_node",
+	}
+
+	err := delegate.NewModelDelegate(n).Add()
+	require.Nil(t, err)
+
+	err = delegate.NewModelDelegate(n).Delete()
+	require.Nil(t, err)
+
+	var a models2.Artifact
+	col := mongo.GetMongoCol(interfaces.ModelColNameArtifact)
+	err = col.FindId(n.Id).One(&a)
+	require.Nil(t, err)
+	require.NotNil(t, a.Obj)
+	require.True(t, a.Del)
+}

core/models/delegate/model_role_test.go

@@ -0,0 +1,65 @@
+package delegate_test
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestRole_Add(t *testing.T) {
+	SetupTest(t)
+
+	p := &models2.Role{}
+
+	err := delegate.NewModelDelegate(p).Add()
+	require.Nil(t, err)
+	require.NotNil(t, p.Id)
+
+	a, err := delegate.NewModelDelegate(p).GetArtifact()
+	require.Nil(t, err)
+	require.Equal(t, p.Id, a.GetId())
+	require.NotNil(t, a.GetSys().GetCreateTs())
+	require.NotNil(t, a.GetSys().GetUpdateTs())
+}
+
+func TestRole_Save(t *testing.T) {
+	SetupTest(t)
+
+	p := &models2.Role{}
+
+	err := delegate.NewModelDelegate(p).Add()
+	require.Nil(t, err)
+
+	name := "test_role"
+	p.Name = name
+	err = delegate.NewModelDelegate(p).Save()
+	require.Nil(t, err)
+
+	err = mongo.GetMongoCol(interfaces.ModelColNameRole).FindId(p.Id).One(&p)
+	require.Nil(t, err)
+	require.Equal(t, name, p.Name)
+}
+
+func TestRole_Delete(t *testing.T) {
+	SetupTest(t)
+
+	p := &models2.Role{
+		Name: "test_role",
+	}
+
+	err := delegate.NewModelDelegate(p).Add()
+	require.Nil(t, err)
+
+	err = delegate.NewModelDelegate(p).Delete()
+	require.Nil(t, err)
+
+	var a models2.Artifact
+	col := mongo.GetMongoCol(interfaces.ModelColNameArtifact)
+	err = col.FindId(p.Id).One(&a)
+	require.Nil(t, err)
+	require.NotNil(t, a.Obj)
+	require.True(t, a.Del)
+}

core/models/delegate/model_test.go

@@ -0,0 +1,65 @@
+package delegate_test
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestProject_Add(t *testing.T) {
+	SetupTest(t)
+
+	p := &models2.Project{}
+
+	err := delegate.NewModelDelegate(p).Add()
+	require.Nil(t, err)
+	require.NotNil(t, p.Id)
+
+	a, err := delegate.NewModelDelegate(p).GetArtifact()
+	require.Nil(t, err)
+	require.Equal(t, p.Id, a.GetId())
+	require.NotNil(t, a.GetSys().GetCreateTs())
+	require.NotNil(t, a.GetSys().GetUpdateTs())
+}
+
+func TestProject_Save(t *testing.T) {
+	SetupTest(t)
+
+	p := &models2.Project{}
+
+	err := delegate.NewModelDelegate(p).Add()
+	require.Nil(t, err)
+
+	name := "test_project"
+	p.Name = name
+	err = delegate.NewModelDelegate(p).Save()
+	require.Nil(t, err)
+
+	err = mongo.GetMongoCol(interfaces.ModelColNameProject).FindId(p.Id).One(&p)
+	require.Nil(t, err)
+	require.Equal(t, name, p.Name)
+}
+
+func TestProject_Delete(t *testing.T) {
+	SetupTest(t)
+
+	p := &models2.Project{
+		Name: "test_project",
+	}
+
+	err := delegate.NewModelDelegate(p).Add()
+	require.Nil(t, err)
+
+	err = delegate.NewModelDelegate(p).Delete()
+	require.Nil(t, err)
+
+	var a models2.Artifact
+	col := mongo.GetMongoCol(interfaces.ModelColNameArtifact)
+	err = col.FindId(p.Id).One(&a)
+	require.Nil(t, err)
+	require.NotNil(t, a.Obj)
+	require.True(t, a.Del)
+}

core/models/delegate/model_user_role_test.go

@@ -0,0 +1,97 @@
+package delegate_test
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/common"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"testing"
+)
+
+func init() {
+	viper.Set("mongo.db", "crawlab_test")
+	common.CreateIndexes()
+}
+
+func TestUserRole_Add(t *testing.T) {
+	SetupTest(t)
+
+	p := &models2.UserRole{}
+
+	err := delegate.NewModelDelegate(p).Add()
+	require.Nil(t, err)
+	require.NotNil(t, p.Id)
+
+	a, err := delegate.NewModelDelegate(p).GetArtifact()
+	require.Nil(t, err)
+	require.Equal(t, p.Id, a.GetId())
+	require.NotNil(t, a.GetSys().GetCreateTs())
+	require.NotNil(t, a.GetSys().GetUpdateTs())
+}
+
+func TestUserRole_Save(t *testing.T) {
+	SetupTest(t)
+
+	p := &models2.UserRole{
+		UserId: primitive.NewObjectID(),
+		RoleId: primitive.NewObjectID(),
+	}
+
+	err := delegate.NewModelDelegate(p).Add()
+	require.Nil(t, err)
+
+	uid := primitive.NewObjectID()
+	rid := primitive.NewObjectID()
+	p.UserId = uid
+	p.RoleId = rid
+	err = delegate.NewModelDelegate(p).Save()
+	require.Nil(t, err)
+
+	err = mongo.GetMongoCol(interfaces.ModelColNameUserRole).FindId(p.Id).One(&p)
+	require.Nil(t, err)
+	require.Equal(t, uid, p.UserId)
+	require.Equal(t, rid, p.RoleId)
+}
+
+func TestUserRole_Delete(t *testing.T) {
+	SetupTest(t)
+
+	p := &models2.UserRole{}
+
+	err := delegate.NewModelDelegate(p).Add()
+	require.Nil(t, err)
+
+	err = delegate.NewModelDelegate(p).Delete()
+	require.Nil(t, err)
+
+	var a models2.Artifact
+	col := mongo.GetMongoCol(interfaces.ModelColNameArtifact)
+	err = col.FindId(p.Id).One(&a)
+	require.Nil(t, err)
+	require.NotNil(t, a.Obj)
+	require.True(t, a.Del)
+}
+
+func TestUserRole_AddDuplicates(t *testing.T) {
+	SetupTest(t)
+
+	uid := primitive.NewObjectID()
+	rid := primitive.NewObjectID()
+	p := &models2.UserRole{
+		UserId: uid,
+		RoleId: rid,
+	}
+	p2 := &models2.UserRole{
+		UserId: uid,
+		RoleId: rid,
+	}
+
+	err := delegate.NewModelDelegate(p).Add()
+	require.Nil(t, err)
+	err = delegate.NewModelDelegate(p2).Add()
+	require.NotNil(t, err)
+}

core/models/delegate/utils_event.go

@@ -0,0 +1,20 @@
+package delegate
+
+import (
+	"fmt"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+)
+
+func GetEventName(d *ModelDelegate, method interfaces.ModelDelegateMethod) (eventName string) {
+	return getEventName(d, method)
+}
+
+func getEventName(d *ModelDelegate, method interfaces.ModelDelegateMethod) (eventName string) {
+	if method == interfaces.ModelDelegateMethodSave {
+		hasChange := d.hasChange()
+		if hasChange {
+			method = interfaces.ModelDelegateMethodChange
+		}
+	}
+	return fmt.Sprintf("model:%s:%s", d.colName, method)
+}

core/models/models/artifact.go

@@ -0,0 +1,56 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Artifact struct {
+	Id     primitive.ObjectID   `bson:"_id" json:"_id"`
+	Col    string               `bson:"_col" json:"_col"`
+	Del    bool                 `bson:"_del" json:"_del"`
+	TagIds []primitive.ObjectID `bson:"_tid" json:"_tid"`
+	Sys    *ArtifactSys         `bson:"_sys" json:"_sys"`
+	Obj    interface{}          `bson:"_obj" json:"_obj"`
+}
+
+func (a *Artifact) GetId() (id primitive.ObjectID) {
+	return a.Id
+}
+
+func (a *Artifact) SetId(id primitive.ObjectID) {
+	a.Id = id
+}
+
+func (a *Artifact) GetSys() (sys interfaces.ModelArtifactSys) {
+	if a.Sys == nil {
+		a.Sys = &ArtifactSys{}
+	}
+	return a.Sys
+}
+
+func (a *Artifact) GetTagIds() (ids []primitive.ObjectID) {
+	return a.TagIds
+}
+
+func (a *Artifact) SetTagIds(ids []primitive.ObjectID) {
+	a.TagIds = ids
+}
+
+func (a *Artifact) SetObj(obj interfaces.Model) {
+	a.Obj = obj
+}
+
+func (a *Artifact) SetDel(del bool) {
+	a.Del = del
+}
+
+type ArtifactList []Artifact
+
+func (l *ArtifactList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/artifact_sys.go

@@ -0,0 +1,63 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type ArtifactSys struct {
+	CreateTs  time.Time          `json:"create_ts" bson:"create_ts"`
+	CreateUid primitive.ObjectID `json:"create_uid" bson:"create_uid"`
+	UpdateTs  time.Time          `json:"update_ts" bson:"update_ts"`
+	UpdateUid primitive.ObjectID `json:"update_uid" bson:"update_uid"`
+	DeleteTs  time.Time          `json:"delete_ts" bson:"delete_ts"`
+	DeleteUid primitive.ObjectID `json:"delete_uid" bson:"delete_uid"`
+}
+
+func (sys *ArtifactSys) GetCreateTs() time.Time {
+	return sys.CreateTs
+}
+
+func (sys *ArtifactSys) SetCreateTs(ts time.Time) {
+	sys.CreateTs = ts
+}
+
+func (sys *ArtifactSys) GetUpdateTs() time.Time {
+	return sys.UpdateTs
+}
+
+func (sys *ArtifactSys) SetUpdateTs(ts time.Time) {
+	sys.UpdateTs = ts
+}
+
+func (sys *ArtifactSys) GetDeleteTs() time.Time {
+	return sys.DeleteTs
+}
+
+func (sys *ArtifactSys) SetDeleteTs(ts time.Time) {
+	sys.DeleteTs = ts
+}
+
+func (sys *ArtifactSys) GetCreateUid() primitive.ObjectID {
+	return sys.CreateUid
+}
+
+func (sys *ArtifactSys) SetCreateUid(id primitive.ObjectID) {
+	sys.CreateUid = id
+}
+
+func (sys *ArtifactSys) GetUpdateUid() primitive.ObjectID {
+	return sys.UpdateUid
+}
+
+func (sys *ArtifactSys) SetUpdateUid(id primitive.ObjectID) {
+	sys.UpdateUid = id
+}
+
+func (sys *ArtifactSys) GetDeleteUid() primitive.ObjectID {
+	return sys.DeleteUid
+}
+
+func (sys *ArtifactSys) SetDeleteUid(id primitive.ObjectID) {
+	sys.DeleteUid = id
+}

core/models/models/base.go

@@ -0,0 +1,13 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type BaseModel struct {
+	Id primitive.ObjectID `json:"_id" bson:"_id"`
+}
+
+func (d *BaseModel) GetId() (id primitive.ObjectID) {
+	return d.Id
+}

core/models/models/base_v2.go

@@ -0,0 +1,64 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type BaseModelV2[T any] struct {
+	Id        primitive.ObjectID `json:"_id" bson:"_id"`
+	CreatedAt time.Time          `json:"created_ts" bson:"created_ts"`
+	CreatedBy primitive.ObjectID `json:"created_by" bson:"created_by"`
+	UpdatedAt time.Time          `json:"updated_ts" bson:"updated_ts"`
+	UpdatedBy primitive.ObjectID `json:"updated_by" bson:"updated_by"`
+}
+
+func (m *BaseModelV2[T]) GetId() primitive.ObjectID {
+	return m.Id
+}
+
+func (m *BaseModelV2[T]) SetId(id primitive.ObjectID) {
+	m.Id = id
+}
+
+func (m *BaseModelV2[T]) GetCreatedAt() time.Time {
+	return m.CreatedAt
+}
+
+func (m *BaseModelV2[T]) SetCreatedAt(t time.Time) {
+	m.CreatedAt = t
+}
+
+func (m *BaseModelV2[T]) GetCreatedBy() primitive.ObjectID {
+	return m.CreatedBy
+}
+
+func (m *BaseModelV2[T]) SetCreatedBy(id primitive.ObjectID) {
+	m.CreatedBy = id
+}
+
+func (m *BaseModelV2[T]) GetUpdatedAt() time.Time {
+	return m.UpdatedAt
+}
+
+func (m *BaseModelV2[T]) SetUpdatedAt(t time.Time) {
+	m.UpdatedAt = t
+}
+
+func (m *BaseModelV2[T]) GetUpdatedBy() primitive.ObjectID {
+	return m.UpdatedBy
+}
+
+func (m *BaseModelV2[T]) SetUpdatedBy(id primitive.ObjectID) {
+	m.UpdatedBy = id
+}
+
+func (m *BaseModelV2[T]) SetCreated(id primitive.ObjectID) {
+	m.SetCreatedAt(time.Now())
+	m.SetCreatedBy(id)
+}
+
+func (m *BaseModelV2[T]) SetUpdated(id primitive.ObjectID) {
+	m.SetUpdatedAt(time.Now())
+	m.SetUpdatedBy(id)
+}

core/models/models/data_collection.go

@@ -0,0 +1,36 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type DataCollection struct {
+	Id     primitive.ObjectID `json:"_id" bson:"_id"`
+	Name   string             `json:"name" bson:"name"`
+	Fields []entity.DataField `json:"fields" bson:"fields"`
+	Dedup  struct {
+		Enabled bool     `json:"enabled" bson:"enabled"`
+		Keys    []string `json:"keys" bson:"keys"`
+		Type    string   `json:"type" bson:"type"`
+	} `json:"dedup" bson:"dedup"`
+}
+
+func (dc *DataCollection) GetId() (id primitive.ObjectID) {
+	return dc.Id
+}
+
+func (dc *DataCollection) SetId(id primitive.ObjectID) {
+	dc.Id = id
+}
+
+type DataCollectionList []DataCollection
+
+func (l *DataCollectionList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/data_collection_v2.go

@@ -0,0 +1,17 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/entity"
+)
+
+type DataCollectionV2 struct {
+	any                         `collection:"data_collections"`
+	BaseModelV2[DataCollection] `bson:",inline"`
+	Name                        string             `json:"name" bson:"name"`
+	Fields                      []entity.DataField `json:"fields" bson:"fields"`
+	Dedup                       struct {
+		Enabled bool     `json:"enabled" bson:"enabled"`
+		Keys    []string `json:"keys" bson:"keys"`
+		Type    string   `json:"type" bson:"type"`
+	} `json:"dedup" bson:"dedup"`
+}

core/models/models/data_source.go

@@ -0,0 +1,42 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type DataSource struct {
+	Id          primitive.ObjectID `json:"_id" bson:"_id"`
+	Name        string             `json:"name" bson:"name"`
+	Type        string             `json:"type" bson:"type"`
+	Description string             `json:"description" bson:"description"`
+	Host        string             `json:"host" bson:"host"`
+	Port        string             `json:"port" bson:"port"`
+	Url         string             `json:"url" bson:"url"`
+	Hosts       []string           `json:"hosts" bson:"hosts"`
+	Database    string             `json:"database" bson:"database"`
+	Username    string             `json:"username" bson:"username"`
+	Password    string             `json:"password,omitempty" bson:"-"`
+	ConnectType string             `json:"connect_type" bson:"connect_type"`
+	Status      string             `json:"status" bson:"status"`
+	Error       string             `json:"error" bson:"error"`
+	Extra       map[string]string  `json:"extra,omitempty" bson:"extra,omitempty"`
+}
+
+func (ds *DataSource) GetId() (id primitive.ObjectID) {
+	return ds.Id
+}
+
+func (ds *DataSource) SetId(id primitive.ObjectID) {
+	ds.Id = id
+}
+
+type DataSourceList []DataSource
+
+func (l *DataSourceList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/data_source_v2.go

@@ -0,0 +1,20 @@
+package models
+
+type DataSourceV2 struct {
+	any                     `collection:"data_sources"`
+	BaseModelV2[DataSource] `bson:",inline"`
+	Name                    string            `json:"name" bson:"name"`
+	Type                    string            `json:"type" bson:"type"`
+	Description             string            `json:"description" bson:"description"`
+	Host                    string            `json:"host" bson:"host"`
+	Port                    string            `json:"port" bson:"port"`
+	Url                     string            `json:"url" bson:"url"`
+	Hosts                   []string          `json:"hosts" bson:"hosts"`
+	Database                string            `json:"database" bson:"database"`
+	Username                string            `json:"username" bson:"username"`
+	Password                string            `json:"password,omitempty" bson:"-"`
+	ConnectType             string            `json:"connect_type" bson:"connect_type"`
+	Status                  string            `json:"status" bson:"status"`
+	Error                   string            `json:"error" bson:"error"`
+	Extra                   map[string]string `json:"extra,omitempty" bson:"extra,omitempty"`
+}

core/models/models/dependency_setting.go

@@ -0,0 +1,36 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type DependencySetting struct {
+	Id           primitive.ObjectID `json:"_id" bson:"_id"`
+	Key          string             `json:"key" bson:"key"`
+	Name         string             `json:"name" bson:"name"`
+	Description  string             `json:"description" bson:"description"`
+	Enabled      bool               `json:"enabled" bson:"enabled"`
+	Cmd          string             `json:"cmd" bson:"cmd"`
+	Proxy        string             `json:"proxy" bson:"proxy"`
+	LastUpdateTs time.Time          `json:"last_update_ts" bson:"last_update_ts"`
+}
+
+func (j *DependencySetting) GetId() (id primitive.ObjectID) {
+	return j.Id
+}
+
+func (j *DependencySetting) SetId(id primitive.ObjectID) {
+	j.Id = id
+}
+
+type DependencySettingList []DependencySetting
+
+func (l *DependencySettingList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/dependency_setting_v2.go

@@ -0,0 +1,17 @@
+package models
+
+import (
+	"time"
+)
+
+type DependencySettingV2 struct {
+	any                            `collection:"dependency_settings"`
+	BaseModelV2[DependencySetting] `bson:",inline"`
+	Key                            string    `json:"key" bson:"key"`
+	Name                           string    `json:"name" bson:"name"`
+	Description                    string    `json:"description" bson:"description"`
+	Enabled                        bool      `json:"enabled" bson:"enabled"`
+	Cmd                            string    `json:"cmd" bson:"cmd"`
+	Proxy                          string    `json:"proxy" bson:"proxy"`
+	LastUpdateTs                   time.Time `json:"last_update_ts" bson:"last_update_ts"`
+}

core/models/models/environment.go

@@ -0,0 +1,46 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Environment struct {
+	Id    primitive.ObjectID `json:"_id" bson:"_id"`
+	Key   string             `json:"key" bson:"key"`
+	Value string             `json:"value" bson:"value"`
+}
+
+func (e *Environment) GetId() (id primitive.ObjectID) {
+	return e.Id
+}
+
+func (e *Environment) SetId(id primitive.ObjectID) {
+	e.Id = id
+}
+
+func (e *Environment) GetKey() (key string) {
+	return e.Key
+}
+
+func (e *Environment) SetKey(key string) {
+	e.Key = key
+}
+
+func (e *Environment) GetValue() (value string) {
+	return e.Value
+}
+
+func (e *Environment) SetValue(value string) {
+	e.Value = value
+}
+
+type EnvironmentList []Environment
+
+func (l *EnvironmentList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/environment_v2.go

@@ -0,0 +1,8 @@
+package models
+
+type EnvironmentV2 struct {
+	any                        `collection:"environments"`
+	BaseModelV2[EnvironmentV2] `bson:",inline"`
+	Key                        string `json:"key" bson:"key"`
+	Value                      string `json:"value" bson:"value"`
+}

core/models/models/extra_value.go

@@ -0,0 +1,64 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type ExtraValue struct {
+	Id       primitive.ObjectID `json:"_id" bson:"_id"`
+	ObjectId primitive.ObjectID `json:"oid" bson:"oid"`
+	Model    string             `json:"model" bson:"m"`
+	Type     string             `json:"type" bson:"t"`
+	Value    interface{}        `json:"value" bson:"v"`
+}
+
+func (ev *ExtraValue) GetId() (id primitive.ObjectID) {
+	return ev.Id
+}
+
+func (ev *ExtraValue) SetId(id primitive.ObjectID) {
+	ev.Id = id
+}
+
+func (ev *ExtraValue) GetValue() (v interface{}) {
+	return ev.Value
+}
+
+func (ev *ExtraValue) SetValue(v interface{}) {
+	ev.Value = v
+}
+
+func (ev *ExtraValue) GetObjectId() (oid primitive.ObjectID) {
+	return ev.ObjectId
+}
+
+func (ev *ExtraValue) SetObjectId(oid primitive.ObjectID) {
+	ev.ObjectId = oid
+}
+
+func (ev *ExtraValue) GetModel() (m string) {
+	return ev.Model
+}
+
+func (ev *ExtraValue) SetModel(m string) {
+	ev.Model = m
+}
+
+func (ev *ExtraValue) GetType() (t string) {
+	return ev.Type
+}
+
+func (ev *ExtraValue) SetType(t string) {
+	ev.Type = t
+}
+
+type ExtraValueList []ExtraValue
+
+func (l *ExtraValueList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/git.go

@@ -0,0 +1,82 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Git struct {
+	Id            primitive.ObjectID `json:"_id" bson:"_id"`
+	Url           string             `json:"url" bson:"url"`
+	AuthType      string             `json:"auth_type" bson:"auth_type"`
+	Username      string             `json:"username" bson:"username"`
+	Password      string             `json:"password" bson:"password"`
+	CurrentBranch string             `json:"current_branch" bson:"current_branch"`
+	AutoPull      bool               `json:"auto_pull" bson:"auto_pull"`
+}
+
+func (g *Git) GetId() (id primitive.ObjectID) {
+	return g.Id
+}
+
+func (g *Git) SetId(id primitive.ObjectID) {
+	g.Id = id
+}
+
+func (g *Git) GetUrl() (url string) {
+	return g.Url
+}
+
+func (g *Git) SetUrl(url string) {
+	g.Url = url
+}
+
+func (g *Git) GetAuthType() (authType string) {
+	return g.AuthType
+}
+
+func (g *Git) SetAuthType(authType string) {
+	g.AuthType = authType
+}
+
+func (g *Git) GetUsername() (username string) {
+	return g.Username
+}
+
+func (g *Git) SetUsername(username string) {
+	g.Username = username
+}
+
+func (g *Git) GetPassword() (password string) {
+	return g.Password
+}
+
+func (g *Git) SetPassword(password string) {
+	g.Password = password
+}
+
+func (g *Git) GetCurrentBranch() (currentBranch string) {
+	return g.CurrentBranch
+}
+
+func (g *Git) SetCurrentBranch(currentBranch string) {
+	g.CurrentBranch = currentBranch
+}
+
+func (g *Git) GetAutoPull() (autoPull bool) {
+	return g.AutoPull
+}
+
+func (g *Git) SetAutoPull(autoPull bool) {
+	g.AutoPull = autoPull
+}
+
+type GitList []Git
+
+func (l *GitList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/git_v2.go

@@ -0,0 +1,12 @@
+package models
+
+type GitV2 struct {
+	any                `collection:"gits"`
+	BaseModelV2[GitV2] `bson:",inline"`
+	Url                string `json:"url" bson:"url"`
+	AuthType           string `json:"auth_type" bson:"auth_type"`
+	Username           string `json:"username" bson:"username"`
+	Password           string `json:"password" bson:"password"`
+	CurrentBranch      string `json:"current_branch" bson:"current_branch"`
+	AutoPull           bool   `json:"auto_pull" bson:"auto_pull"`
+}

core/models/models/job.go

@@ -0,0 +1,29 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Job struct {
+	Id     primitive.ObjectID `bson:"_id" json:"_id"`
+	TaskId primitive.ObjectID `bson:"task_id" json:"task_id"`
+}
+
+func (j *Job) GetId() (id primitive.ObjectID) {
+	return j.Id
+}
+
+func (j *Job) SetId(id primitive.ObjectID) {
+	j.Id = id
+}
+
+type JobList []Job
+
+func (l *JobList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/node.go

@@ -0,0 +1,119 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type Node struct {
+	Id               primitive.ObjectID `json:"_id" bson:"_id"`
+	Key              string             `json:"key" bson:"key"`
+	Name             string             `json:"name" bson:"name"`
+	Ip               string             `json:"ip" bson:"ip"`
+	Port             string             `json:"port" bson:"port"`
+	Mac              string             `json:"mac" bson:"mac"`
+	Hostname         string             `json:"hostname" bson:"hostname"`
+	Description      string             `json:"description" bson:"description"`
+	IsMaster         bool               `json:"is_master" bson:"is_master"`
+	Status           string             `json:"status" bson:"status"`
+	Enabled          bool               `json:"enabled" bson:"enabled"`
+	Active           bool               `json:"active" bson:"active"`
+	ActiveTs         time.Time          `json:"active_ts" bson:"active_ts"`
+	AvailableRunners int                `json:"available_runners" bson:"available_runners"`
+	MaxRunners       int                `json:"max_runners" bson:"max_runners"`
+}
+
+func (n *Node) GetId() (id primitive.ObjectID) {
+	return n.Id
+}
+
+func (n *Node) SetId(id primitive.ObjectID) {
+	n.Id = id
+}
+
+func (n *Node) GetName() (name string) {
+	return n.Name
+}
+
+func (n *Node) SetName(name string) {
+	n.Name = name
+}
+
+func (n *Node) GetDescription() (description string) {
+	return n.Description
+}
+
+func (n *Node) SetDescription(description string) {
+	n.Description = description
+}
+
+func (n *Node) GetKey() (key string) {
+	return n.Key
+}
+
+func (n *Node) GetIsMaster() (ok bool) {
+	return n.IsMaster
+}
+
+func (n *Node) GetActive() (active bool) {
+	return n.Active
+}
+
+func (n *Node) SetActive(active bool) {
+	n.Active = active
+}
+
+func (n *Node) SetActiveTs(activeTs time.Time) {
+	n.ActiveTs = activeTs
+}
+
+func (n *Node) GetStatus() (status string) {
+	return n.Status
+}
+
+func (n *Node) SetStatus(status string) {
+	n.Status = status
+}
+
+func (n *Node) GetEnabled() (enabled bool) {
+	return n.Enabled
+}
+
+func (n *Node) SetEnabled(enabled bool) {
+	n.Enabled = enabled
+}
+
+func (n *Node) GetAvailableRunners() (runners int) {
+	return n.AvailableRunners
+}
+
+func (n *Node) SetAvailableRunners(runners int) {
+	n.AvailableRunners = runners
+}
+
+func (n *Node) GetMaxRunners() (runners int) {
+	return n.MaxRunners
+}
+
+func (n *Node) SetMaxRunners(runners int) {
+	n.MaxRunners = runners
+}
+
+func (n *Node) IncrementAvailableRunners() {
+	n.AvailableRunners++
+}
+
+func (n *Node) DecrementAvailableRunners() {
+	n.AvailableRunners--
+}
+
+type NodeList []Node
+
+func (l *NodeList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/node_v2.go

@@ -0,0 +1,24 @@
+package models
+
+import (
+	"time"
+)
+
+type NodeV2 struct {
+	any                 `collection:"nodes"`
+	BaseModelV2[NodeV2] `bson:",inline"`
+	Key                 string    `json:"key" bson:"key"`
+	Name                string    `json:"name" bson:"name"`
+	Ip                  string    `json:"ip" bson:"ip"`
+	Port                string    `json:"port" bson:"port"`
+	Mac                 string    `json:"mac" bson:"mac"`
+	Hostname            string    `json:"hostname" bson:"hostname"`
+	Description         string    `json:"description" bson:"description"`
+	IsMaster            bool      `json:"is_master" bson:"is_master"`
+	Status              string    `json:"status" bson:"status"`
+	Enabled             bool      `json:"enabled" bson:"enabled"`
+	Active              bool      `json:"active" bson:"active"`
+	ActiveAt            time.Time `json:"active_at" bson:"active_ts"`
+	AvailableRunners    int       `json:"available_runners" bson:"available_runners"`
+	MaxRunners          int       `json:"max_runners" bson:"max_runners"`
+}

core/models/models/notification_setting_v2.go

@@ -0,0 +1,32 @@
+package models
+
+import "go.mongodb.org/mongo-driver/bson/primitive"
+
+type NotificationSettingV2 struct {
+	Id          primitive.ObjectID        `json:"_id" bson:"_id"`
+	Type        string                    `json:"type" bson:"type"`
+	Name        string                    `json:"name" bson:"name"`
+	Description string                    `json:"description" bson:"description"`
+	Enabled     bool                      `json:"enabled" bson:"enabled"`
+	Global      bool                      `json:"global" bson:"global"`
+	Title       string                    `json:"title,omitempty" bson:"title,omitempty"`
+	Template    string                    `json:"template,omitempty" bson:"template,omitempty"`
+	TaskTrigger string                    `json:"task_trigger" bson:"task_trigger"`
+	Mail        NotificationSettingMail   `json:"mail,omitempty" bson:"mail,omitempty"`
+	Mobile      NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"`
+}
+
+type NotificationSettingMail struct {
+	Server         string `json:"server" bson:"server"`
+	Port           string `json:"port,omitempty" bson:"port,omitempty"`
+	User           string `json:"user,omitempty" bson:"user,omitempty"`
+	Password       string `json:"password,omitempty" bson:"password,omitempty"`
+	SenderEmail    string `json:"sender_email,omitempty" bson:"sender_email,omitempty"`
+	SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"`
+	To             string `json:"to,omitempty" bson:"to,omitempty"`
+	Cc             string `json:"cc,omitempty" bson:"cc,omitempty"`
+}
+
+type NotificationSettingMobile struct {
+	Webhook string `json:"webhook" bson:"webhook"`
+}

core/models/models/password.go

@@ -0,0 +1,29 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Password struct {
+	Id       primitive.ObjectID `json:"_id" bson:"_id"`
+	Password string             `json:"password" bson:"p"`
+}
+
+func (p *Password) GetId() (id primitive.ObjectID) {
+	return p.Id
+}
+
+func (p *Password) SetId(id primitive.ObjectID) {
+	p.Id = id
+}
+
+type PasswordList []Password
+
+func (l *PasswordList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/permission.go

@@ -0,0 +1,91 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Permission struct {
+	Id          primitive.ObjectID `json:"_id" bson:"_id"`
+	Key         string             `json:"key" bson:"key"`
+	Name        string             `json:"name" bson:"name"`
+	Description string             `json:"description" bson:"description"`
+	Type        string             `json:"type" bson:"type"`
+	Target      []string           `json:"target" bson:"target"`
+	Allow       []string           `json:"allow" bson:"allow"`
+	Deny        []string           `json:"deny" bson:"deny"`
+}
+
+func (p *Permission) GetId() (id primitive.ObjectID) {
+	return p.Id
+}
+
+func (p *Permission) SetId(id primitive.ObjectID) {
+	p.Id = id
+}
+
+func (p *Permission) GetKey() (key string) {
+	return p.Key
+}
+
+func (p *Permission) SetKey(key string) {
+	p.Key = key
+}
+
+func (p *Permission) GetName() (name string) {
+	return p.Name
+}
+
+func (p *Permission) SetName(name string) {
+	p.Name = name
+}
+
+func (p *Permission) GetDescription() (description string) {
+	return p.Description
+}
+
+func (p *Permission) SetDescription(description string) {
+	p.Description = description
+}
+
+func (p *Permission) GetType() (t string) {
+	return p.Type
+}
+
+func (p *Permission) SetType(t string) {
+	p.Type = t
+}
+
+func (p *Permission) GetTarget() (target []string) {
+	return p.Target
+}
+
+func (p *Permission) SetTarget(target []string) {
+	p.Target = target
+}
+
+func (p *Permission) GetAllow() (include []string) {
+	return p.Allow
+}
+
+func (p *Permission) SetAllow(include []string) {
+	p.Allow = include
+}
+
+func (p *Permission) GetDeny() (exclude []string) {
+	return p.Deny
+}
+
+func (p *Permission) SetDeny(exclude []string) {
+	p.Deny = exclude
+}
+
+type PermissionList []Permission
+
+func (l *PermissionList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/permission_v2.go

@@ -0,0 +1,13 @@
+package models
+
+type PermissionV2 struct {
+	any                       `collection:"permissions"`
+	BaseModelV2[PermissionV2] `bson:",inline"`
+	Key                       string   `json:"key" bson:"key"`
+	Name                      string   `json:"name" bson:"name"`
+	Description               string   `json:"description" bson:"description"`
+	Type                      string   `json:"type" bson:"type"`
+	Target                    []string `json:"target" bson:"target"`
+	Allow                     []string `json:"allow" bson:"allow"`
+	Deny                      []string `json:"deny" bson:"deny"`
+}

core/models/models/project.go

@@ -0,0 +1,47 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Project struct {
+	Id          primitive.ObjectID `json:"_id" bson:"_id"`
+	Name        string             `json:"name" bson:"name"`
+	Description string             `json:"description" bson:"description"`
+	Spiders     int                `json:"spiders" bson:"-"`
+}
+
+func (p *Project) GetId() (id primitive.ObjectID) {
+	return p.Id
+}
+
+func (p *Project) SetId(id primitive.ObjectID) {
+	p.Id = id
+}
+
+func (p *Project) GetName() (name string) {
+	return p.Name
+}
+
+func (p *Project) SetName(name string) {
+	p.Name = name
+}
+
+func (p *Project) GetDescription() (description string) {
+	return p.Description
+}
+
+func (p *Project) SetDescription(description string) {
+	p.Description = description
+}
+
+type ProjectList []Project
+
+func (l *ProjectList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/project_v2.go

@@ -0,0 +1,9 @@
+package models
+
+type ProjectV2 struct {
+	any                    `collection:"projects"`
+	BaseModelV2[ProjectV2] `bson:",inline"`
+	Name                   string `json:"name" bson:"name"`
+	Description            string `json:"description" bson:"description"`
+	Spiders                int    `json:"spiders" bson:"-"`
+}

core/models/models/result.go

@@ -0,0 +1,60 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Result bson.M
+
+func (r *Result) GetId() (id primitive.ObjectID) {
+	res, ok := r.Value()["_id"]
+	if ok {
+		id, ok = res.(primitive.ObjectID)
+		if ok {
+			return id
+		}
+	}
+	return id
+}
+
+func (r *Result) SetId(id primitive.ObjectID) {
+	(*r)["_id"] = id
+}
+
+func (r *Result) Value() map[string]interface{} {
+	return *r
+}
+
+func (r *Result) SetValue(key string, value interface{}) {
+	(*r)[key] = value
+}
+
+func (r *Result) GetValue(key string) (value interface{}) {
+	return (*r)[key]
+}
+
+func (r *Result) GetTaskId() (id primitive.ObjectID) {
+	res := r.GetValue(constants.TaskKey)
+	if res == nil {
+		return id
+	}
+	id, _ = res.(primitive.ObjectID)
+	return id
+}
+
+func (r *Result) SetTaskId(id primitive.ObjectID) {
+	r.SetValue(constants.TaskKey, id)
+}
+
+type ResultList []Result
+
+func (l *ResultList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/role.go

@@ -0,0 +1,55 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Role struct {
+	Id          primitive.ObjectID `json:"_id" bson:"_id"`
+	Key         string             `json:"key" bson:"key"`
+	Name        string             `json:"name" bson:"name"`
+	Description string             `json:"description" bson:"description"`
+}
+
+func (r *Role) GetId() (id primitive.ObjectID) {
+	return r.Id
+}
+
+func (r *Role) SetId(id primitive.ObjectID) {
+	r.Id = id
+}
+
+func (r *Role) GetKey() (key string) {
+	return r.Key
+}
+
+func (r *Role) SetKey(key string) {
+	r.Key = key
+}
+
+func (r *Role) GetName() (name string) {
+	return r.Name
+}
+
+func (r *Role) SetName(name string) {
+	r.Name = name
+}
+
+func (r *Role) GetDescription() (description string) {
+	return r.Description
+}
+
+func (r *Role) SetDescription(description string) {
+	r.Description = description
+}
+
+type RoleList []Role
+
+func (l *RoleList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/role_permission.go

@@ -0,0 +1,30 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type RolePermission struct {
+	Id           primitive.ObjectID `json:"_id" bson:"_id"`
+	RoleId       primitive.ObjectID `json:"role_id" bson:"role_id"`
+	PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"`
+}
+
+func (ur *RolePermission) GetId() (id primitive.ObjectID) {
+	return ur.Id
+}
+
+func (ur *RolePermission) SetId(id primitive.ObjectID) {
+	ur.Id = id
+}
+
+type RolePermissionList []RolePermission
+
+func (l *RolePermissionList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/role_permission_v2.go

@@ -0,0 +1,12 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type RolePermissionV2 struct {
+	any                           `collection:"role_permissions"`
+	BaseModelV2[RolePermissionV2] `bson:",inline"`
+	RoleId                        primitive.ObjectID `json:"role_id" bson:"role_id"`
+	PermissionId                  primitive.ObjectID `json:"permission_id" bson:"permission_id"`
+}

core/models/models/role_v2.go

@@ -0,0 +1,9 @@
+package models
+
+type RoleV2 struct {
+	any                 `collection:"roles"`
+	BaseModelV2[RoleV2] `bson:",inline"`
+	Key                 string `json:"key" bson:"key"`
+	Name                string `json:"name" bson:"name"`
+	Description         string `json:"description" bson:"description"`
+}

core/models/models/schedule.go

@@ -0,0 +1,113 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/robfig/cron/v3"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Schedule struct {
+	Id          primitive.ObjectID   `json:"_id" bson:"_id"`
+	Name        string               `json:"name" bson:"name"`
+	Description string               `json:"description" bson:"description"`
+	SpiderId    primitive.ObjectID   `json:"spider_id" bson:"spider_id"`
+	Cron        string               `json:"cron" bson:"cron"`
+	EntryId     cron.EntryID         `json:"entry_id" bson:"entry_id"`
+	Cmd         string               `json:"cmd" bson:"cmd"`
+	Param       string               `json:"param" bson:"param"`
+	Mode        string               `json:"mode" bson:"mode"`
+	NodeIds     []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
+	Priority    int                  `json:"priority" bson:"priority"`
+	Enabled     bool                 `json:"enabled" bson:"enabled"`
+	UserId      primitive.ObjectID   `json:"user_id" bson:"user_id"`
+}
+
+func (s *Schedule) GetId() (id primitive.ObjectID) {
+	return s.Id
+}
+
+func (s *Schedule) SetId(id primitive.ObjectID) {
+	s.Id = id
+}
+
+func (s *Schedule) GetEnabled() (enabled bool) {
+	return s.Enabled
+}
+
+func (s *Schedule) SetEnabled(enabled bool) {
+	s.Enabled = enabled
+}
+
+func (s *Schedule) GetEntryId() (id cron.EntryID) {
+	return s.EntryId
+}
+
+func (s *Schedule) SetEntryId(id cron.EntryID) {
+	s.EntryId = id
+}
+
+func (s *Schedule) GetCron() (c string) {
+	return s.Cron
+}
+
+func (s *Schedule) SetCron(c string) {
+	s.Cron = c
+}
+
+func (s *Schedule) GetSpiderId() (id primitive.ObjectID) {
+	return s.SpiderId
+}
+
+func (s *Schedule) SetSpiderId(id primitive.ObjectID) {
+	s.SpiderId = id
+}
+
+func (s *Schedule) GetMode() (mode string) {
+	return s.Mode
+}
+
+func (s *Schedule) SetMode(mode string) {
+	s.Mode = mode
+}
+
+func (s *Schedule) GetNodeIds() (ids []primitive.ObjectID) {
+	return s.NodeIds
+}
+
+func (s *Schedule) SetNodeIds(ids []primitive.ObjectID) {
+	s.NodeIds = ids
+}
+
+func (s *Schedule) GetCmd() (cmd string) {
+	return s.Cmd
+}
+
+func (s *Schedule) SetCmd(cmd string) {
+	s.Cmd = cmd
+}
+
+func (s *Schedule) GetParam() (param string) {
+	return s.Param
+}
+
+func (s *Schedule) SetParam(param string) {
+	s.Param = param
+}
+
+func (s *Schedule) GetPriority() (p int) {
+	return s.Priority
+}
+
+func (s *Schedule) SetPriority(p int) {
+	s.Priority = p
+}
+
+type ScheduleList []Schedule
+
+func (l *ScheduleList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/schedule_v2.go

@@ -0,0 +1,23 @@
+package models
+
+import (
+	"github.com/robfig/cron/v3"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type ScheduleV2 struct {
+	any                     `collection:"schedules"`
+	BaseModelV2[ScheduleV2] `bson:",inline"`
+	Name                    string               `json:"name" bson:"name"`
+	Description             string               `json:"description" bson:"description"`
+	SpiderId                primitive.ObjectID   `json:"spider_id" bson:"spider_id"`
+	Cron                    string               `json:"cron" bson:"cron"`
+	EntryId                 cron.EntryID         `json:"entry_id" bson:"entry_id"`
+	Cmd                     string               `json:"cmd" bson:"cmd"`
+	Param                   string               `json:"param" bson:"param"`
+	Mode                    string               `json:"mode" bson:"mode"`
+	NodeIds                 []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
+	Priority                int                  `json:"priority" bson:"priority"`
+	Enabled                 bool                 `json:"enabled" bson:"enabled"`
+	UserId                  primitive.ObjectID   `json:"user_id" bson:"user_id"`
+}

core/models/models/setting.go

@@ -0,0 +1,31 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Setting struct {
+	Id    primitive.ObjectID `json:"_id" bson:"_id"`
+	Key   string             `json:"key" bson:"key"`
+	Value bson.M             `json:"value" bson:"value"`
+}
+
+func (s *Setting) GetId() (id primitive.ObjectID) {
+	return s.Id
+}
+
+func (s *Setting) SetId(id primitive.ObjectID) {
+	s.Id = id
+}
+
+type SettingList []Setting
+
+func (l *SettingList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/setting_v2.go

@@ -0,0 +1,12 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+type SettingV2 struct {
+	any                    `collection:"settings"`
+	BaseModelV2[SettingV2] `bson:",inline"`
+	Key                    string `json:"key" bson:"key"`
+	Value                  bson.M `json:"value" bson:"value"`
+}

core/models/models/spider.go

@@ -0,0 +1,137 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Env struct {
+	Name  string `json:"name" bson:"name"`
+	Value string `json:"value" bson:"value"`
+}
+
+type Spider struct {
+	Id           primitive.ObjectID   `json:"_id" bson:"_id"`                       // spider id
+	Name         string               `json:"name" bson:"name"`                     // spider name
+	Type         string               `json:"type" bson:"type"`                     // spider type
+	ColId        primitive.ObjectID   `json:"col_id" bson:"col_id"`                 // data collection id
+	ColName      string               `json:"col_name,omitempty" bson:"-"`          // data collection name
+	DataSourceId primitive.ObjectID   `json:"data_source_id" bson:"data_source_id"` // data source id
+	DataSource   *DataSource          `json:"data_source,omitempty" bson:"-"`       // data source
+	Description  string               `json:"description" bson:"description"`       // description
+	ProjectId    primitive.ObjectID   `json:"project_id" bson:"project_id"`         // Project.Id
+	Mode         string               `json:"mode" bson:"mode"`                     // default Task.Mode
+	NodeIds      []primitive.ObjectID `json:"node_ids" bson:"node_ids"`             // default Task.NodeIds
+	Stat         *SpiderStat          `json:"stat,omitempty" bson:"-"`
+
+	// execution
+	Cmd         string `json:"cmd" bson:"cmd"`     // execute command
+	Param       string `json:"param" bson:"param"` // default task param
+	Priority    int    `json:"priority" bson:"priority"`
+	AutoInstall bool   `json:"auto_install" bson:"auto_install"`
+
+	// settings
+	IncrementalSync bool `json:"incremental_sync" bson:"incremental_sync"` // whether to incrementally sync files
+}
+
+func (s *Spider) GetId() (id primitive.ObjectID) {
+	return s.Id
+}
+
+func (s *Spider) SetId(id primitive.ObjectID) {
+	s.Id = id
+}
+
+func (s *Spider) GetName() (name string) {
+	return s.Name
+}
+
+func (s *Spider) SetName(name string) {
+	s.Name = name
+}
+
+func (s *Spider) GetDescription() (description string) {
+	return s.Description
+}
+
+func (s *Spider) SetDescription(description string) {
+	s.Description = description
+}
+
+func (s *Spider) GetType() (ty string) {
+	return s.Type
+}
+
+func (s *Spider) GetMode() (mode string) {
+	return s.Mode
+}
+
+func (s *Spider) SetMode(mode string) {
+	s.Mode = mode
+}
+
+func (s *Spider) GetNodeIds() (ids []primitive.ObjectID) {
+	return s.NodeIds
+}
+
+func (s *Spider) SetNodeIds(ids []primitive.ObjectID) {
+	s.NodeIds = ids
+}
+
+func (s *Spider) GetCmd() (cmd string) {
+	return s.Cmd
+}
+
+func (s *Spider) SetCmd(cmd string) {
+	s.Cmd = cmd
+}
+
+func (s *Spider) GetParam() (param string) {
+	return s.Param
+}
+
+func (s *Spider) SetParam(param string) {
+	s.Param = param
+}
+
+func (s *Spider) GetPriority() (p int) {
+	return s.Priority
+}
+
+func (s *Spider) SetPriority(p int) {
+	s.Priority = p
+}
+
+func (s *Spider) GetColId() (id primitive.ObjectID) {
+	return s.ColId
+}
+
+func (s *Spider) SetColId(id primitive.ObjectID) {
+	s.ColId = id
+}
+
+func (s *Spider) GetIncrementalSync() (incrementalSync bool) {
+	return s.IncrementalSync
+}
+
+func (s *Spider) SetIncrementalSync(incrementalSync bool) {
+	s.IncrementalSync = incrementalSync
+}
+
+func (s *Spider) GetAutoInstall() (autoInstall bool) {
+	return s.AutoInstall
+}
+
+func (s *Spider) SetAutoInstall(autoInstall bool) {
+	s.AutoInstall = autoInstall
+}
+
+type SpiderList []Spider
+
+func (l *SpiderList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/spider_stat.go

@@ -0,0 +1,38 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type SpiderStat struct {
+	Id                     primitive.ObjectID `json:"_id" bson:"_id"`
+	LastTaskId             primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"`
+	LastTask               *Task              `json:"last_task,omitempty" bson:"-"`
+	Tasks                  int                `json:"tasks" bson:"tasks"`
+	Results                int                `json:"results" bson:"results"`
+	WaitDuration           int64              `json:"wait_duration" bson:"wait_duration,omitempty"`       // in second
+	RuntimeDuration        int64              `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second
+	TotalDuration          int64              `json:"total_duration" bson:"total_duration,omitempty"`     // in second
+	AverageWaitDuration    int64              `json:"average_wait_duration" bson:"-"`                     // in second
+	AverageRuntimeDuration int64              `json:"average_runtime_duration" bson:"-"`                  // in second
+	AverageTotalDuration   int64              `json:"average_total_duration" bson:"-"`                    // in second
+}
+
+func (s *SpiderStat) GetId() (id primitive.ObjectID) {
+	return s.Id
+}
+
+func (s *SpiderStat) SetId(id primitive.ObjectID) {
+	s.Id = id
+}
+
+type SpiderStatList []SpiderStat
+
+func (l *SpiderStatList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/spider_stat_v2.go

@@ -0,0 +1,20 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type SpiderStatV2 struct {
+	any                       `collection:"spider_stats"`
+	BaseModelV2[SpiderStatV2] `bson:",inline"`
+	LastTaskId                primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"`
+	LastTask                  *TaskV2            `json:"last_task,omitempty" bson:"-"`
+	Tasks                     int                `json:"tasks" bson:"tasks"`
+	Results                   int                `json:"results" bson:"results"`
+	WaitDuration              int64              `json:"wait_duration" bson:"wait_duration,omitempty"`       // in second
+	RuntimeDuration           int64              `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second
+	TotalDuration             int64              `json:"total_duration" bson:"total_duration,omitempty"`     // in second
+	AverageWaitDuration       int64              `json:"average_wait_duration" bson:"-"`                     // in second
+	AverageRuntimeDuration    int64              `json:"average_runtime_duration" bson:"-"`                  // in second
+	AverageTotalDuration      int64              `json:"average_total_duration" bson:"-"`                    // in second
+}

core/models/models/spider_v2.go

@@ -0,0 +1,30 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type SpiderV2 struct {
+	any                   `collection:"spiders"` // spider id
+	BaseModelV2[SpiderV2] `bson:",inline"`
+	Name                  string               `json:"name" bson:"name"`                     // spider name
+	Type                  string               `json:"type" bson:"type"`                     // spider type
+	ColId                 primitive.ObjectID   `json:"col_id" bson:"col_id"`                 // data collection id
+	ColName               string               `json:"col_name,omitempty" bson:"-"`          // data collection name
+	DataSourceId          primitive.ObjectID   `json:"data_source_id" bson:"data_source_id"` // data source id
+	DataSource            *DataSourceV2        `json:"data_source,omitempty" bson:"-"`       // data source
+	Description           string               `json:"description" bson:"description"`       // description
+	ProjectId             primitive.ObjectID   `json:"project_id" bson:"project_id"`         // Project.Id
+	Mode                  string               `json:"mode" bson:"mode"`                     // default Task.Mode
+	NodeIds               []primitive.ObjectID `json:"node_ids" bson:"node_ids"`             // default Task.NodeIds
+	Stat                  *SpiderStatV2        `json:"stat,omitempty" bson:"-"`
+
+	// execution
+	Cmd         string `json:"cmd" bson:"cmd"`     // execute command
+	Param       string `json:"param" bson:"param"` // default task param
+	Priority    int    `json:"priority" bson:"priority"`
+	AutoInstall bool   `json:"auto_install" bson:"auto_install"`
+
+	// settings
+	IncrementalSync bool `json:"incremental_sync" bson:"incremental_sync"` // whether to incrementally sync files
+}

core/models/models/tag.go

@@ -0,0 +1,44 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Tag struct {
+	Id          primitive.ObjectID `json:"_id" bson:"_id"`
+	Name        string             `json:"name" bson:"name"`
+	Color       string             `json:"color" bson:"color"`
+	Description string             `json:"description" bson:"description"`
+	Col         string             `json:"col" bson:"col"`
+}
+
+func (t *Tag) GetId() (id primitive.ObjectID) {
+	return t.Id
+}
+
+func (t *Tag) SetId(id primitive.ObjectID) {
+	t.Id = id
+}
+
+func (t *Tag) GetName() (res string) {
+	return t.Name
+}
+
+func (t *Tag) GetColor() (res string) {
+	return t.Color
+}
+
+func (t *Tag) SetCol(col string) {
+	t.Col = col
+}
+
+type TagList []Tag
+
+func (l *TagList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/task.go

@@ -0,0 +1,118 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type Task struct {
+	Id         primitive.ObjectID   `json:"_id" bson:"_id"`
+	SpiderId   primitive.ObjectID   `json:"spider_id" bson:"spider_id"`
+	Status     string               `json:"status" bson:"status"`
+	NodeId     primitive.ObjectID   `json:"node_id" bson:"node_id"`
+	Cmd        string               `json:"cmd" bson:"cmd"`
+	Param      string               `json:"param" bson:"param"`
+	Error      string               `json:"error" bson:"error"`
+	Pid        int                  `json:"pid" bson:"pid"`
+	ScheduleId primitive.ObjectID   `json:"schedule_id" bson:"schedule_id"` // Schedule.Id
+	Type       string               `json:"type" bson:"type"`
+	Mode       string               `json:"mode" bson:"mode"`           // running mode of Task
+	NodeIds    []primitive.ObjectID `json:"node_ids" bson:"node_ids"`   // list of Node.Id
+	ParentId   primitive.ObjectID   `json:"parent_id" bson:"parent_id"` // parent Task.Id if it'Spider a sub-task
+	Priority   int                  `json:"priority" bson:"priority"`
+	Stat       *TaskStat            `json:"stat,omitempty" bson:"-"`
+	HasSub     bool                 `json:"has_sub" json:"has_sub"` // whether to have sub-tasks
+	SubTasks   []Task               `json:"sub_tasks,omitempty" bson:"-"`
+	Spider     *Spider              `json:"spider,omitempty" bson:"-"`
+	UserId     primitive.ObjectID   `json:"-" bson:"-"`
+	CreateTs   time.Time            `json:"create_ts" bson:"create_ts"`
+}
+
+func (t *Task) GetId() (id primitive.ObjectID) {
+	return t.Id
+}
+
+func (t *Task) SetId(id primitive.ObjectID) {
+	t.Id = id
+}
+
+func (t *Task) GetNodeId() (id primitive.ObjectID) {
+	return t.NodeId
+}
+
+func (t *Task) SetNodeId(id primitive.ObjectID) {
+	t.NodeId = id
+}
+
+func (t *Task) GetNodeIds() (ids []primitive.ObjectID) {
+	return t.NodeIds
+}
+
+func (t *Task) GetStatus() (status string) {
+	return t.Status
+}
+
+func (t *Task) SetStatus(status string) {
+	t.Status = status
+}
+
+func (t *Task) GetError() (error string) {
+	return t.Error
+}
+
+func (t *Task) SetError(error string) {
+	t.Error = error
+}
+
+func (t *Task) GetPid() (pid int) {
+	return t.Pid
+}
+
+func (t *Task) SetPid(pid int) {
+	t.Pid = pid
+}
+
+func (t *Task) GetSpiderId() (id primitive.ObjectID) {
+	return t.SpiderId
+}
+
+func (t *Task) GetType() (ty string) {
+	return t.Type
+}
+
+func (t *Task) GetCmd() (cmd string) {
+	return t.Cmd
+}
+
+func (t *Task) GetParam() (param string) {
+	return t.Param
+}
+
+func (t *Task) GetPriority() (p int) {
+	return t.Priority
+}
+
+func (t *Task) GetUserId() (id primitive.ObjectID) {
+	return t.UserId
+}
+
+func (t *Task) SetUserId(id primitive.ObjectID) {
+	t.UserId = id
+}
+
+type TaskList []Task
+
+func (l *TaskList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}
+
+type TaskDailyItem struct {
+	Date               string  `json:"date" bson:"_id"`
+	TaskCount          int     `json:"task_count" bson:"task_count"`
+	AvgRuntimeDuration float64 `json:"avg_runtime_duration" bson:"avg_runtime_duration"`
+}

core/models/models/task_queue_item.go

@@ -0,0 +1,30 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type TaskQueueItem struct {
+	Id       primitive.ObjectID `json:"_id" bson:"_id"`
+	Priority int                `json:"p" bson:"p"`
+	NodeId   primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"`
+}
+
+func (t *TaskQueueItem) GetId() (id primitive.ObjectID) {
+	return t.Id
+}
+
+func (t *TaskQueueItem) SetId(id primitive.ObjectID) {
+	t.Id = id
+}
+
+type TaskQueueItemList []TaskQueueItem
+
+func (l *TaskQueueItemList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/task_queue_item_v2.go

@@ -0,0 +1,12 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type TaskQueueItemV2 struct {
+	any                          `collection:"task_queue"`
+	BaseModelV2[TaskQueueItemV2] `bson:",inline"`
+	Priority                     int                `json:"p" bson:"p"`
+	NodeId                       primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"`
+}

core/models/models/task_stat.go

@@ -0,0 +1,101 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type TaskStat struct {
+	Id              primitive.ObjectID `json:"_id" bson:"_id"`
+	CreateTs        time.Time          `json:"create_ts" bson:"create_ts,omitempty"`
+	StartTs         time.Time          `json:"start_ts" bson:"start_ts,omitempty"`
+	EndTs           time.Time          `json:"end_ts" bson:"end_ts,omitempty"`
+	WaitDuration    int64              `json:"wait_duration" bson:"wait_duration,omitempty"`       // in millisecond
+	RuntimeDuration int64              `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond
+	TotalDuration   int64              `json:"total_duration" bson:"total_duration,omitempty"`     // in millisecond
+	ResultCount     int64              `json:"result_count" bson:"result_count"`
+	ErrorLogCount   int64              `json:"error_log_count" bson:"error_log_count"`
+}
+
+func (s *TaskStat) GetId() (id primitive.ObjectID) {
+	return s.Id
+}
+
+func (s *TaskStat) SetId(id primitive.ObjectID) {
+	s.Id = id
+}
+
+func (s *TaskStat) GetCreateTs() (ts time.Time) {
+	return s.CreateTs
+}
+
+func (s *TaskStat) SetCreateTs(ts time.Time) {
+	s.CreateTs = ts
+}
+
+func (s *TaskStat) GetStartTs() (ts time.Time) {
+	return s.StartTs
+}
+
+func (s *TaskStat) SetStartTs(ts time.Time) {
+	s.StartTs = ts
+}
+
+func (s *TaskStat) GetEndTs() (ts time.Time) {
+	return s.EndTs
+}
+
+func (s *TaskStat) SetEndTs(ts time.Time) {
+	s.EndTs = ts
+}
+
+func (s *TaskStat) GetWaitDuration() (d int64) {
+	return s.WaitDuration
+}
+
+func (s *TaskStat) SetWaitDuration(d int64) {
+	s.WaitDuration = d
+}
+
+func (s *TaskStat) GetRuntimeDuration() (d int64) {
+	return s.RuntimeDuration
+}
+
+func (s *TaskStat) SetRuntimeDuration(d int64) {
+	s.RuntimeDuration = d
+}
+
+func (s *TaskStat) GetTotalDuration() (d int64) {
+	return s.WaitDuration + s.RuntimeDuration
+}
+
+func (s *TaskStat) SetTotalDuration(d int64) {
+	s.TotalDuration = d
+}
+
+func (s *TaskStat) GetResultCount() (c int64) {
+	return s.ResultCount
+}
+
+func (s *TaskStat) SetResultCount(c int64) {
+	s.ResultCount = c
+}
+
+func (s *TaskStat) GetErrorLogCount() (c int64) {
+	return s.ErrorLogCount
+}
+
+func (s *TaskStat) SetErrorLogCount(c int64) {
+	s.ErrorLogCount = c
+}
+
+type TaskStatList []TaskStat
+
+func (l *TaskStatList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/task_stat_v2.go

@@ -0,0 +1,18 @@
+package models
+
+import (
+	"time"
+)
+
+type TaskStatV2 struct {
+	any                     `collection:"task_stats"`
+	BaseModelV2[TaskStatV2] `bson:",inline"`
+	CreateTs                time.Time `json:"create_ts" bson:"create_ts,omitempty"`
+	StartTs                 time.Time `json:"start_ts" bson:"start_ts,omitempty"`
+	EndTs                   time.Time `json:"end_ts" bson:"end_ts,omitempty"`
+	WaitDuration            int64     `json:"wait_duration" bson:"wait_duration,omitempty"`       // in millisecond
+	RuntimeDuration         int64     `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond
+	TotalDuration           int64     `json:"total_duration" bson:"total_duration,omitempty"`     // in millisecond
+	ResultCount             int64     `json:"result_count" bson:"result_count"`
+	ErrorLogCount           int64     `json:"error_log_count" bson:"error_log_count"`
+}

core/models/models/task_v2.go

@@ -0,0 +1,30 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type TaskV2 struct {
+	any                 `collection:"tasks"`
+	BaseModelV2[TaskV2] `bson:",inline"`
+	SpiderId            primitive.ObjectID   `json:"spider_id" bson:"spider_id"`
+	Status              string               `json:"status" bson:"status"`
+	NodeId              primitive.ObjectID   `json:"node_id" bson:"node_id"`
+	Cmd                 string               `json:"cmd" bson:"cmd"`
+	Param               string               `json:"param" bson:"param"`
+	Error               string               `json:"error" bson:"error"`
+	Pid                 int                  `json:"pid" bson:"pid"`
+	ScheduleId          primitive.ObjectID   `json:"schedule_id" bson:"schedule_id"`
+	Type                string               `json:"type" bson:"type"`
+	Mode                string               `json:"mode" bson:"mode"`
+	NodeIds             []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
+	ParentId            primitive.ObjectID   `json:"parent_id" bson:"parent_id"`
+	Priority            int                  `json:"priority" bson:"priority"`
+	Stat                *TaskStatV2          `json:"stat,omitempty" bson:"-"`
+	HasSub              bool                 `json:"has_sub" json:"has_sub"`
+	SubTasks            []TaskV2             `json:"sub_tasks,omitempty" bson:"-"`
+	Spider              *SpiderV2            `json:"spider,omitempty" bson:"-"`
+	UserId              primitive.ObjectID   `json:"-" bson:"-"`
+	CreateTs            time.Time            `json:"create_ts" bson:"create_ts"`
+}

core/models/models/test.go

@@ -0,0 +1,7 @@
+package models
+
+type TestModel struct {
+	any                    `collection:"testmodels"`
+	BaseModelV2[TestModel] `bson:",inline"`
+	Name                   string `json:"name" bson:"name"`
+}

core/models/models/token.go

@@ -0,0 +1,30 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Token struct {
+	Id    primitive.ObjectID `json:"_id" bson:"_id"`
+	Name  string             `json:"name" bson:"name"`
+	Token string             `json:"token" bson:"token"`
+}
+
+func (t *Token) GetId() (id primitive.ObjectID) {
+	return t.Id
+}
+
+func (t *Token) SetId(id primitive.ObjectID) {
+	t.Id = id
+}
+
+type TokenList []Token
+
+func (l *TokenList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/token_v2.go

@@ -0,0 +1,8 @@
+package models
+
+type TokenV2 struct {
+	any                  `collection:"tokens"`
+	BaseModelV2[TokenV2] `bson:",inline"`
+	Name                 string `json:"name" bson:"name"`
+	Token                string `json:"token" bson:"token"`
+}

core/models/models/user.go

@@ -0,0 +1,59 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type User struct {
+	Id       primitive.ObjectID `json:"_id" bson:"_id"`
+	Username string             `json:"username" bson:"username"`
+	Password string             `json:"password,omitempty" bson:"-"`
+	Role     string             `json:"role" bson:"role"`
+	Email    string             `json:"email" bson:"email"`
+	//Setting  UserSetting        `json:"setting" bson:"setting"`
+}
+
+func (u *User) GetId() (id primitive.ObjectID) {
+	return u.Id
+}
+
+func (u *User) SetId(id primitive.ObjectID) {
+	u.Id = id
+}
+
+func (u *User) GetUsername() (name string) {
+	return u.Username
+}
+
+func (u *User) GetPassword() (p string) {
+	return u.Password
+}
+
+func (u *User) GetRole() (r string) {
+	return u.Role
+}
+
+func (u *User) GetEmail() (email string) {
+	return u.Email
+}
+
+//type UserSetting struct {
+//	NotificationTrigger  string   `json:"notification_trigger" bson:"notification_trigger"`
+//	DingTalkRobotWebhook string   `json:"ding_talk_robot_webhook" bson:"ding_talk_robot_webhook"`
+//	WechatRobotWebhook   string   `json:"wechat_robot_webhook" bson:"wechat_robot_webhook"`
+//	EnabledNotifications []string `json:"enabled_notifications" bson:"enabled_notifications"`
+//	ErrorRegexPattern    string   `json:"error_regex_pattern" bson:"error_regex_pattern"`
+//	MaxErrorLog          int      `json:"max_error_log" bson:"max_error_log"`
+//	LogExpireDuration    int64    `json:"log_expire_duration" bson:"log_expire_duration"`
+//}
+
+type UserList []User
+
+func (l *UserList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/user_role.go

@@ -0,0 +1,30 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type UserRole struct {
+	Id     primitive.ObjectID `json:"_id" bson:"_id"`
+	RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
+	UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
+}
+
+func (ur *UserRole) GetId() (id primitive.ObjectID) {
+	return ur.Id
+}
+
+func (ur *UserRole) SetId(id primitive.ObjectID) {
+	ur.Id = id
+}
+
+type UserRoleList []UserRole
+
+func (l *UserRoleList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/user_role_v2.go

@@ -0,0 +1,12 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type UserRoleV2 struct {
+	any                     `collection:"user_roles"`
+	BaseModelV2[UserRoleV2] `bson:",inline"`
+	RoleId                  primitive.ObjectID `json:"role_id" bson:"role_id"`
+	UserId                  primitive.ObjectID `json:"user_id" bson:"user_id"`
+}

core/models/models/user_v2.go

@@ -0,0 +1,10 @@
+package models
+
+type UserV2 struct {
+	any                 `collection:"users"`
+	BaseModelV2[UserV2] `bson:",inline"`
+	Username            string `json:"username" bson:"username"`
+	Password            string `json:"-,omitempty" bson:"password"`
+	Role                string `json:"role" bson:"role"`
+	Email               string `json:"email" bson:"email"`
+}

core/models/models/utils_binder_legacy.go

@@ -0,0 +1,96 @@
+package models
+
+//func AssignFields(d interface{}, fieldIds ...interfaces.ModelId) (res interface{}, err error) {
+//	return assignFields(d, fieldIds...)
+//}
+//
+//func AssignListFields(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
+//	return assignListFields(list, fieldIds...)
+//}
+//
+//func AssignListFieldsAsPtr(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
+//	return assignListFieldsAsPtr(list, fieldIds...)
+//}
+//
+//func assignFields(d interface{}, fieldIds ...interfaces.ModelId) (res interface{}, err error) {
+//	doc, ok := d.(interfaces.Model)
+//	if !ok {
+//		return nil, errors.ErrorModelInvalidType
+//	}
+//	if len(fieldIds) == 0 {
+//		return doc, nil
+//	}
+//	for _, fid := range fieldIds {
+//		switch fid {
+//		case interfaces.ModelIdTag:
+//			// convert interface
+//			d, ok := doc.(interfaces.ModelWithTags)
+//			if !ok {
+//				return nil, errors.ErrorModelInvalidType
+//			}
+//
+//			// attempt to get artifact
+//			a, err := doc.GetArtifact()
+//			if err != nil {
+//				return nil, err
+//			}
+//
+//			// skip if no artifact found
+//			if a == nil {
+//				return d, nil
+//			}
+//
+//			// assign tags
+//			tags, err := a.GetTags()
+//			if err != nil {
+//				return nil, err
+//			}
+//			d.SetTags(tags)
+//
+//			return d, nil
+//		}
+//	}
+//	return doc, nil
+//}
+//
+//func _assignListFields(asPtr bool, list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
+//	vList := reflect.ValueOf(list)
+//	if vList.Kind() != reflect.Array &&
+//		vList.Kind() != reflect.Slice {
+//		return res, errors.ErrorModelInvalidType
+//	}
+//	for i := 0; i < vList.Len(); i++ {
+//		vItem := vList.Index(i)
+//		var item interface{}
+//		if vItem.CanAddr() {
+//			item = vItem.Addr().Interface()
+//		} else {
+//			item = vItem.Interface()
+//		}
+//		doc, ok := item.(interfaces.Model)
+//		if !ok {
+//			return res, errors.ErrorModelInvalidType
+//		}
+//		ptr, err := assignFields(doc, fieldIds...)
+//		if err != nil {
+//			return res, err
+//		}
+//		v := reflect.ValueOf(ptr)
+//		if !asPtr {
+//			// non-pointer item
+//			res.Add(v.Elem().Interface())
+//		} else {
+//			// pointer item
+//			res.Add(v.Interface())
+//		}
+//	}
+//	return res, nil
+//}
+//
+//func assignListFields(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
+//	return _assignListFields(false, list, fieldIds...)
+//}
+//
+//func assignListFieldsAsPtr(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
+//	return _assignListFields(true, list, fieldIds...)
+//}

core/models/models/utils_col.go

@@ -0,0 +1,10 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/utils/binders"
+)
+
+func GetModelColName(id interfaces.ModelId) (colName string) {
+	return binders.NewColNameBinder(id).MustBindString()
+}

core/models/models/utils_model_map.go

@@ -0,0 +1,95 @@
+package models
+
+type ModelMap struct {
+	Artifact          Artifact
+	Tag               Tag
+	Node              Node
+	Project           Project
+	Spider            Spider
+	Task              Task
+	Job               Job
+	Schedule          Schedule
+	User              User
+	Setting           Setting
+	Token             Token
+	Variable          Variable
+	TaskQueueItem     TaskQueueItem
+	TaskStat          TaskStat
+	SpiderStat        SpiderStat
+	DataSource        DataSource
+	DataCollection    DataCollection
+	Result            Result
+	Password          Password
+	ExtraValue        ExtraValue
+	Git               Git
+	Role              Role
+	UserRole          UserRole
+	Permission        Permission
+	RolePermission    RolePermission
+	Environment       Environment
+	DependencySetting DependencySetting
+}
+
+type ModelListMap struct {
+	Artifacts          ArtifactList
+	Tags               TagList
+	Nodes              NodeList
+	Projects           ProjectList
+	Spiders            SpiderList
+	Tasks              TaskList
+	Jobs               JobList
+	Schedules          ScheduleList
+	Users              UserList
+	Settings           SettingList
+	Tokens             TokenList
+	Variables          VariableList
+	TaskQueueItems     TaskQueueItemList
+	TaskStats          TaskStatList
+	SpiderStats        SpiderStatList
+	DataSources        DataSourceList
+	DataCollections    DataCollectionList
+	Results            ResultList
+	Passwords          PasswordList
+	ExtraValues        ExtraValueList
+	Gits               GitList
+	Roles              RoleList
+	UserRoles          UserRoleList
+	PermissionList     PermissionList
+	RolePermissionList RolePermissionList
+	Environments       EnvironmentList
+	DependencySettings DependencySettingList
+}
+
+func NewModelMap() (m *ModelMap) {
+	return &ModelMap{}
+}
+
+func NewModelListMap() (m *ModelListMap) {
+	return &ModelListMap{
+		Artifacts:          ArtifactList{},
+		Tags:               TagList{},
+		Nodes:              NodeList{},
+		Projects:           ProjectList{},
+		Spiders:            SpiderList{},
+		Tasks:              TaskList{},
+		Jobs:               JobList{},
+		Schedules:          ScheduleList{},
+		Users:              UserList{},
+		Settings:           SettingList{},
+		Tokens:             TokenList{},
+		Variables:          VariableList{},
+		TaskQueueItems:     TaskQueueItemList{},
+		TaskStats:          TaskStatList{},
+		SpiderStats:        SpiderStatList{},
+		DataSources:        DataSourceList{},
+		DataCollections:    DataCollectionList{},
+		Results:            ResultList{},
+		Passwords:          PasswordList{},
+		ExtraValues:        ExtraValueList{},
+		Gits:               GitList{},
+		Roles:              RoleList{},
+		PermissionList:     PermissionList{},
+		RolePermissionList: RolePermissionList{},
+		Environments:       EnvironmentList{},
+	}
+}

core/models/models/utils_tag.go

@@ -0,0 +1,34 @@
+package models
+
+import (
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/go-trace"
+)
+
+func convertInterfacesToTags(tags []interfaces.Tag) (res []Tag) {
+	if tags == nil {
+		return nil
+	}
+	for _, t := range tags {
+		tag, ok := t.(*Tag)
+		if !ok {
+			log.Warnf("%v: cannot convert tag", trace.TraceError(errors.ErrorModelInvalidType))
+			return nil
+		}
+		if tag == nil {
+			log.Warnf("%v: cannot convert tag", trace.TraceError(errors.ErrorModelInvalidType))
+			return nil
+		}
+		res = append(res, *tag)
+	}
+	return res
+}
+
+func convertTagsToInterfaces(tags []Tag) (res []interfaces.Tag) {
+	for _, t := range tags {
+		res = append(res, &t)
+	}
+	return res
+}

core/models/models/variable.go

@@ -0,0 +1,31 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Variable struct {
+	Id     primitive.ObjectID `json:"_id" bson:"_id"`
+	Key    string             `json:"key" bson:"key"`
+	Value  string             `json:"value" bson:"value"`
+	Remark string             `json:"remark" bson:"remark"`
+}
+
+func (v *Variable) GetId() (id primitive.ObjectID) {
+	return v.Id
+}
+
+func (v *Variable) SetId(id primitive.ObjectID) {
+	v.Id = id
+}
+
+type VariableList []Variable
+
+func (l *VariableList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/variable_v2.go

@@ -0,0 +1,9 @@
+package models
+
+type VariableV2 struct {
+	any                     `collection:"variables"`
+	BaseModelV2[VariableV2] `bson:",inline"`
+	Key                     string `json:"key" bson:"key"`
+	Value                   string `json:"value" bson:"value"`
+	Remark                  string `json:"remark" bson:"remark"`
+}

core/models/service/artifact_service.go

@@ -0,0 +1,40 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func convertTypeArtifact(d interface{}, err error) (res *models2.Artifact, err2 error) {
+	if err != nil {
+		return nil, err
+	}
+	res, ok := d.(*models2.Artifact)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return res, nil
+}
+
+func (svc *Service) GetArtifactById(id primitive.ObjectID) (res *models2.Artifact, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdArtifact).GetById(id)
+	return convertTypeArtifact(d, err)
+}
+
+func (svc *Service) GetArtifact(query bson.M, opts *mongo.FindOptions) (res *models2.Artifact, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdArtifact).Get(query, opts)
+	return convertTypeArtifact(d, err)
+}
+
+func (svc *Service) GetArtifactList(query bson.M, opts *mongo.FindOptions) (res []models2.Artifact, err error) {
+	l, err := svc.GetBaseService(interfaces.ModelIdArtifact).GetList(query, opts)
+	for _, doc := range l.GetModels() {
+		d := doc.(*models2.Artifact)
+		res = append(res, *d)
+	}
+	return res, nil
+}

core/models/service/base_service.go

@@ -0,0 +1,420 @@
+package service
+
+import (
+	"encoding/json"
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"reflect"
+	"strings"
+	"sync"
+	"time"
+)
+
+type BaseService struct {
+	id  interfaces.ModelId
+	col *mongo.Col
+}
+
+func (svc *BaseService) GetModelId() (id interfaces.ModelId) {
+	return svc.id
+}
+
+func (svc *BaseService) SetModelId(id interfaces.ModelId) {
+	svc.id = id
+}
+
+func (svc *BaseService) GetCol() (col *mongo.Col) {
+	return svc.col
+}
+
+func (svc *BaseService) SetCol(col *mongo.Col) {
+	svc.col = col
+}
+
+func (svc *BaseService) GetById(id primitive.ObjectID) (res interfaces.Model, err error) {
+	// find result
+	fr := svc.findId(id)
+
+	// bind
+	return NewBasicBinder(svc.id, fr).Bind()
+}
+
+func (svc *BaseService) Get(query bson.M, opts *mongo.FindOptions) (res interfaces.Model, err error) {
+	// find result
+	fr := svc.find(query, opts)
+
+	// bind
+	return NewBasicBinder(svc.id, fr).Bind()
+}
+
+func (svc *BaseService) GetList(query bson.M, opts *mongo.FindOptions) (l interfaces.List, err error) {
+	// find result
+	tic := time.Now()
+	log.Debugf("baseService.GetMany -> svc.find:start")
+	log.Debugf("baseService.GetMany -> svc.id: %v", svc.id)
+	log.Debugf("baseService.GetMany -> svc.col.GetName(): %v", svc.col.GetName())
+	log.Debugf("baseService.GetMany -> query: %v", query)
+	log.Debugf("baseService.GetMany -> opts: %v", opts)
+	fr := svc.find(query, opts)
+	log.Debugf("baseService.GetMany -> svc.find:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds())
+
+	// bind
+	return NewListBinder(svc.id, fr).Bind()
+}
+
+func (svc *BaseService) DeleteById(id primitive.ObjectID, args ...interface{}) (err error) {
+	return svc.deleteId(id, args...)
+}
+
+func (svc *BaseService) Delete(query bson.M, args ...interface{}) (err error) {
+	return svc.delete(query)
+}
+
+func (svc *BaseService) DeleteList(query bson.M, args ...interface{}) (err error) {
+	return svc.deleteList(query)
+}
+
+func (svc *BaseService) ForceDeleteList(query bson.M, args ...interface{}) (err error) {
+	return svc.forceDeleteList(query)
+}
+
+func (svc *BaseService) UpdateById(id primitive.ObjectID, update bson.M, args ...interface{}) (err error) {
+	return svc.updateId(id, update)
+}
+
+func (svc *BaseService) Update(query bson.M, update bson.M, fields []string, args ...interface{}) (err error) {
+	return svc.update(query, update, fields)
+}
+
+func (svc *BaseService) UpdateDoc(query bson.M, doc interfaces.Model, fields []string, args ...interface{}) (err error) {
+	return svc.update(query, doc, fields)
+}
+
+func (svc *BaseService) Insert(u interfaces.User, docs ...interface{}) (err error) {
+	log.Debugf("baseService.Insert -> svc.col.GetName(): %v", svc.col.GetName())
+	log.Debugf("baseService.Insert -> docs: %v", docs)
+	return svc.insert(u, docs...)
+}
+
+func (svc *BaseService) Count(query bson.M) (total int, err error) {
+	return svc.count(query)
+}
+
+func (svc *BaseService) findId(id primitive.ObjectID) (fr *mongo.FindResult) {
+	if svc.col == nil {
+		return mongo.NewFindResultWithError(constants.ErrMissingCol)
+	}
+	return svc.col.FindId(id)
+}
+
+func (svc *BaseService) find(query bson.M, opts *mongo.FindOptions) (fr *mongo.FindResult) {
+	if svc.col == nil {
+		return mongo.NewFindResultWithError(constants.ErrMissingCol)
+	}
+	return svc.col.Find(query, opts)
+}
+
+func (svc *BaseService) deleteId(id primitive.ObjectID, args ...interface{}) (err error) {
+	if svc.col == nil {
+		return trace.TraceError(constants.ErrMissingCol)
+	}
+	fr := svc.findId(id)
+	doc, err := NewBasicBinder(svc.id, fr).Bind()
+	if err != nil {
+		return err
+	}
+	return delegate.NewModelDelegate(doc, svc._getUserFromArgs(args...)).Delete()
+}
+
+func (svc *BaseService) delete(query bson.M, args ...interface{}) (err error) {
+	if svc.col == nil {
+		return trace.TraceError(constants.ErrMissingCol)
+	}
+	var doc models2.BaseModel
+	if err := svc.find(query, nil).One(&doc); err != nil {
+		return err
+	}
+	return svc.deleteId(doc.Id, svc._getUserFromArgs(args...))
+}
+
+func (svc *BaseService) deleteList(query bson.M, args ...interface{}) (err error) {
+	if svc.col == nil {
+		return trace.TraceError(constants.ErrMissingCol)
+	}
+	fr := svc.find(query, nil)
+	list, err := NewListBinder(svc.id, fr).Bind()
+	if err != nil {
+		return err
+	}
+	for _, doc := range list.GetModels() {
+		if err := delegate.NewModelDelegate(doc, svc._getUserFromArgs(args...)).Delete(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (svc *BaseService) forceDeleteList(query bson.M, args ...interface{}) (err error) {
+	return svc.col.Delete(query)
+}
+
+func (svc *BaseService) count(query bson.M) (total int, err error) {
+	if svc.col == nil {
+		return total, trace.TraceError(constants.ErrMissingCol)
+	}
+	return svc.col.Count(query)
+}
+
+func (svc *BaseService) update(query bson.M, update interface{}, fields []string, args ...interface{}) (err error) {
+	update, err = svc._getUpdateBsonM(update, fields)
+	if err != nil {
+		return err
+	}
+	return svc._update(query, update, svc._getUserFromArgs(args...))
+}
+
+func (svc *BaseService) updateId(id primitive.ObjectID, update interface{}, args ...interface{}) (err error) {
+	update, err = svc._getUpdateBsonM(update, nil)
+	if err != nil {
+		return err
+	}
+	return svc._updateById(id, update, svc._getUserFromArgs(args...))
+}
+
+func (svc *BaseService) insert(u interfaces.User, docs ...interface{}) (err error) {
+	// validate col
+	if svc.col == nil {
+		return trace.TraceError(constants.ErrMissingCol)
+	}
+
+	// iterate docs
+	for i, doc := range docs {
+		switch doc.(type) {
+		case map[string]interface{}:
+			// doc type: map[string]interface{}, need to handle _id
+			d := doc.(map[string]interface{})
+			vId, ok := d["_id"]
+			if !ok {
+				// _id not exists
+				d["_id"] = primitive.NewObjectID()
+			} else {
+				// _id exists
+				switch vId.(type) {
+				case string:
+					// _id type: string
+					sId, ok := vId.(string)
+					if ok {
+						d["_id"], err = primitive.ObjectIDFromHex(sId)
+						if err != nil {
+							return trace.TraceError(err)
+						}
+					}
+				case primitive.ObjectID:
+					// _id type: primitive.ObjectID
+					// do nothing
+				default:
+					return trace.TraceError(errors.ErrorModelInvalidType)
+				}
+			}
+		}
+		docs[i] = doc
+	}
+
+	// perform insert
+	ids, err := svc.col.InsertMany(docs)
+	if err != nil {
+		return err
+	}
+
+	// upsert artifacts
+	query := bson.M{
+		"_id": bson.M{
+			"$in": ids,
+		},
+	}
+	fr := svc.col.Find(query, nil)
+	list, err := NewListBinder(svc.id, fr).Bind()
+	for _, doc := range list.GetModels() {
+		// upsert artifact when performing model delegate save
+		if err := delegate.NewModelDelegate(doc, u).Save(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (svc *BaseService) _update(query bson.M, update interface{}, args ...interface{}) (err error) {
+	// ids of query
+	var ids []primitive.ObjectID
+	list, err := NewListBinder(svc.id, svc.find(query, nil)).Bind()
+	if err != nil {
+		return err
+	}
+	for _, doc := range list.GetModels() {
+		ids = append(ids, doc.GetId())
+	}
+
+	// update model objects
+	if err := svc.col.Update(query, update); err != nil {
+		return err
+	}
+
+	// update artifacts
+	u := svc._getUserFromArgs(args...)
+	return mongo.GetMongoCol(interfaces.ModelColNameArtifact).Update(query, svc._getUpdateArtifactUpdate(u))
+}
+
+func (svc *BaseService) _updateById(id primitive.ObjectID, update interface{}, args ...interface{}) (err error) {
+	// update model object
+	if err := svc.col.UpdateId(id, update); err != nil {
+		return err
+	}
+
+	// update artifact
+	u := svc._getUserFromArgs(args...)
+	return mongo.GetMongoCol(interfaces.ModelColNameArtifact).UpdateId(id, svc._getUpdateArtifactUpdate(u))
+}
+
+func (svc *BaseService) _getUpdateBsonM(update interface{}, fields []string) (res bson.M, err error) {
+	switch update.(type) {
+	case interfaces.Model:
+		// convert to bson.M
+		var updateBsonM bson.M
+		bytes, err := json.Marshal(&update)
+		if err != nil {
+			return nil, err
+		}
+		if err := json.Unmarshal(bytes, &updateBsonM); err != nil {
+			return nil, err
+		}
+		return svc._getUpdateBsonM(updateBsonM, fields)
+
+	case bson.M:
+		// convert to bson.M
+		updateBsonM := update.(bson.M)
+
+		// filter fields if not nil
+		if fields != nil {
+			// fields map
+			fieldsMap := map[string]bool{}
+			for _, f := range fields {
+				fieldsMap[f] = true
+			}
+
+			// remove unselected fields
+			for k := range updateBsonM {
+				if _, ok := fieldsMap[k]; !ok {
+					delete(updateBsonM, k)
+				}
+			}
+		}
+
+		// normalize update bson.M
+		if !svc._containsDollar(updateBsonM) {
+			if _, ok := updateBsonM["$set"]; !ok {
+				updateBsonM = bson.M{
+					"$set": updateBsonM,
+				}
+			}
+		}
+
+		return updateBsonM, nil
+	}
+
+	v := reflect.ValueOf(update)
+	switch v.Kind() {
+	case reflect.Struct:
+		if v.CanAddr() {
+			update = v.Addr().Interface()
+			return svc._getUpdateBsonM(update, fields)
+		}
+		return nil, errors.ErrorModelInvalidType
+	default:
+		return nil, errors.ErrorModelInvalidType
+	}
+}
+
+func (svc *BaseService) _getUpdateArtifactUpdate(u interfaces.User) (res bson.M) {
+	var uid primitive.ObjectID
+	if u != nil {
+		uid = u.GetId()
+	}
+	return bson.M{
+		"$set": bson.M{
+			"_sys.update_ts":  time.Now(),
+			"_sys.update_uid": uid,
+		},
+	}
+}
+
+func (svc *BaseService) _getUserFromArgs(args ...interface{}) (u interfaces.User) {
+	return utils.GetUserFromArgs(args...)
+}
+
+func (svc *BaseService) _containsDollar(updateBsonM bson.M) (ok bool) {
+	for k := range updateBsonM {
+		if strings.HasPrefix(k, "$") {
+			return true
+		}
+	}
+	return false
+}
+
+func NewBaseService(id interfaces.ModelId, opts ...BaseServiceOption) (svc2 interfaces.ModelBaseService) {
+	// service
+	svc := &BaseService{
+		id: id,
+	}
+
+	// apply options
+	for _, opt := range opts {
+		opt(svc)
+	}
+
+	// get collection name if not set
+	if svc.GetCol() == nil {
+		colName := models2.GetModelColName(id)
+		svc.SetCol(mongo.GetMongoCol(colName))
+	}
+
+	return svc
+}
+
+var store = sync.Map{}
+
+func GetBaseService(id interfaces.ModelId) (svc interfaces.ModelBaseService) {
+	res, ok := store.Load(id)
+	if ok {
+		svc, ok = res.(interfaces.ModelBaseService)
+		if ok {
+			return svc
+		}
+	}
+	svc = NewBaseService(id)
+	store.Store(id, svc)
+	return svc
+}
+
+func GetBaseServiceByColName(id interfaces.ModelId, colName string) (svc interfaces.ModelBaseService) {
+	res, ok := store.Load(colName)
+	if ok {
+		svc, ok = res.(interfaces.ModelBaseService)
+		if ok {
+			return svc
+		}
+	}
+	col := mongo.GetMongoCol(colName)
+	svc = NewBaseService(id, WithBaseServiceCol(col))
+	store.Store(colName, svc)
+	return svc
+}

core/models/service/base_service_v2.go

@@ -0,0 +1,181 @@
+package service
+
+import (
+	"fmt"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"reflect"
+	"sync"
+)
+
+var (
+	instanceMap    = make(map[string]any)
+	onceMap        = make(map[string]*sync.Once)
+	onceColNameMap = make(map[string]*sync.Once)
+	mu             sync.Mutex
+)
+
+type ModelServiceV2[T any] struct {
+	col *mongo.Col
+}
+
+func (svc *ModelServiceV2[T]) GetById(id primitive.ObjectID) (model *T, err error) {
+	var result T
+	err = svc.col.FindId(id).One(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (svc *ModelServiceV2[T]) GetOne(query bson.M, options *mongo.FindOptions) (model *T, err error) {
+	var result T
+	err = svc.col.Find(query, options).One(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (svc *ModelServiceV2[T]) GetMany(query bson.M, options *mongo.FindOptions) (models []T, err error) {
+	var result []T
+	err = svc.col.Find(query, options).All(&result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (svc *ModelServiceV2[T]) DeleteById(id primitive.ObjectID) (err error) {
+	return svc.col.DeleteId(id)
+}
+
+func (svc *ModelServiceV2[T]) DeleteOne(query bson.M) (err error) {
+	_, err = svc.col.GetCollection().DeleteOne(svc.col.GetContext(), query)
+	return err
+}
+
+func (svc *ModelServiceV2[T]) DeleteMany(query bson.M) (err error) {
+	_, err = svc.col.GetCollection().DeleteMany(svc.col.GetContext(), query, nil)
+	return err
+}
+
+func (svc *ModelServiceV2[T]) UpdateById(id primitive.ObjectID, update bson.M) (err error) {
+	return svc.col.UpdateId(id, update)
+}
+
+func (svc *ModelServiceV2[T]) UpdateOne(query bson.M, update bson.M) (err error) {
+	_, err = svc.col.GetCollection().UpdateOne(svc.col.GetContext(), query, update)
+	return err
+}
+
+func (svc *ModelServiceV2[T]) UpdateMany(query bson.M, update bson.M) (err error) {
+	_, err = svc.col.GetCollection().UpdateMany(svc.col.GetContext(), query, update)
+	return err
+}
+
+func (svc *ModelServiceV2[T]) ReplaceById(id primitive.ObjectID, model T) (err error) {
+	_, err = svc.col.GetCollection().ReplaceOne(svc.col.GetContext(), bson.M{"_id": id}, model)
+	return err
+}
+
+func (svc *ModelServiceV2[T]) ReplaceOne(query bson.M, model T) (err error) {
+	_, err = svc.col.GetCollection().ReplaceOne(svc.col.GetContext(), query, model)
+	return err
+}
+
+func (svc *ModelServiceV2[T]) InsertOne(model T) (id primitive.ObjectID, err error) {
+	m := any(&model).(interfaces.Model)
+	if m.GetId().IsZero() {
+		m.SetId(primitive.NewObjectID())
+	}
+	res, err := svc.col.GetCollection().InsertOne(svc.col.GetContext(), m)
+	if err != nil {
+		return primitive.NilObjectID, err
+	}
+	return res.InsertedID.(primitive.ObjectID), nil
+}
+
+func (svc *ModelServiceV2[T]) InsertMany(models []T) (ids []primitive.ObjectID, err error) {
+	var _models []any
+	for _, model := range models {
+		m := any(&model).(interfaces.Model)
+		if m.GetId().IsZero() {
+			m.SetId(primitive.NewObjectID())
+		}
+		_models = append(_models, m)
+	}
+	res, err := svc.col.GetCollection().InsertMany(svc.col.GetContext(), _models)
+	if err != nil {
+		return nil, err
+	}
+	for _, v := range res.InsertedIDs {
+		ids = append(ids, v.(primitive.ObjectID))
+	}
+	return ids, nil
+}
+
+func (svc *ModelServiceV2[T]) Count(query bson.M) (total int, err error) {
+	return svc.col.Count(query)
+}
+
+func (svc *ModelServiceV2[T]) GetCol() (col *mongo.Col) {
+	return svc.col
+}
+
+func GetCollectionNameByInstance(v any) string {
+	t := reflect.TypeOf(v)
+	field := t.Field(0)
+	return field.Tag.Get("collection")
+}
+
+func getCollectionName[T any]() string {
+	var instance T
+	t := reflect.TypeOf(instance)
+	field := t.Field(0)
+	return field.Tag.Get("collection")
+}
+
+// NewModelServiceV2 return singleton instance of ModelServiceV2
+func NewModelServiceV2[T any]() *ModelServiceV2[T] {
+	typeName := fmt.Sprintf("%T", *new(T))
+
+	mu.Lock()
+	defer mu.Unlock()
+
+	if _, exists := onceMap[typeName]; !exists {
+		onceMap[typeName] = &sync.Once{}
+	}
+
+	var instance *ModelServiceV2[T]
+
+	onceMap[typeName].Do(func() {
+		collectionName := getCollectionName[T]()
+		collection := mongo.GetMongoCol(collectionName)
+		instance = &ModelServiceV2[T]{col: collection}
+		instanceMap[typeName] = instance
+	})
+
+	return instanceMap[typeName].(*ModelServiceV2[T])
+}
+
+func NewModelServiceV2WithColName[T any](colName string) *ModelServiceV2[T] {
+	mu.Lock()
+	defer mu.Unlock()
+
+	if _, exists := onceColNameMap[colName]; !exists {
+		onceColNameMap[colName] = &sync.Once{}
+	}
+
+	var instance *ModelServiceV2[T]
+
+	onceColNameMap[colName].Do(func() {
+		collection := mongo.GetMongoCol(colName)
+		instance = &ModelServiceV2[T]{col: collection}
+		instanceMap[colName] = instance
+	})
+
+	return instanceMap[colName].(*ModelServiceV2[T])
+}

core/models/service/base_service_v2_test.go

@@ -0,0 +1,259 @@
+package service_test
+
+import (
+	"context"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"testing"
+	"time"
+)
+
+type TestModel struct {
+	Id                            primitive.ObjectID `bson:"_id,omitempty" collection:"testmodels"`
+	models.BaseModelV2[TestModel] `bson:",inline"`
+	Name                          string `bson:"name"`
+}
+
+func setupTestDB() {
+	viper.Set("mongo.db", "testdb")
+}
+
+func teardownTestDB() {
+	db := mongo.GetMongoDb("testdb")
+	err := db.Drop(context.Background())
+	if err != nil {
+		return
+	}
+}
+
+func TestModelServiceV2_GetById(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+
+	svc := service.NewModelServiceV2[TestModel]()
+	testModel := TestModel{Name: "Test Name"}
+
+	id, err := svc.InsertOne(testModel)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	result, err := svc.GetById(id)
+	require.Nil(t, err)
+	assert.Equal(t, testModel.Name, result.Name)
+}
+
+func TestModelServiceV2_GetOne(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+
+	svc := service.NewModelServiceV2[TestModel]()
+	testModel := TestModel{Name: "Test Name"}
+
+	_, err := svc.InsertOne(testModel)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	result, err := svc.GetOne(bson.M{"name": "Test Name"}, nil)
+	require.Nil(t, err)
+	assert.Equal(t, testModel.Name, result.Name)
+}
+
+func TestModelServiceV2_GetMany(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+
+	svc := service.NewModelServiceV2[TestModel]()
+	testModels := []TestModel{
+		{Name: "Name1"},
+		{Name: "Name2"},
+	}
+
+	_, err := svc.InsertMany(testModels)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	results, err := svc.GetMany(bson.M{}, nil)
+	require.Nil(t, err)
+	assert.Equal(t, 2, len(results))
+}
+
+func TestModelServiceV2_InsertOne(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+
+	svc := service.NewModelServiceV2[TestModel]()
+	testModel := TestModel{Name: "Test Name"}
+
+	id, err := svc.InsertOne(testModel)
+	require.Nil(t, err)
+	assert.NotEqual(t, primitive.NilObjectID, id)
+}
+
+func TestModelServiceV2_InsertMany(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+
+	svc := service.NewModelServiceV2[TestModel]()
+	testModels := []TestModel{
+		{Name: "Name1"},
+		{Name: "Name2"},
+	}
+
+	ids, err := svc.InsertMany(testModels)
+	require.Nil(t, err)
+	assert.Equal(t, 2, len(ids))
+}
+
+func TestModelServiceV2_UpdateById(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+
+	svc := service.NewModelServiceV2[TestModel]()
+	testModel := TestModel{Name: "Old Name"}
+
+	id, err := svc.InsertOne(testModel)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	update := bson.M{"$set": bson.M{"name": "New Name"}}
+	err = svc.UpdateById(id, update)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	result, err := svc.GetById(id)
+	require.Nil(t, err)
+	assert.Equal(t, "New Name", result.Name)
+}
+
+func TestModelServiceV2_UpdateOne(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+
+	svc := service.NewModelServiceV2[TestModel]()
+	testModel := TestModel{Name: "Old Name"}
+
+	_, err := svc.InsertOne(testModel)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	update := bson.M{"$set": bson.M{"name": "New Name"}}
+	err = svc.UpdateOne(bson.M{"name": "Old Name"}, update)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	result, err := svc.GetOne(bson.M{"name": "New Name"}, nil)
+	require.Nil(t, err)
+	assert.Equal(t, "New Name", result.Name)
+}
+
+func TestModelServiceV2_UpdateMany(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+
+	svc := service.NewModelServiceV2[TestModel]()
+	testModels := []TestModel{
+		{Name: "Old Name1"},
+		{Name: "Old Name2"},
+	}
+
+	_, err := svc.InsertMany(testModels)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	update := bson.M{"$set": bson.M{"name": "New Name"}}
+	err = svc.UpdateMany(bson.M{"name": bson.M{"$regex": "^Old"}}, update)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	results, err := svc.GetMany(bson.M{"name": "New Name"}, nil)
+	require.Nil(t, err)
+	assert.Equal(t, 2, len(results))
+}
+
+func TestModelServiceV2_DeleteById(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+
+	svc := service.NewModelServiceV2[TestModel]()
+	testModel := TestModel{Name: "Test Name"}
+
+	id, err := svc.InsertOne(testModel)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	err = svc.DeleteById(id)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	result, err := svc.GetById(id)
+	assert.NotNil(t, err)
+	assert.Nil(t, result)
+}
+
+func TestModelServiceV2_DeleteOne(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+
+	svc := service.NewModelServiceV2[TestModel]()
+	testModel := TestModel{Name: "Test Name"}
+
+	_, err := svc.InsertOne(testModel)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	err = svc.DeleteOne(bson.M{"name": "Test Name"})
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	result, err := svc.GetOne(bson.M{"name": "Test Name"}, nil)
+	assert.NotNil(t, err)
+	assert.Nil(t, result)
+}
+
+func TestModelServiceV2_DeleteMany(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+
+	svc := service.NewModelServiceV2[TestModel]()
+	testModels := []TestModel{
+		{Name: "Test Name1"},
+		{Name: "Test Name2"},
+	}
+
+	_, err := svc.InsertMany(testModels)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	err = svc.DeleteMany(bson.M{"name": bson.M{"$regex": "^Test Name"}})
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	results, err := svc.GetMany(bson.M{"name": bson.M{"$regex": "^Test Name"}}, nil)
+	require.Nil(t, err)
+	assert.Equal(t, 0, len(results))
+}
+
+func TestModelServiceV2_Count(t *testing.T) {
+	setupTestDB()
+	defer teardownTestDB()
+
+	svc := service.NewModelServiceV2[TestModel]()
+	testModels := []TestModel{
+		{Name: "Name1"},
+		{Name: "Name2"},
+	}
+
+	_, err := svc.InsertMany(testModels)
+	require.Nil(t, err)
+	time.Sleep(100 * time.Millisecond)
+
+	total, err := svc.Count(bson.M{})
+	require.Nil(t, err)
+	assert.Equal(t, 2, total)
+}

core/models/service/binder_basic.go

@@ -0,0 +1,92 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+)
+
+func NewBasicBinder(id interfaces.ModelId, fr *mongo.FindResult) (b interfaces.ModelBinder) {
+	return &BasicBinder{
+		id: id,
+		fr: fr,
+		m:  models.NewModelMap(),
+	}
+}
+
+type BasicBinder struct {
+	id interfaces.ModelId
+	fr *mongo.FindResult
+	m  *models.ModelMap
+}
+
+func (b *BasicBinder) Bind() (res interfaces.Model, err error) {
+	m := b.m
+
+	switch b.id {
+	case interfaces.ModelIdArtifact:
+		return b.Process(&m.Artifact)
+	case interfaces.ModelIdTag:
+		return b.Process(&m.Tag)
+	case interfaces.ModelIdNode:
+		return b.Process(&m.Node)
+	case interfaces.ModelIdProject:
+		return b.Process(&m.Project)
+	case interfaces.ModelIdSpider:
+		return b.Process(&m.Spider)
+	case interfaces.ModelIdTask:
+		return b.Process(&m.Task)
+	case interfaces.ModelIdJob:
+		return b.Process(&m.Job)
+	case interfaces.ModelIdSchedule:
+		return b.Process(&m.Schedule)
+	case interfaces.ModelIdUser:
+		return b.Process(&m.User)
+	case interfaces.ModelIdSetting:
+		return b.Process(&m.Setting)
+	case interfaces.ModelIdToken:
+		return b.Process(&m.Token)
+	case interfaces.ModelIdVariable:
+		return b.Process(&m.Variable)
+	case interfaces.ModelIdTaskQueue:
+		return b.Process(&m.TaskQueueItem)
+	case interfaces.ModelIdTaskStat:
+		return b.Process(&m.TaskStat)
+	case interfaces.ModelIdSpiderStat:
+		return b.Process(&m.SpiderStat)
+	case interfaces.ModelIdDataSource:
+		return b.Process(&m.DataSource)
+	case interfaces.ModelIdDataCollection:
+		return b.Process(&m.DataCollection)
+	case interfaces.ModelIdResult:
+		return b.Process(&m.Result)
+	case interfaces.ModelIdPassword:
+		return b.Process(&m.Password)
+	case interfaces.ModelIdExtraValue:
+		return b.Process(&m.ExtraValue)
+	case interfaces.ModelIdGit:
+		return b.Process(&m.Git)
+	case interfaces.ModelIdRole:
+		return b.Process(&m.Role)
+	case interfaces.ModelIdUserRole:
+		return b.Process(&m.UserRole)
+	case interfaces.ModelIdPermission:
+		return b.Process(&m.Permission)
+	case interfaces.ModelIdRolePermission:
+		return b.Process(&m.RolePermission)
+	case interfaces.ModelIdEnvironment:
+		return b.Process(&m.Environment)
+	case interfaces.ModelIdDependencySetting:
+		return b.Process(&m.DependencySetting)
+	default:
+		return nil, errors.ErrorModelInvalidModelId
+	}
+}
+
+func (b *BasicBinder) Process(d interfaces.Model) (res interfaces.Model, err error) {
+	if err := b.fr.One(d); err != nil {
+		return nil, err
+	}
+	return d, nil
+}

core/models/service/binder_list.go

@@ -0,0 +1,93 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/go-trace"
+)
+
+func NewListBinder(id interfaces.ModelId, fr *mongo.FindResult) (b interfaces.ModelListBinder) {
+	return &ListBinder{
+		id: id,
+		m:  models.NewModelListMap(),
+		fr: fr,
+		b:  NewBasicBinder(id, fr),
+	}
+}
+
+type ListBinder struct {
+	id interfaces.ModelId
+	m  *models.ModelListMap
+	fr *mongo.FindResult
+	b  interfaces.ModelBinder
+}
+
+func (b *ListBinder) Bind() (l interfaces.List, err error) {
+	m := b.m
+
+	switch b.id {
+	case interfaces.ModelIdArtifact:
+		return b.Process(&m.Artifacts)
+	case interfaces.ModelIdTag:
+		return b.Process(&m.Tags)
+	case interfaces.ModelIdNode:
+		return b.Process(&m.Nodes)
+	case interfaces.ModelIdProject:
+		return b.Process(&m.Projects)
+	case interfaces.ModelIdSpider:
+		return b.Process(&m.Spiders)
+	case interfaces.ModelIdTask:
+		return b.Process(&m.Tasks)
+	case interfaces.ModelIdSchedule:
+		return b.Process(&m.Schedules)
+	case interfaces.ModelIdUser:
+		return b.Process(&m.Users)
+	case interfaces.ModelIdSetting:
+		return b.Process(&m.Settings)
+	case interfaces.ModelIdToken:
+		return b.Process(&m.Tokens)
+	case interfaces.ModelIdVariable:
+		return b.Process(&m.Variables)
+	case interfaces.ModelIdTaskQueue:
+		return b.Process(&m.TaskQueueItems)
+	case interfaces.ModelIdTaskStat:
+		return b.Process(&m.TaskStats)
+	case interfaces.ModelIdSpiderStat:
+		return b.Process(&m.SpiderStats)
+	case interfaces.ModelIdDataSource:
+		return b.Process(&m.DataSources)
+	case interfaces.ModelIdDataCollection:
+		return b.Process(&m.DataCollections)
+	case interfaces.ModelIdResult:
+		return b.Process(&m.Results)
+	case interfaces.ModelIdPassword:
+		return b.Process(&m.Passwords)
+	case interfaces.ModelIdExtraValue:
+		return b.Process(&m.ExtraValues)
+	case interfaces.ModelIdGit:
+		return b.Process(&m.Gits)
+	case interfaces.ModelIdRole:
+		return b.Process(&m.Roles)
+	case interfaces.ModelIdUserRole:
+		return b.Process(&m.UserRoles)
+	case interfaces.ModelIdPermission:
+		return b.Process(&m.PermissionList)
+	case interfaces.ModelIdRolePermission:
+		return b.Process(&m.RolePermissionList)
+	case interfaces.ModelIdEnvironment:
+		return b.Process(&m.Environments)
+	case interfaces.ModelIdDependencySetting:
+		return b.Process(&m.DependencySettings)
+	default:
+		return l, errors.ErrorModelInvalidModelId
+	}
+}
+
+func (b *ListBinder) Process(d interface{}) (l interfaces.List, err error) {
+	if err := b.fr.All(d); err != nil {
+		return l, trace.TraceError(err)
+	}
+	return d.(interfaces.List), nil
+}

core/models/service/data_collection_service.go

@@ -0,0 +1,45 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func convertTypeDataCollection(d interface{}, err error) (res *models2.DataCollection, err2 error) {
+	if err != nil {
+		return nil, err
+	}
+	res, ok := d.(*models2.DataCollection)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return res, nil
+}
+
+func (svc *Service) GetDataCollectionById(id primitive.ObjectID) (res *models2.DataCollection, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdDataCollection).GetById(id)
+	return convertTypeDataCollection(d, err)
+}
+
+func (svc *Service) GetDataCollection(query bson.M, opts *mongo.FindOptions) (res *models2.DataCollection, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdDataCollection).Get(query, opts)
+	return convertTypeDataCollection(d, err)
+}
+
+func (svc *Service) GetDataCollectionList(query bson.M, opts *mongo.FindOptions) (res []models2.DataCollection, err error) {
+	l, err := svc.GetBaseService(interfaces.ModelIdDataCollection).GetList(query, opts)
+	for _, doc := range l.GetModels() {
+		d := doc.(*models2.DataCollection)
+		res = append(res, *d)
+	}
+	return res, nil
+}
+
+func (svc *Service) GetDataCollectionByName(name string, opts *mongo.FindOptions) (res *models2.DataCollection, err error) {
+	query := bson.M{"name": name}
+	return svc.GetDataCollection(query, opts)
+}

core/models/service/data_source_service.go

@@ -0,0 +1,40 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func convertTypeDataSource(d interface{}, err error) (res *models2.DataSource, err2 error) {
+	if err != nil {
+		return nil, err
+	}
+	res, ok := d.(*models2.DataSource)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return res, nil
+}
+
+func (svc *Service) GetDataSourceById(id primitive.ObjectID) (res *models2.DataSource, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdDataSource).GetById(id)
+	return convertTypeDataSource(d, err)
+}
+
+func (svc *Service) GetDataSource(query bson.M, opts *mongo.FindOptions) (res *models2.DataSource, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdDataSource).Get(query, opts)
+	return convertTypeDataSource(d, err)
+}
+
+func (svc *Service) GetDataSourceList(query bson.M, opts *mongo.FindOptions) (res []models2.DataSource, err error) {
+	l, err := svc.GetBaseService(interfaces.ModelIdDataSource).GetList(query, opts)
+	for _, doc := range l.GetModels() {
+		d := doc.(*models2.DataSource)
+		res = append(res, *d)
+	}
+	return res, nil
+}

core/models/service/dependency_setting_service.go

@@ -0,0 +1,40 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func convertTypeDependencySetting(d interface{}, err error) (res *models2.DependencySetting, err2 error) {
+	if err != nil {
+		return nil, err
+	}
+	res, ok := d.(*models2.DependencySetting)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return res, nil
+}
+
+func (svc *Service) GetDependencySettingById(id primitive.ObjectID) (res *models2.DependencySetting, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdDependencySetting).GetById(id)
+	return convertTypeDependencySetting(d, err)
+}
+
+func (svc *Service) GetDependencySetting(query bson.M, opts *mongo.FindOptions) (res *models2.DependencySetting, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdDependencySetting).Get(query, opts)
+	return convertTypeDependencySetting(d, err)
+}
+
+func (svc *Service) GetDependencySettingList(query bson.M, opts *mongo.FindOptions) (res []models2.DependencySetting, err error) {
+	l, err := svc.GetBaseService(interfaces.ModelIdDependencySetting).GetList(query, opts)
+	for _, doc := range l.GetModels() {
+		d := doc.(*models2.DependencySetting)
+		res = append(res, *d)
+	}
+	return res, nil
+}

core/models/service/environment_service.go

@@ -0,0 +1,40 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func convertTypeEnvironment(d interface{}, err error) (res *models2.Environment, err2 error) {
+	if err != nil {
+		return nil, err
+	}
+	res, ok := d.(*models2.Environment)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return res, nil
+}
+
+func (svc *Service) GetEnvironmentById(id primitive.ObjectID) (res *models2.Environment, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdEnvironment).GetById(id)
+	return convertTypeEnvironment(d, err)
+}
+
+func (svc *Service) GetEnvironment(query bson.M, opts *mongo.FindOptions) (res *models2.Environment, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdEnvironment).Get(query, opts)
+	return convertTypeEnvironment(d, err)
+}
+
+func (svc *Service) GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []models2.Environment, err error) {
+	l, err := svc.GetBaseService(interfaces.ModelIdEnvironment).GetList(query, opts)
+	for _, doc := range l.GetModels() {
+		d := doc.(*models2.Environment)
+		res = append(res, *d)
+	}
+	return res, nil
+}

core/models/service/extra_value_service.go

@@ -0,0 +1,45 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func convertTypeExtraValue(d interface{}, err error) (res *models.ExtraValue, err2 error) {
+	if err != nil {
+		return nil, err
+	}
+	res, ok := d.(*models.ExtraValue)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return res, nil
+}
+
+func (svc *Service) GetExtraValueById(id primitive.ObjectID) (res *models.ExtraValue, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdExtraValue).GetById(id)
+	return convertTypeExtraValue(d, err)
+}
+
+func (svc *Service) GetExtraValue(query bson.M, opts *mongo.FindOptions) (res *models.ExtraValue, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdExtraValue).Get(query, opts)
+	return convertTypeExtraValue(d, err)
+}
+
+func (svc *Service) GetExtraValueList(query bson.M, opts *mongo.FindOptions) (res []models.ExtraValue, err error) {
+	l, err := svc.GetBaseService(interfaces.ModelIdExtraValue).GetList(query, opts)
+	for _, doc := range l.GetModels() {
+		d := doc.(*models.ExtraValue)
+		res = append(res, *d)
+	}
+	return res, nil
+}
+
+func (svc *Service) GetExtraValueByObjectIdModel(oid primitive.ObjectID, m string, opts *mongo.FindOptions) (res *models.ExtraValue, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdExtraValue).Get(bson.M{"oid": oid, "m": m}, opts)
+	return convertTypeExtraValue(d, err)
+}

core/models/service/git_service.go

@@ -0,0 +1,43 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func convertTypeGit(d interface{}, err error) (res *models2.Git, err2 error) {
+	if err != nil {
+		return nil, err
+	}
+	res, ok := d.(*models2.Git)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return res, nil
+}
+
+func (svc *Service) GetGitById(id primitive.ObjectID) (res *models2.Git, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdGit).GetById(id)
+	return convertTypeGit(d, err)
+}
+
+func (svc *Service) GetGit(query bson.M, opts *mongo.FindOptions) (res *models2.Git, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdGit).Get(query, opts)
+	return convertTypeGit(d, err)
+}
+
+func (svc *Service) GetGitList(query bson.M, opts *mongo.FindOptions) (res []models2.Git, err error) {
+	l, err := svc.GetBaseService(interfaces.ModelIdGit).GetList(query, opts)
+	if l == nil {
+		return nil, nil
+	}
+	for _, doc := range l.GetModels() {
+		d := doc.(*models2.Git)
+		res = append(res, *d)
+	}
+	return res, nil
+}

core/models/service/interface.go

@@ -0,0 +1,105 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type ModelService interface {
+	interfaces.ModelService
+	DropAll() (err error)
+	GetNodeById(id primitive.ObjectID) (res *models.Node, err error)
+	GetNode(query bson.M, opts *mongo.FindOptions) (res *models.Node, err error)
+	GetNodeList(query bson.M, opts *mongo.FindOptions) (res []models.Node, err error)
+	GetNodeByKey(key string, opts *mongo.FindOptions) (res *models.Node, err error)
+	GetProjectById(id primitive.ObjectID) (res *models.Project, err error)
+	GetProject(query bson.M, opts *mongo.FindOptions) (res *models.Project, err error)
+	GetProjectList(query bson.M, opts *mongo.FindOptions) (res []models.Project, err error)
+	GetArtifactById(id primitive.ObjectID) (res *models.Artifact, err error)
+	GetArtifact(query bson.M, opts *mongo.FindOptions) (res *models.Artifact, err error)
+	GetArtifactList(query bson.M, opts *mongo.FindOptions) (res []models.Artifact, err error)
+	GetTagById(id primitive.ObjectID) (res *models.Tag, err error)
+	GetTag(query bson.M, opts *mongo.FindOptions) (res *models.Tag, err error)
+	GetTagList(query bson.M, opts *mongo.FindOptions) (res []models.Tag, err error)
+	GetTagIds(colName string, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error)
+	UpdateTagsById(colName string, id primitive.ObjectID, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error)
+	UpdateTags(colName string, query bson.M, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error)
+	GetJobById(id primitive.ObjectID) (res *models.Job, err error)
+	GetJob(query bson.M, opts *mongo.FindOptions) (res *models.Job, err error)
+	GetJobList(query bson.M, opts *mongo.FindOptions) (res []models.Job, err error)
+	GetScheduleById(id primitive.ObjectID) (res *models.Schedule, err error)
+	GetSchedule(query bson.M, opts *mongo.FindOptions) (res *models.Schedule, err error)
+	GetScheduleList(query bson.M, opts *mongo.FindOptions) (res []models.Schedule, err error)
+	GetUserById(id primitive.ObjectID) (res *models.User, err error)
+	GetUser(query bson.M, opts *mongo.FindOptions) (res *models.User, err error)
+	GetUserList(query bson.M, opts *mongo.FindOptions) (res []models.User, err error)
+	GetUserByUsername(username string, opts *mongo.FindOptions) (res *models.User, err error)
+	GetUserByUsernameWithPassword(username string, opts *mongo.FindOptions) (res *models.User, err error)
+	GetSettingById(id primitive.ObjectID) (res *models.Setting, err error)
+	GetSetting(query bson.M, opts *mongo.FindOptions) (res *models.Setting, err error)
+	GetSettingList(query bson.M, opts *mongo.FindOptions) (res []models.Setting, err error)
+	GetSettingByKey(key string, opts *mongo.FindOptions) (res *models.Setting, err error)
+	GetSpiderById(id primitive.ObjectID) (res *models.Spider, err error)
+	GetSpider(query bson.M, opts *mongo.FindOptions) (res *models.Spider, err error)
+	GetSpiderList(query bson.M, opts *mongo.FindOptions) (res []models.Spider, err error)
+	GetTaskById(id primitive.ObjectID) (res *models.Task, err error)
+	GetTask(query bson.M, opts *mongo.FindOptions) (res *models.Task, err error)
+	GetTaskList(query bson.M, opts *mongo.FindOptions) (res []models.Task, err error)
+	GetTokenById(id primitive.ObjectID) (res *models.Token, err error)
+	GetToken(query bson.M, opts *mongo.FindOptions) (res *models.Token, err error)
+	GetTokenList(query bson.M, opts *mongo.FindOptions) (res []models.Token, err error)
+	GetVariableById(id primitive.ObjectID) (res *models.Variable, err error)
+	GetVariable(query bson.M, opts *mongo.FindOptions) (res *models.Variable, err error)
+	GetVariableList(query bson.M, opts *mongo.FindOptions) (res []models.Variable, err error)
+	GetVariableByKey(key string, opts *mongo.FindOptions) (res *models.Variable, err error)
+	GetTaskQueueItemById(id primitive.ObjectID) (res *models.TaskQueueItem, err error)
+	GetTaskQueueItem(query bson.M, opts *mongo.FindOptions) (res *models.TaskQueueItem, err error)
+	GetTaskQueueItemList(query bson.M, opts *mongo.FindOptions) (res []models.TaskQueueItem, err error)
+	GetTaskStatById(id primitive.ObjectID) (res *models.TaskStat, err error)
+	GetTaskStat(query bson.M, opts *mongo.FindOptions) (res *models.TaskStat, err error)
+	GetTaskStatList(query bson.M, opts *mongo.FindOptions) (res []models.TaskStat, err error)
+	GetSpiderStatById(id primitive.ObjectID) (res *models.SpiderStat, err error)
+	GetSpiderStat(query bson.M, opts *mongo.FindOptions) (res *models.SpiderStat, err error)
+	GetSpiderStatList(query bson.M, opts *mongo.FindOptions) (res []models.SpiderStat, err error)
+	GetDataSourceById(id primitive.ObjectID) (res *models.DataSource, err error)
+	GetDataSource(query bson.M, opts *mongo.FindOptions) (res *models.DataSource, err error)
+	GetDataSourceList(query bson.M, opts *mongo.FindOptions) (res []models.DataSource, err error)
+	GetDataCollectionById(id primitive.ObjectID) (res *models.DataCollection, err error)
+	GetDataCollection(query bson.M, opts *mongo.FindOptions) (res *models.DataCollection, err error)
+	GetDataCollectionList(query bson.M, opts *mongo.FindOptions) (res []models.DataCollection, err error)
+	GetDataCollectionByName(name string, opts *mongo.FindOptions) (res *models.DataCollection, err error)
+	GetPasswordById(id primitive.ObjectID) (res *models.Password, err error)
+	GetPassword(query bson.M, opts *mongo.FindOptions) (res *models.Password, err error)
+	GetPasswordList(query bson.M, opts *mongo.FindOptions) (res []models.Password, err error)
+	GetExtraValueById(id primitive.ObjectID) (res *models.ExtraValue, err error)
+	GetExtraValue(query bson.M, opts *mongo.FindOptions) (res *models.ExtraValue, err error)
+	GetExtraValueList(query bson.M, opts *mongo.FindOptions) (res []models.ExtraValue, err error)
+	GetExtraValueByObjectIdModel(oid primitive.ObjectID, m string, opts *mongo.FindOptions) (res *models.ExtraValue, err error)
+	GetGitById(id primitive.ObjectID) (res *models.Git, err error)
+	GetGit(query bson.M, opts *mongo.FindOptions) (res *models.Git, err error)
+	GetGitList(query bson.M, opts *mongo.FindOptions) (res []models.Git, err error)
+	GetRoleById(id primitive.ObjectID) (res *models.Role, err error)
+	GetRole(query bson.M, opts *mongo.FindOptions) (res *models.Role, err error)
+	GetRoleList(query bson.M, opts *mongo.FindOptions) (res []models.Role, err error)
+	GetRoleByName(name string, opts *mongo.FindOptions) (res *models.Role, err error)
+	GetRoleByKey(key string, opts *mongo.FindOptions) (res *models.Role, err error)
+	GetUserRoleById(id primitive.ObjectID) (res *models.UserRole, err error)
+	GetUserRole(query bson.M, opts *mongo.FindOptions) (res *models.UserRole, err error)
+	GetUserRoleList(query bson.M, opts *mongo.FindOptions) (res []models.UserRole, err error)
+	GetUserRoleListByUserId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.UserRole, err error)
+	GetUserRoleListByRoleId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.UserRole, err error)
+	GetPermissionById(id primitive.ObjectID) (res *models.Permission, err error)
+	GetPermission(query bson.M, opts *mongo.FindOptions) (res *models.Permission, err error)
+	GetPermissionList(query bson.M, opts *mongo.FindOptions) (res []models.Permission, err error)
+	GetPermissionByKey(key string, opts *mongo.FindOptions) (res *models.Permission, err error)
+	GetRolePermission(query bson.M, opts *mongo.FindOptions) (res *models.RolePermission, err error)
+	GetRolePermissionList(query bson.M, opts *mongo.FindOptions) (res []models.RolePermission, err error)
+	GetRolePermissionListByRoleId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.RolePermission, err error)
+	GetRolePermissionListByPermissionId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.RolePermission, err error)
+	GetEnvironmentById(id primitive.ObjectID) (res *models.Environment, err error)
+	GetEnvironment(query bson.M, opts *mongo.FindOptions) (res *models.Environment, err error)
+	GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []models.Environment, err error)
+}

core/models/service/job_service.go

@@ -0,0 +1,40 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func convertTypeJob(d interface{}, err error) (res *models2.Job, err2 error) {
+	if err != nil {
+		return nil, err
+	}
+	res, ok := d.(*models2.Job)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return res, nil
+}
+
+func (svc *Service) GetJobById(id primitive.ObjectID) (res *models2.Job, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdJob).GetById(id)
+	return convertTypeJob(d, err)
+}
+
+func (svc *Service) GetJob(query bson.M, opts *mongo.FindOptions) (res *models2.Job, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdJob).Get(query, opts)
+	return convertTypeJob(d, err)
+}
+
+func (svc *Service) GetJobList(query bson.M, opts *mongo.FindOptions) (res []models2.Job, err error) {
+	l, err := svc.GetBaseService(interfaces.ModelIdJob).GetList(query, opts)
+	for _, doc := range l.GetModels() {
+		d := doc.(*models2.Job)
+		res = append(res, *d)
+	}
+	return res, nil
+}

core/models/service/node_service.go

@@ -0,0 +1,45 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func convertTypeNode(d interface{}, err error) (res *models2.Node, err2 error) {
+	if err != nil {
+		return nil, err
+	}
+	res, ok := d.(*models2.Node)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return res, nil
+}
+
+func (svc *Service) GetNodeById(id primitive.ObjectID) (res *models2.Node, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdNode).GetById(id)
+	return convertTypeNode(d, err)
+}
+
+func (svc *Service) GetNode(query bson.M, opts *mongo.FindOptions) (res *models2.Node, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdNode).Get(query, opts)
+	return convertTypeNode(d, err)
+}
+
+func (svc *Service) GetNodeList(query bson.M, opts *mongo.FindOptions) (res []models2.Node, err error) {
+	l, err := svc.GetBaseService(interfaces.ModelIdNode).GetList(query, opts)
+	for _, doc := range l.GetModels() {
+		d := doc.(*models2.Node)
+		res = append(res, *d)
+	}
+	return res, nil
+}
+
+func (svc *Service) GetNodeByKey(key string, opts *mongo.FindOptions) (res *models2.Node, err error) {
+	query := bson.M{"key": key}
+	return svc.GetNode(query, opts)
+}

core/models/service/options.go

@@ -0,0 +1,25 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+)
+
+type Option func(ModelService)
+
+type BaseServiceOption func(svc interfaces.ModelBaseService)
+
+func WithBaseServiceModelId(id interfaces.ModelId) BaseServiceOption {
+	return func(svc interfaces.ModelBaseService) {
+		svc.SetModelId(id)
+	}
+}
+
+func WithBaseServiceCol(col *mongo.Col) BaseServiceOption {
+	return func(svc interfaces.ModelBaseService) {
+		_svc, ok := svc.(*BaseService)
+		if ok {
+			_svc.SetCol(col)
+		}
+	}
+}

core/models/service/password_service.go

@@ -0,0 +1,40 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func convertTypePassword(d interface{}, err error) (res *models2.Password, err2 error) {
+	if err != nil {
+		return nil, err
+	}
+	res, ok := d.(*models2.Password)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return res, nil
+}
+
+func (svc *Service) GetPasswordById(id primitive.ObjectID) (res *models2.Password, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdPassword).GetById(id)
+	return convertTypePassword(d, err)
+}
+
+func (svc *Service) GetPassword(query bson.M, opts *mongo.FindOptions) (res *models2.Password, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdPassword).Get(query, opts)
+	return convertTypePassword(d, err)
+}
+
+func (svc *Service) GetPasswordList(query bson.M, opts *mongo.FindOptions) (res []models2.Password, err error) {
+	l, err := svc.GetBaseService(interfaces.ModelIdPassword).GetList(query, opts)
+	for _, doc := range l.GetModels() {
+		d := doc.(*models2.Password)
+		res = append(res, *d)
+	}
+	return res, nil
+}

core/models/service/permission_service.go

@@ -0,0 +1,48 @@
+package service
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	models2 "github.com/crawlab-team/crawlab/core/models/models"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func convertTypePermission(d interface{}, err error) (res *models2.Permission, err2 error) {
+	if err != nil {
+		return nil, err
+	}
+	res, ok := d.(*models2.Permission)
+	if !ok {
+		return nil, errors.ErrorModelInvalidType
+	}
+	return res, nil
+}
+
+func (svc *Service) GetPermissionById(id primitive.ObjectID) (res *models2.Permission, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdPermission).GetById(id)
+	return convertTypePermission(d, err)
+}
+
+func (svc *Service) GetPermission(query bson.M, opts *mongo.FindOptions) (res *models2.Permission, err error) {
+	d, err := svc.GetBaseService(interfaces.ModelIdPermission).Get(query, opts)
+	return convertTypePermission(d, err)
+}
+
+func (svc *Service) GetPermissionList(query bson.M, opts *mongo.FindOptions) (res []models2.Permission, err error) {
+	l, err := svc.GetBaseService(interfaces.ModelIdPermission).GetList(query, opts)
+	if err != nil {
+		return nil, err
+	}
+	for _, doc := range l.GetModels() {
+		d := doc.(*models2.Permission)
+		res = append(res, *d)
+	}
+	return res, nil
+}
+
+func (svc *Service) GetPermissionByKey(key string, opts *mongo.FindOptions) (res *models2.Permission, err error) {
+	query := bson.M{"key": key}
+	return svc.GetPermission(query, opts)
+}