feat: added modules

core/interfaces/address.go

@@ -0,0 +1,7 @@
+package interfaces
+
+type Address interface {
+	Entity
+	String() string
+	IsEmpty() bool
+}

core/interfaces/color.go

@@ -0,0 +1,7 @@
+package interfaces
+
+type Color interface {
+	Entity
+	GetHex() string
+	GetName() string
+}

core/interfaces/color_service.go

@@ -0,0 +1,7 @@
+package interfaces
+
+type ColorService interface {
+	Injectable
+	GetByName(name string) (res Color, err error)
+	GetRandom() (res Color, err error)
+}

core/interfaces/controller_params.go

@@ -0,0 +1,6 @@
+package interfaces
+
+type ControllerParams interface {
+	IsZero() (ok bool)
+	IsDefault() (ok bool)
+}

core/interfaces/data_source_service.go

@@ -0,0 +1,14 @@
+package interfaces
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type DataSourceService interface {
+	ChangePassword(id primitive.ObjectID, password string) (err error)
+	Monitor()
+	CheckStatus(id primitive.ObjectID) (err error)
+	SetTimeout(duration time.Duration)
+	SetMonitorInterval(duration time.Duration)
+}

core/interfaces/entity.go

@@ -0,0 +1,5 @@
+package interfaces
+
+type Entity interface {
+	Value() interface{}
+}

core/interfaces/event_data.go

@@ -0,0 +1,6 @@
+package interfaces
+
+type EventData interface {
+	GetEvent() string
+	GetData() interface{}
+}

core/interfaces/event_service.go

@@ -0,0 +1,9 @@
+package interfaces
+
+type EventFn func(data ...interface{}) (err error)
+
+type EventService interface {
+	Register(key, include, exclude string, ch *chan EventData)
+	Unregister(key string)
+	SendEvent(eventName string, data ...interface{})
+}

core/interfaces/export.go

@@ -0,0 +1,14 @@
+package interfaces
+
+import "time"
+
+type Export interface {
+	GetId() string
+	GetType() string
+	GetTarget() string
+	GetFilter() Filter
+	GetStatus() string
+	GetStartTs() time.Time
+	GetEndTs() time.Time
+	GetDownloadPath() string
+}

core/interfaces/export_service.go

@@ -0,0 +1,7 @@
+package interfaces
+
+type ExportService interface {
+	GenerateId() (exportId string, err error)
+	Export(exportType, target string, filter Filter) (exportId string, err error)
+	GetExport(exportId string) (export Export, err error)
+}

core/interfaces/filter.go

@@ -0,0 +1,9 @@
+package interfaces
+
+type Filter interface {
+	GetIsOr() (isOr bool)
+	SetIsOr(isOr bool)
+	GetConditions() (conditions []FilterCondition)
+	SetConditions(conditions []FilterCondition)
+	IsNil() (ok bool)
+}

core/interfaces/filter_condition.go

@@ -0,0 +1,10 @@
+package interfaces
+
+type FilterCondition interface {
+	GetKey() (key string)
+	SetKey(key string)
+	GetOp() (op string)
+	SetOp(op string)
+	GetValue() (value interface{})
+	SetValue(value interface{})
+}

core/interfaces/fs_file_info.go

@@ -0,0 +1,19 @@
+package interfaces
+
+import (
+	"os"
+	"time"
+)
+
+type FsFileInfo interface {
+	GetName() string
+	GetPath() string
+	GetFullPath() string
+	GetExtension() string
+	GetIsDir() bool
+	GetFileSize() int64
+	GetModTime() time.Time
+	GetMode() os.FileMode
+	GetHash() string
+	GetChildren() []FsFileInfo
+}

core/interfaces/fs_service.go

@@ -0,0 +1,28 @@
+package interfaces
+
+import (
+	cfs "github.com/crawlab-team/crawlab-fs"
+	vcs "github.com/crawlab-team/crawlab-vcs"
+)
+
+type FsService interface {
+	WithConfigPath
+	List(path string, opts ...ServiceCrudOption) (files []FsFileInfo, err error)
+	GetFile(path string, opts ...ServiceCrudOption) (data []byte, err error)
+	GetFileInfo(path string, opts ...ServiceCrudOption) (file FsFileInfo, err error)
+	Save(path string, data []byte, opts ...ServiceCrudOption) (err error)
+	Rename(path, newPath string, opts ...ServiceCrudOption) (err error)
+	Delete(path string, opts ...ServiceCrudOption) (err error)
+	Copy(path, newPath string, opts ...ServiceCrudOption) (err error)
+	Commit(msg string) (err error)
+	SyncToFs(opts ...ServiceCrudOption) (err error)
+	SyncToWorkspace() (err error)
+	GetFsPath() (path string)
+	SetFsPath(path string)
+	GetWorkspacePath() (path string)
+	SetWorkspacePath(path string)
+	GetRepoPath() (path string)
+	SetRepoPath(path string)
+	GetFs() (fs cfs.Manager)
+	GetGitClient() (c *vcs.GitClient)
+}

core/interfaces/fs_service_options.go

@@ -0,0 +1,21 @@
+package interfaces
+
+type ServiceCrudOptions struct {
+	IsAbsolute         bool // whether the path is absolute
+	OnlyFromWorkspace  bool // whether only sync from workspace
+	NotSyncToWorkspace bool // whether not sync to workspace
+}
+
+type ServiceCrudOption func(o *ServiceCrudOptions)
+
+func WithOnlyFromWorkspace() ServiceCrudOption {
+	return func(o *ServiceCrudOptions) {
+		o.OnlyFromWorkspace = true
+	}
+}
+
+func WithNotSyncToWorkspace() ServiceCrudOption {
+	return func(o *ServiceCrudOptions) {
+		o.NotSyncToWorkspace = true
+	}
+}

core/interfaces/fs_service_v2.go

@@ -0,0 +1,12 @@
+package interfaces
+
+type FsServiceV2 interface {
+	List(path string) (files []FsFileInfo, err error)
+	GetFile(path string) (data []byte, err error)
+	GetFileInfo(path string) (file FsFileInfo, err error)
+	Save(path string, data []byte) (err error)
+	CreateDir(path string) (err error)
+	Rename(path, newPath string) (err error)
+	Delete(path string) (err error)
+	Copy(path, newPath string) (err error)
+}

core/interfaces/grpc_base.go

@@ -0,0 +1,9 @@
+package interfaces
+
+type GrpcBase interface {
+	WithConfigPath
+	Init() (err error)
+	Start() (err error)
+	Stop() (err error)
+	Register() (err error)
+}

core/interfaces/grpc_base_service_params.go

@@ -0,0 +1,5 @@
+package interfaces
+
+type GrpcBaseServiceParams interface {
+	Entity
+}

core/interfaces/grpc_client.go

@@ -0,0 +1,30 @@
+package interfaces
+
+import (
+	"context"
+	grpc "github.com/crawlab-team/crawlab/grpc"
+	"time"
+)
+
+type GrpcClient interface {
+	GrpcBase
+	WithConfigPath
+	GetModelDelegateClient() grpc.ModelDelegateClient
+	GetModelBaseServiceClient() grpc.ModelBaseServiceClient
+	GetNodeClient() grpc.NodeServiceClient
+	GetTaskClient() grpc.TaskServiceClient
+	GetMessageClient() grpc.MessageServiceClient
+	SetAddress(Address)
+	SetTimeout(time.Duration)
+	SetSubscribeType(string)
+	SetHandleMessage(bool)
+	Context() (context.Context, context.CancelFunc)
+	NewRequest(interface{}) *grpc.Request
+	GetMessageChannel() chan *grpc.StreamMessage
+	Restart() error
+	NewModelBaseServiceRequest(ModelId, GrpcBaseServiceParams) (*grpc.Request, error)
+	IsStarted() bool
+	IsClosed() bool
+	Err() error
+	GetStream() grpc.NodeService_SubscribeClient
+}

core/interfaces/grpc_client_model_base_service.go

@@ -0,0 +1,7 @@
+package interfaces
+
+type GrpcClientModelBaseService interface {
+	WithModelId
+	WithConfigPath
+	ModelBaseService
+}

core/interfaces/grpc_client_model_delegate.go

@@ -0,0 +1,7 @@
+package interfaces
+
+type GrpcClientModelDelegate interface {
+	ModelDelegate
+	WithConfigPath
+	Close() error
+}

core/interfaces/grpc_client_model_environment_service.go

@@ -0,0 +1,14 @@
+package interfaces
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type GrpcClientModelEnvironmentService interface {
+	ModelBaseService
+	GetEnvironmentById(id primitive.ObjectID) (s Environment, err error)
+	GetEnvironment(query bson.M, opts *mongo.FindOptions) (s Environment, err error)
+	GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []Environment, err error)
+}

core/interfaces/grpc_client_model_node_service.go

@@ -0,0 +1,15 @@
+package interfaces
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type GrpcClientModelNodeService interface {
+	ModelBaseService
+	GetNodeById(id primitive.ObjectID) (n Node, err error)
+	GetNode(query bson.M, opts *mongo.FindOptions) (n Node, err error)
+	GetNodeByKey(key string) (n Node, err error)
+	GetNodeList(query bson.M, opts *mongo.FindOptions) (res []Node, err error)
+}

core/interfaces/grpc_client_model_service.go

@@ -0,0 +1,6 @@
+package interfaces
+
+type GrpcClientModelService interface {
+	WithConfigPath
+	NewBaseServiceDelegate(id ModelId) (GrpcClientModelBaseService, error)
+}

core/interfaces/grpc_client_model_spider_service.go

@@ -0,0 +1,14 @@
+package interfaces
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type GrpcClientModelSpiderService interface {
+	ModelBaseService
+	GetSpiderById(id primitive.ObjectID) (s Spider, err error)
+	GetSpider(query bson.M, opts *mongo.FindOptions) (s Spider, err error)
+	GetSpiderList(query bson.M, opts *mongo.FindOptions) (res []Spider, err error)
+}

core/interfaces/grpc_client_model_task_service.go

@@ -0,0 +1,14 @@
+package interfaces
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type GrpcClientModelTaskService interface {
+	ModelBaseService
+	GetTaskById(id primitive.ObjectID) (s Task, err error)
+	GetTask(query bson.M, opts *mongo.FindOptions) (s Task, err error)
+	GetTaskList(query bson.M, opts *mongo.FindOptions) (res []Task, err error)
+}

core/interfaces/grpc_client_model_task_stat_service.go

@@ -0,0 +1,14 @@
+package interfaces
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type GrpcClientModelTaskStatService interface {
+	ModelBaseService
+	GetTaskStatById(id primitive.ObjectID) (s TaskStat, err error)
+	GetTaskStat(query bson.M, opts *mongo.FindOptions) (s TaskStat, err error)
+	GetTaskStatList(query bson.M, opts *mongo.FindOptions) (res []TaskStat, err error)
+}

core/interfaces/grpc_client_pool.go

@@ -0,0 +1,9 @@
+package interfaces
+
+type GrpcClientPool interface {
+	WithConfigPath
+	Init() error
+	NewClient() error
+	GetClient() (GrpcClient, error)
+	SetSize(int)
+}

core/interfaces/grpc_model_base_service_message.go

@@ -0,0 +1,7 @@
+package interfaces
+
+type GrpcModelBaseServiceMessage interface {
+	GetModelId() ModelId
+	GetData() []byte
+	ToBytes() (data []byte)
+}

core/interfaces/grpc_model_binder.go

@@ -0,0 +1,5 @@
+package interfaces
+
+type GrpcModelBinder interface {
+	ModelBinder
+}

core/interfaces/grpc_model_delegate_message.go

@@ -0,0 +1,8 @@
+package interfaces
+
+type GrpcModelDelegateMessage interface {
+	GetModelId() ModelId
+	GetMethod() ModelDelegateMethod
+	GetData() []byte
+	ToBytes() (data []byte)
+}

core/interfaces/grpc_model_list_binder.go

@@ -0,0 +1,5 @@
+package interfaces
+
+type GrpcModelListBinder interface {
+	ModelListBinder
+}

core/interfaces/grpc_server.go

@@ -0,0 +1,16 @@
+package interfaces
+
+import (
+	grpc "github.com/crawlab-team/crawlab/grpc"
+)
+
+type GrpcServer interface {
+	GrpcBase
+	SetAddress(Address)
+	GetSubscribe(key string) (sub GrpcSubscribe, err error)
+	SetSubscribe(key string, sub GrpcSubscribe)
+	DeleteSubscribe(key string)
+	SendStreamMessage(key string, code grpc.StreamMessageCode) (err error)
+	SendStreamMessageWithData(nodeKey string, code grpc.StreamMessageCode, d interface{}) (err error)
+	IsStopped() (res bool)
+}

core/interfaces/grpc_stream.go

@@ -0,0 +1,12 @@
+package interfaces
+
+import grpc "github.com/crawlab-team/crawlab/grpc"
+
+type GrpcStream interface {
+	Send(msg *grpc.StreamMessage) (err error)
+}
+
+type GrpcStreamBidirectional interface {
+	GrpcStream
+	Recv() (msg *grpc.StreamMessage, err error)
+}

core/interfaces/grpc_subscribe.go

@@ -0,0 +1,7 @@
+package interfaces
+
+type GrpcSubscribe interface {
+	GetStream() GrpcStream
+	GetStreamBidirectional() GrpcStreamBidirectional
+	GetFinished() chan bool
+}

core/interfaces/i18n_service.go

@@ -0,0 +1,6 @@
+package interfaces
+
+type I18nService interface {
+	AddTranslations(t []Translation)
+	GetTranslations() (t []Translation)
+}

core/interfaces/injectable.go

@@ -0,0 +1,5 @@
+package interfaces
+
+type Injectable interface {
+	Inject() error
+}

core/interfaces/list.go

@@ -0,0 +1,5 @@
+package interfaces
+
+type List interface {
+	GetModels() (res []Model)
+}

core/interfaces/model.go

@@ -0,0 +1,98 @@
+package interfaces
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Model interface {
+	GetId() (id primitive.ObjectID)
+	SetId(id primitive.ObjectID)
+}
+
+type ModelV2 interface {
+	GetId() (id primitive.ObjectID)
+	SetId(id primitive.ObjectID)
+	SetCreated(by primitive.ObjectID)
+	SetUpdated(by primitive.ObjectID)
+}
+
+type ModelId int
+
+const (
+	ModelIdArtifact = iota
+	ModelIdTag
+	ModelIdNode
+	ModelIdProject
+	ModelIdSpider
+	ModelIdTask
+	ModelIdJob
+	ModelIdSchedule
+	ModelIdUser
+	ModelIdSetting
+	ModelIdToken
+	ModelIdVariable
+	ModelIdTaskQueue
+	ModelIdTaskStat
+	ModelIdSpiderStat
+	ModelIdDataSource
+	ModelIdDataCollection
+	ModelIdResult
+	ModelIdPassword
+	ModelIdExtraValue
+	ModelIdGit
+	ModelIdRole
+	ModelIdUserRole
+	ModelIdPermission
+	ModelIdRolePermission
+	ModelIdEnvironment
+	ModelIdDependencySetting
+)
+
+const (
+	ModelColNameArtifact          = "artifacts"
+	ModelColNameTag               = "tags"
+	ModelColNameNode              = "nodes"
+	ModelColNameProject           = "projects"
+	ModelColNameSpider            = "spiders"
+	ModelColNameTask              = "tasks"
+	ModelColNameJob               = "jobs"
+	ModelColNameSchedule          = "schedules"
+	ModelColNameUser              = "users"
+	ModelColNameSetting           = "settings"
+	ModelColNameToken             = "tokens"
+	ModelColNameVariable          = "variables"
+	ModelColNameTaskQueue         = "task_queue"
+	ModelColNameTaskStat          = "task_stats"
+	ModelColNameSpiderStat        = "spider_stats"
+	ModelColNameDataSource        = "data_sources"
+	ModelColNameDataCollection    = "data_collections"
+	ModelColNamePasswords         = "passwords"
+	ModelColNameExtraValues       = "extra_values"
+	ModelColNameGit               = "gits"
+	ModelColNameRole              = "roles"
+	ModelColNameUserRole          = "user_roles"
+	ModelColNamePermission        = "permissions"
+	ModelColNameRolePermission    = "role_permissions"
+	ModelColNameEnvironment       = "environments"
+	ModelColNameDependencySetting = "dependency_settings"
+)
+
+type ModelWithTags interface {
+	Model
+	SetTags(tags []Tag)
+	GetTags() (tags []Tag)
+}
+
+type ModelWithNameDescription interface {
+	Model
+	GetName() (name string)
+	SetName(name string)
+	GetDescription() (description string)
+	SetDescription(description string)
+}
+
+type ModelWithKey interface {
+	Model
+	GetKey() (key string)
+	SetKey(key string)
+}

core/interfaces/model_artifact.go

@@ -0,0 +1,12 @@
+package interfaces
+
+import "go.mongodb.org/mongo-driver/bson/primitive"
+
+type ModelArtifact interface {
+	Model
+	GetSys() (sys ModelArtifactSys)
+	GetTagIds() (ids []primitive.ObjectID)
+	SetTagIds(ids []primitive.ObjectID)
+	SetObj(obj Model)
+	SetDel(del bool)
+}

core/interfaces/model_artifact_sys.go

@@ -0,0 +1,21 @@
+package interfaces
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type ModelArtifactSys interface {
+	GetCreateTs() time.Time
+	SetCreateTs(ts time.Time)
+	GetUpdateTs() time.Time
+	SetUpdateTs(ts time.Time)
+	GetDeleteTs() time.Time
+	SetDeleteTs(ts time.Time)
+	GetCreateUid() primitive.ObjectID
+	SetCreateUid(id primitive.ObjectID)
+	GetUpdateUid() primitive.ObjectID
+	SetUpdateUid(id primitive.ObjectID)
+	GetDeleteUid() primitive.ObjectID
+	SetDeleteUid(id primitive.ObjectID)
+}

core/interfaces/model_base_service.go

@@ -0,0 +1,28 @@
+package interfaces
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type ModelBaseService interface {
+	GetModelId() (id ModelId)
+	SetModelId(id ModelId)
+	GetById(id primitive.ObjectID) (res Model, err error)
+	Get(query bson.M, opts *mongo.FindOptions) (res Model, err error)
+	GetList(query bson.M, opts *mongo.FindOptions) (res List, err error)
+	DeleteById(id primitive.ObjectID, args ...interface{}) (err error)
+	Delete(query bson.M, args ...interface{}) (err error)
+	DeleteList(query bson.M, args ...interface{}) (err error)
+	ForceDeleteList(query bson.M, args ...interface{}) (err error)
+	UpdateById(id primitive.ObjectID, update bson.M, args ...interface{}) (err error)
+	Update(query bson.M, update bson.M, fields []string, args ...interface{}) (err error)
+	UpdateDoc(query bson.M, doc Model, fields []string, args ...interface{}) (err error)
+	Insert(u User, docs ...interface{}) (err error)
+	Count(query bson.M) (total int, err error)
+}
+
+type ModelService interface {
+	GetBaseService(id ModelId) (svc ModelBaseService)
+}

core/interfaces/model_binder.go

@@ -0,0 +1,6 @@
+package interfaces
+
+type ModelBinder interface {
+	Bind() (res Model, err error)
+	Process(d Model) (res Model, err error)
+}

core/interfaces/model_delegate.go

@@ -0,0 +1,22 @@
+package interfaces
+
+type ModelDelegateMethod string
+
+type ModelDelegate interface {
+	Add() error
+	Save() error
+	Delete() error
+	GetArtifact() (ModelArtifact, error)
+	GetModel() Model
+	Refresh() error
+	ToBytes(interface{}) ([]byte, error)
+}
+
+const (
+	ModelDelegateMethodAdd         = "add"
+	ModelDelegateMethodSave        = "save"
+	ModelDelegateMethodDelete      = "delete"
+	ModelDelegateMethodGetArtifact = "get-artifact"
+	ModelDelegateMethodRefresh     = "refresh"
+	ModelDelegateMethodChange      = "change"
+)

core/interfaces/model_environment.go

@@ -0,0 +1,9 @@
+package interfaces
+
+type Environment interface {
+	Model
+	GetKey() (key string)
+	SetKey(key string)
+	GetValue() (value string)
+	SetValue(value string)
+}

core/interfaces/model_extra_value.go

@@ -0,0 +1,17 @@
+package interfaces
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type ExtraValue interface {
+	Model
+	GetValue() (v interface{})
+	SetValue(v interface{})
+	GetObjectId() (oid primitive.ObjectID)
+	SetObjectId(oid primitive.ObjectID)
+	GetModel() (m string)
+	SetModel(m string)
+	GetType() (t string)
+	SetType(t string)
+}

core/interfaces/model_git.go

@@ -0,0 +1,18 @@
+package interfaces
+
+// Git interface
+type Git interface {
+	Model
+	GetUrl() (url string)
+	SetUrl(url string)
+	GetAuthType() (authType string)
+	SetAuthType(authType string)
+	GetUsername() (username string)
+	SetUsername(username string)
+	GetPassword() (password string)
+	SetPassword(password string)
+	GetCurrentBranch() (currentBranch string)
+	SetCurrentBranch(currentBranch string)
+	GetAutoPull() (autoPull bool)
+	SetAutoPull(autoPull bool)
+}

core/interfaces/model_list_binder.go

@@ -0,0 +1,6 @@
+package interfaces
+
+type ModelListBinder interface {
+	Bind() (l List, err error)
+	Process(d interface{}) (l List, err error)
+}

core/interfaces/model_node.go

@@ -0,0 +1,22 @@
+package interfaces
+
+import "time"
+
+type Node interface {
+	ModelWithNameDescription
+	GetKey() (key string)
+	GetIsMaster() (ok bool)
+	GetActive() (active bool)
+	SetActive(active bool)
+	SetActiveTs(activeTs time.Time)
+	GetStatus() (status string)
+	SetStatus(status string)
+	GetEnabled() (enabled bool)
+	SetEnabled(enabled bool)
+	GetAvailableRunners() (runners int)
+	SetAvailableRunners(runners int)
+	GetMaxRunners() (runners int)
+	SetMaxRunners(runners int)
+	IncrementAvailableRunners()
+	DecrementAvailableRunners()
+}

core/interfaces/model_node_delegate.go

@@ -0,0 +1,10 @@
+package interfaces
+
+import "time"
+
+type ModelNodeDelegate interface {
+	ModelDelegate
+	UpdateStatus(active bool, activeTs *time.Time, status string) (err error)
+	UpdateStatusOnline() (err error)
+	UpdateStatusOffline() (err error)
+}

core/interfaces/model_permission.go

@@ -0,0 +1,14 @@
+package interfaces
+
+type Permission interface {
+	ModelWithKey
+	ModelWithNameDescription
+	GetType() (t string)
+	SetType(t string)
+	GetTarget() (target []string)
+	SetTarget(target []string)
+	GetAllow() (allow []string)
+	SetAllow(allow []string)
+	GetDeny() (deny []string)
+	SetDeny(deny []string)
+}

core/interfaces/model_result.go

@@ -0,0 +1,11 @@
+package interfaces
+
+import "go.mongodb.org/mongo-driver/bson/primitive"
+
+type Result interface {
+	Value() map[string]interface{}
+	SetValue(key string, value interface{})
+	GetValue(key string) (value interface{})
+	GetTaskId() (id primitive.ObjectID)
+	SetTaskId(id primitive.ObjectID)
+}

core/interfaces/model_role.go

@@ -0,0 +1,6 @@
+package interfaces
+
+type Role interface {
+	ModelWithKey
+	ModelWithNameDescription
+}

core/interfaces/model_schedule.go

@@ -0,0 +1,28 @@
+package interfaces
+
+import (
+	"github.com/robfig/cron/v3"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Schedule interface {
+	Model
+	GetEnabled() (enabled bool)
+	SetEnabled(enabled bool)
+	GetEntryId() (id cron.EntryID)
+	SetEntryId(id cron.EntryID)
+	GetCron() (c string)
+	SetCron(c string)
+	GetSpiderId() (id primitive.ObjectID)
+	SetSpiderId(id primitive.ObjectID)
+	GetMode() (mode string)
+	SetMode(mode string)
+	GetNodeIds() (ids []primitive.ObjectID)
+	SetNodeIds(ids []primitive.ObjectID)
+	GetCmd() (cmd string)
+	SetCmd(cmd string)
+	GetParam() (param string)
+	SetParam(param string)
+	GetPriority() (p int)
+	SetPriority(p int)
+}

core/interfaces/model_service_v2.go

@@ -0,0 +1,25 @@
+package interfaces
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type ModelServiceV2[T any] interface {
+	GetById(id primitive.ObjectID) (model *T, err error)
+	Get(query bson.M, options *mongo.FindOptions) (model *T, err error)
+	GetList(query bson.M, options *mongo.FindOptions) (models []T, err error)
+	DeleteById(id primitive.ObjectID) (err error)
+	Delete(query bson.M) (err error)
+	DeleteList(query bson.M) (err error)
+	UpdateById(id primitive.ObjectID, update bson.M) (err error)
+	UpdateOne(query bson.M, update bson.M) (err error)
+	UpdateMany(query bson.M, update bson.M) (err error)
+	ReplaceById(id primitive.ObjectID, model T) (err error)
+	Replace(query bson.M, model T) (err error)
+	InsertOne(model T) (id primitive.ObjectID, err error)
+	InsertMany(models []T) (ids []primitive.ObjectID, err error)
+	Count(query bson.M) (total int, err error)
+	GetCol() (col *mongo.Col)
+}

core/interfaces/model_spider.go

@@ -0,0 +1,24 @@
+package interfaces
+
+import "go.mongodb.org/mongo-driver/bson/primitive"
+
+type Spider interface {
+	ModelWithNameDescription
+	GetType() (ty string)
+	GetMode() (mode string)
+	SetMode(mode string)
+	GetNodeIds() (ids []primitive.ObjectID)
+	SetNodeIds(ids []primitive.ObjectID)
+	GetCmd() (cmd string)
+	SetCmd(cmd string)
+	GetParam() (param string)
+	SetParam(param string)
+	GetPriority() (p int)
+	SetPriority(p int)
+	GetColId() (id primitive.ObjectID)
+	SetColId(id primitive.ObjectID)
+	GetIncrementalSync() (incrementalSync bool)
+	SetIncrementalSync(incrementalSync bool)
+	GetAutoInstall() (autoInstall bool)
+	SetAutoInstall(autoInstall bool)
+}

core/interfaces/model_tag.go

@@ -0,0 +1,8 @@
+package interfaces
+
+type Tag interface {
+	Model
+	GetName() string
+	GetColor() string
+	SetCol(string)
+}

core/interfaces/model_task.go

@@ -0,0 +1,23 @@
+package interfaces
+
+import "go.mongodb.org/mongo-driver/bson/primitive"
+
+type Task interface {
+	Model
+	GetNodeId() (id primitive.ObjectID)
+	SetNodeId(id primitive.ObjectID)
+	GetNodeIds() (ids []primitive.ObjectID)
+	GetStatus() (status string)
+	SetStatus(status string)
+	GetError() (error string)
+	SetError(error string)
+	GetPid() (pid int)
+	SetPid(pid int)
+	GetSpiderId() (id primitive.ObjectID)
+	GetType() (ty string)
+	GetCmd() (cmd string)
+	GetParam() (param string)
+	GetPriority() (p int)
+	GetUserId() (id primitive.ObjectID)
+	SetUserId(id primitive.ObjectID)
+}

core/interfaces/model_task_stat.go

@@ -0,0 +1,23 @@
+package interfaces
+
+import "time"
+
+type TaskStat interface {
+	Model
+	GetCreateTs() (ts time.Time)
+	SetCreateTs(ts time.Time)
+	GetStartTs() (ts time.Time)
+	SetStartTs(ts time.Time)
+	GetEndTs() (ts time.Time)
+	SetEndTs(ts time.Time)
+	GetWaitDuration() (d int64)
+	SetWaitDuration(d int64)
+	GetRuntimeDuration() (d int64)
+	SetRuntimeDuration(d int64)
+	GetTotalDuration() (d int64)
+	SetTotalDuration(d int64)
+	GetResultCount() (c int64)
+	SetResultCount(c int64)
+	GetErrorLogCount() (c int64)
+	SetErrorLogCount(c int64)
+}

core/interfaces/model_user.go

@@ -0,0 +1,9 @@
+package interfaces
+
+type User interface {
+	Model
+	GetUsername() (name string)
+	GetPassword() (p string)
+	GetRole() (r string)
+	GetEmail() (email string)
+}

core/interfaces/model_user_group.go

@@ -0,0 +1,6 @@
+package interfaces
+
+type UserGroup interface {
+	Model
+	GetUsers() (users []User, err error)
+}

core/interfaces/module.go

@@ -0,0 +1,10 @@
+package interfaces
+
+type ModuleId int
+
+type Module interface {
+	Init() error
+	Start()
+	Wait()
+	Stop()
+}

core/interfaces/node_config_service.go

@@ -0,0 +1,13 @@
+package interfaces
+
+type NodeConfigService interface {
+	WithConfigPath
+	Init() error
+	Reload() error
+	GetBasicNodeInfo() Entity
+	GetNodeKey() string
+	GetNodeName() string
+	IsMaster() bool
+	GetAuthKey() string
+	GetMaxRunners() int
+}

core/interfaces/node_master_service.go

@@ -0,0 +1,14 @@
+package interfaces
+
+import (
+	"time"
+)
+
+type NodeMasterService interface {
+	NodeService
+	Monitor()
+	SetMonitorInterval(duration time.Duration)
+	Register() error
+	StopOnError()
+	GetServer() GrpcServer
+}

core/interfaces/node_service.go

@@ -0,0 +1,8 @@
+package interfaces
+
+type NodeService interface {
+	Module
+	WithConfigPath
+	WithAddress
+	GetConfigService() NodeConfigService
+}

core/interfaces/node_service_option.go

@@ -0,0 +1,4 @@
+package interfaces
+
+type NodeServiceOption interface {
+}

core/interfaces/node_worker_service.go

@@ -0,0 +1,11 @@
+package interfaces
+
+import "time"
+
+type NodeWorkerService interface {
+	NodeService
+	Register()
+	Recv()
+	ReportStatus()
+	SetHeartbeatInterval(duration time.Duration)
+}

core/interfaces/options.go

@@ -0,0 +1 @@
+package interfaces

core/interfaces/process_daemon.go

@@ -0,0 +1,17 @@
+package interfaces
+
+import (
+	"os/exec"
+	"time"
+)
+
+type ProcessDaemon interface {
+	Start() (err error)
+	Stop()
+	GetMaxErrors() (maxErrors int)
+	SetMaxErrors(maxErrors int)
+	GetExitTimeout() (timeout time.Duration)
+	SetExitTimeout(timeout time.Duration)
+	GetCmd() (cmd *exec.Cmd)
+	GetCh() (ch chan int)
+}

core/interfaces/provide.go

@@ -0,0 +1,3 @@
+package interfaces
+
+type Provide func(env string)

core/interfaces/result_service.go

@@ -0,0 +1,15 @@
+package interfaces
+
+import (
+	"github.com/crawlab-team/crawlab-db/generic"
+	"time"
+)
+
+type ResultService interface {
+	Insert(records ...interface{}) (err error)
+	List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error)
+	Count(query generic.ListQuery) (n int, err error)
+	Index(fields []string)
+	SetTime(t time.Time)
+	GetTime() (t time.Time)
+}

core/interfaces/result_service_mongo.go

@@ -0,0 +1,15 @@
+package interfaces
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type ResultServiceMongo interface {
+	GetId() (id primitive.ObjectID)
+	SetId(id primitive.ObjectID)
+	List(query bson.M, opts *mongo.FindOptions) (results []Result, err error)
+	Count(query bson.M) (total int, err error)
+	Insert(docs ...interface{}) (err error)
+}

core/interfaces/result_service_registry.go

@@ -0,0 +1,11 @@
+package interfaces
+
+import "go.mongodb.org/mongo-driver/bson/primitive"
+
+type ResultServiceRegistry interface {
+	Register(key string, fn ResultServiceRegistryFn)
+	Unregister(key string)
+	Get(key string) (fn ResultServiceRegistryFn)
+}
+
+type ResultServiceRegistryFn func(colId primitive.ObjectID, dsId primitive.ObjectID) (ResultService, error)

core/interfaces/schedule_service.go

@@ -0,0 +1,23 @@
+package interfaces
+
+import (
+	"github.com/robfig/cron/v3"
+	"time"
+)
+
+type ScheduleService interface {
+	WithConfigPath
+	Module
+	GetLocation() (loc *time.Location)
+	SetLocation(loc *time.Location)
+	GetDelay() (delay bool)
+	SetDelay(delay bool)
+	GetSkip() (skip bool)
+	SetSkip(skip bool)
+	GetUpdateInterval() (interval time.Duration)
+	SetUpdateInterval(interval time.Duration)
+	Enable(s Schedule, args ...interface{}) (err error)
+	Disable(s Schedule, args ...interface{}) (err error)
+	Update()
+	GetCron() (c *cron.Cron)
+}

core/interfaces/spider_admin_service.go

@@ -0,0 +1,22 @@
+package interfaces
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type SpiderAdminService interface {
+	WithConfigPath
+	Start() (err error)
+	// Schedule a new task of the spider
+	Schedule(id primitive.ObjectID, opts *SpiderRunOptions) (taskIds []primitive.ObjectID, err error)
+	// Clone the spider
+	Clone(id primitive.ObjectID, opts *SpiderCloneOptions) (err error)
+	// Delete the spider
+	Delete(id primitive.ObjectID) (err error)
+	// SyncGit syncs all git repositories
+	SyncGit() (err error)
+	// SyncGitOne syncs one git repository
+	SyncGitOne(g Git) (err error)
+	// Export exports the spider and return zip file path
+	Export(id primitive.ObjectID) (filePath string, err error)
+}

core/interfaces/spider_service_options.go

@@ -0,0 +1,17 @@
+package interfaces
+
+import "go.mongodb.org/mongo-driver/bson/primitive"
+
+type SpiderRunOptions struct {
+	Mode       string               `json:"mode"`
+	NodeIds    []primitive.ObjectID `json:"node_ids"`
+	Cmd        string               `json:"cmd"`
+	Param      string               `json:"param"`
+	ScheduleId primitive.ObjectID   `json:"schedule_id"`
+	Priority   int                  `json:"priority"`
+	UserId     primitive.ObjectID   `json:"-"`
+}
+
+type SpiderCloneOptions struct {
+	Name string
+}

core/interfaces/stats_service.go

@@ -0,0 +1,9 @@
+package interfaces
+
+import "go.mongodb.org/mongo-driver/bson"
+
+type StatsService interface {
+	GetOverviewStats(query bson.M) (data interface{}, err error)
+	GetDailyStats(query bson.M) (data interface{}, err error)
+	GetTaskStats(query bson.M) (data interface{}, err error)
+}

core/interfaces/task_base_service.go

@@ -0,0 +1,11 @@
+package interfaces
+
+import "go.mongodb.org/mongo-driver/bson/primitive"
+
+type TaskBaseService interface {
+	WithConfigPath
+	Module
+	SaveTask(t Task, status string) (err error)
+	IsStopped() (res bool)
+	GetQueue(nodeId primitive.ObjectID) (queue string)
+}

core/interfaces/task_handler_service.go

@@ -0,0 +1,60 @@
+package interfaces
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type TaskHandlerService interface {
+	TaskBaseService
+	// Run task and execute locally
+	Run(taskId primitive.ObjectID) (err error)
+	// Cancel task locally
+	Cancel(taskId primitive.ObjectID) (err error)
+	// Fetch tasks and run
+	Fetch()
+	// ReportStatus periodically report handler status to master
+	ReportStatus()
+	// Reset reset internals to default
+	Reset()
+	// IsSyncLocked whether the given task is locked for files sync
+	IsSyncLocked(path string) (ok bool)
+	// LockSync lock files sync for given task
+	LockSync(path string)
+	// UnlockSync unlock files sync for given task
+	UnlockSync(path string)
+	// GetExitWatchDuration get max runners
+	GetExitWatchDuration() (duration time.Duration)
+	// SetExitWatchDuration set max runners
+	SetExitWatchDuration(duration time.Duration)
+	// GetFetchInterval get report interval
+	GetFetchInterval() (interval time.Duration)
+	// SetFetchInterval set report interval
+	SetFetchInterval(interval time.Duration)
+	// GetReportInterval get report interval
+	GetReportInterval() (interval time.Duration)
+	// SetReportInterval set report interval
+	SetReportInterval(interval time.Duration)
+	// GetCancelTimeout get report interval
+	GetCancelTimeout() (timeout time.Duration)
+	// SetCancelTimeout set report interval
+	SetCancelTimeout(timeout time.Duration)
+	// GetModelService get model service
+	GetModelService() (modelSvc GrpcClientModelService)
+	// GetModelSpiderService get model spider service
+	GetModelSpiderService() (modelSpiderSvc GrpcClientModelSpiderService)
+	// GetModelTaskService get model task service
+	GetModelTaskService() (modelTaskSvc GrpcClientModelTaskService)
+	// GetModelTaskStatService get model task stat service
+	GetModelTaskStatService() (modelTaskStatSvc GrpcClientModelTaskStatService)
+	// GetModelEnvironmentService get model environment service
+	GetModelEnvironmentService() (modelEnvironmentSvc GrpcClientModelEnvironmentService)
+	// GetNodeConfigService get node config service
+	GetNodeConfigService() (cfgSvc NodeConfigService)
+	// GetCurrentNode get node of the handler
+	GetCurrentNode() (n Node, err error)
+	// GetTaskById get task by id
+	GetTaskById(id primitive.ObjectID) (t Task, err error)
+	// GetSpiderById get task by id
+	GetSpiderById(id primitive.ObjectID) (t Spider, err error)
+}

core/interfaces/task_hook_service.go

@@ -0,0 +1,6 @@
+package interfaces
+
+type TaskHookService interface {
+	PreActions(Task, Spider, FsServiceV2, TaskHandlerService) (err error)
+	PostActions(Task, Spider, FsServiceV2, TaskHandlerService) (err error)
+}

core/interfaces/task_runner.go

@@ -0,0 +1,15 @@
+package interfaces
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type TaskRunner interface {
+	Init() (err error)
+	Run() (err error)
+	Cancel() (err error)
+	SetSubscribeTimeout(timeout time.Duration)
+	GetTaskId() (id primitive.ObjectID)
+	CleanUp() (err error)
+}

core/interfaces/task_scheduler_service.go

@@ -0,0 +1,16 @@
+package interfaces
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type TaskSchedulerService interface {
+	TaskBaseService
+	// Enqueue task into the task queue
+	Enqueue(t Task) (t2 Task, err error)
+	// Cancel task to corresponding node
+	Cancel(id primitive.ObjectID, args ...interface{}) (err error)
+	// SetInterval set the interval or duration between two adjacent fetches
+	SetInterval(interval time.Duration)
+}

core/interfaces/task_stats_service.go

@@ -0,0 +1,9 @@
+package interfaces
+
+import "go.mongodb.org/mongo-driver/bson/primitive"
+
+type TaskStatsService interface {
+	TaskBaseService
+	InsertData(id primitive.ObjectID, records ...interface{}) (err error)
+	InsertLogs(id primitive.ObjectID, logs ...string) (err error)
+}

core/interfaces/test.go

@@ -0,0 +1,8 @@
+package interfaces
+
+import "testing"
+
+type Test interface {
+	Setup(*testing.T)
+	Cleanup()
+}

core/interfaces/translation.go

@@ -0,0 +1,5 @@
+package interfaces
+
+type Translation interface {
+	GetLang() (l string)
+}

core/interfaces/user_service.go

@@ -0,0 +1,19 @@
+package interfaces
+
+import (
+	"github.com/gin-gonic/gin"
+	"github.com/golang-jwt/jwt/v5"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type UserService interface {
+	Init() (err error)
+	SetJwtSecret(secret string)
+	SetJwtSigningMethod(method jwt.SigningMethod)
+	Create(opts *UserCreateOptions, args ...interface{}) (err error)
+	Login(opts *UserLoginOptions) (token string, u User, err error)
+	CheckToken(token string) (u User, err error)
+	ChangePassword(id primitive.ObjectID, password string, args ...interface{}) (err error)
+	MakeToken(user User) (tokenStr string, err error)
+	GetCurrentUser(c *gin.Context) (u User, err error)
+}

core/interfaces/user_service_options.go

@@ -0,0 +1,13 @@
+package interfaces
+
+type UserCreateOptions struct {
+	Username string
+	Password string
+	Email    string
+	Role     string
+}
+
+type UserLoginOptions struct {
+	Username string
+	Password string
+}

core/interfaces/with_address.go

@@ -0,0 +1,6 @@
+package interfaces
+
+type WithAddress interface {
+	GetAddress() (address Address)
+	SetAddress(address Address)
+}

core/interfaces/with_config_path.go

@@ -0,0 +1,6 @@
+package interfaces
+
+type WithConfigPath interface {
+	GetConfigPath() (path string)
+	SetConfigPath(path string)
+}

core/interfaces/with_model_id.go

@@ -0,0 +1,6 @@
+package interfaces
+
+type WithModelId interface {
+	GetModelId() (id ModelId)
+	SetModelId(id ModelId)
+}