feat: added modules

core/models/models/artifact.go

@@ -0,0 +1,56 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Artifact struct {
+	Id     primitive.ObjectID   `bson:"_id" json:"_id"`
+	Col    string               `bson:"_col" json:"_col"`
+	Del    bool                 `bson:"_del" json:"_del"`
+	TagIds []primitive.ObjectID `bson:"_tid" json:"_tid"`
+	Sys    *ArtifactSys         `bson:"_sys" json:"_sys"`
+	Obj    interface{}          `bson:"_obj" json:"_obj"`
+}
+
+func (a *Artifact) GetId() (id primitive.ObjectID) {
+	return a.Id
+}
+
+func (a *Artifact) SetId(id primitive.ObjectID) {
+	a.Id = id
+}
+
+func (a *Artifact) GetSys() (sys interfaces.ModelArtifactSys) {
+	if a.Sys == nil {
+		a.Sys = &ArtifactSys{}
+	}
+	return a.Sys
+}
+
+func (a *Artifact) GetTagIds() (ids []primitive.ObjectID) {
+	return a.TagIds
+}
+
+func (a *Artifact) SetTagIds(ids []primitive.ObjectID) {
+	a.TagIds = ids
+}
+
+func (a *Artifact) SetObj(obj interfaces.Model) {
+	a.Obj = obj
+}
+
+func (a *Artifact) SetDel(del bool) {
+	a.Del = del
+}
+
+type ArtifactList []Artifact
+
+func (l *ArtifactList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/artifact_sys.go

@@ -0,0 +1,63 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type ArtifactSys struct {
+	CreateTs  time.Time          `json:"create_ts" bson:"create_ts"`
+	CreateUid primitive.ObjectID `json:"create_uid" bson:"create_uid"`
+	UpdateTs  time.Time          `json:"update_ts" bson:"update_ts"`
+	UpdateUid primitive.ObjectID `json:"update_uid" bson:"update_uid"`
+	DeleteTs  time.Time          `json:"delete_ts" bson:"delete_ts"`
+	DeleteUid primitive.ObjectID `json:"delete_uid" bson:"delete_uid"`
+}
+
+func (sys *ArtifactSys) GetCreateTs() time.Time {
+	return sys.CreateTs
+}
+
+func (sys *ArtifactSys) SetCreateTs(ts time.Time) {
+	sys.CreateTs = ts
+}
+
+func (sys *ArtifactSys) GetUpdateTs() time.Time {
+	return sys.UpdateTs
+}
+
+func (sys *ArtifactSys) SetUpdateTs(ts time.Time) {
+	sys.UpdateTs = ts
+}
+
+func (sys *ArtifactSys) GetDeleteTs() time.Time {
+	return sys.DeleteTs
+}
+
+func (sys *ArtifactSys) SetDeleteTs(ts time.Time) {
+	sys.DeleteTs = ts
+}
+
+func (sys *ArtifactSys) GetCreateUid() primitive.ObjectID {
+	return sys.CreateUid
+}
+
+func (sys *ArtifactSys) SetCreateUid(id primitive.ObjectID) {
+	sys.CreateUid = id
+}
+
+func (sys *ArtifactSys) GetUpdateUid() primitive.ObjectID {
+	return sys.UpdateUid
+}
+
+func (sys *ArtifactSys) SetUpdateUid(id primitive.ObjectID) {
+	sys.UpdateUid = id
+}
+
+func (sys *ArtifactSys) GetDeleteUid() primitive.ObjectID {
+	return sys.DeleteUid
+}
+
+func (sys *ArtifactSys) SetDeleteUid(id primitive.ObjectID) {
+	sys.DeleteUid = id
+}

core/models/models/base.go

@@ -0,0 +1,13 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type BaseModel struct {
+	Id primitive.ObjectID `json:"_id" bson:"_id"`
+}
+
+func (d *BaseModel) GetId() (id primitive.ObjectID) {
+	return d.Id
+}

core/models/models/base_v2.go

@@ -0,0 +1,64 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type BaseModelV2[T any] struct {
+	Id        primitive.ObjectID `json:"_id" bson:"_id"`
+	CreatedAt time.Time          `json:"created_ts" bson:"created_ts"`
+	CreatedBy primitive.ObjectID `json:"created_by" bson:"created_by"`
+	UpdatedAt time.Time          `json:"updated_ts" bson:"updated_ts"`
+	UpdatedBy primitive.ObjectID `json:"updated_by" bson:"updated_by"`
+}
+
+func (m *BaseModelV2[T]) GetId() primitive.ObjectID {
+	return m.Id
+}
+
+func (m *BaseModelV2[T]) SetId(id primitive.ObjectID) {
+	m.Id = id
+}
+
+func (m *BaseModelV2[T]) GetCreatedAt() time.Time {
+	return m.CreatedAt
+}
+
+func (m *BaseModelV2[T]) SetCreatedAt(t time.Time) {
+	m.CreatedAt = t
+}
+
+func (m *BaseModelV2[T]) GetCreatedBy() primitive.ObjectID {
+	return m.CreatedBy
+}
+
+func (m *BaseModelV2[T]) SetCreatedBy(id primitive.ObjectID) {
+	m.CreatedBy = id
+}
+
+func (m *BaseModelV2[T]) GetUpdatedAt() time.Time {
+	return m.UpdatedAt
+}
+
+func (m *BaseModelV2[T]) SetUpdatedAt(t time.Time) {
+	m.UpdatedAt = t
+}
+
+func (m *BaseModelV2[T]) GetUpdatedBy() primitive.ObjectID {
+	return m.UpdatedBy
+}
+
+func (m *BaseModelV2[T]) SetUpdatedBy(id primitive.ObjectID) {
+	m.UpdatedBy = id
+}
+
+func (m *BaseModelV2[T]) SetCreated(id primitive.ObjectID) {
+	m.SetCreatedAt(time.Now())
+	m.SetCreatedBy(id)
+}
+
+func (m *BaseModelV2[T]) SetUpdated(id primitive.ObjectID) {
+	m.SetUpdatedAt(time.Now())
+	m.SetUpdatedBy(id)
+}

core/models/models/data_collection.go

@@ -0,0 +1,36 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type DataCollection struct {
+	Id     primitive.ObjectID `json:"_id" bson:"_id"`
+	Name   string             `json:"name" bson:"name"`
+	Fields []entity.DataField `json:"fields" bson:"fields"`
+	Dedup  struct {
+		Enabled bool     `json:"enabled" bson:"enabled"`
+		Keys    []string `json:"keys" bson:"keys"`
+		Type    string   `json:"type" bson:"type"`
+	} `json:"dedup" bson:"dedup"`
+}
+
+func (dc *DataCollection) GetId() (id primitive.ObjectID) {
+	return dc.Id
+}
+
+func (dc *DataCollection) SetId(id primitive.ObjectID) {
+	dc.Id = id
+}
+
+type DataCollectionList []DataCollection
+
+func (l *DataCollectionList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/data_collection_v2.go

@@ -0,0 +1,17 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/entity"
+)
+
+type DataCollectionV2 struct {
+	any                         `collection:"data_collections"`
+	BaseModelV2[DataCollection] `bson:",inline"`
+	Name                        string             `json:"name" bson:"name"`
+	Fields                      []entity.DataField `json:"fields" bson:"fields"`
+	Dedup                       struct {
+		Enabled bool     `json:"enabled" bson:"enabled"`
+		Keys    []string `json:"keys" bson:"keys"`
+		Type    string   `json:"type" bson:"type"`
+	} `json:"dedup" bson:"dedup"`
+}

core/models/models/data_source.go

@@ -0,0 +1,42 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type DataSource struct {
+	Id          primitive.ObjectID `json:"_id" bson:"_id"`
+	Name        string             `json:"name" bson:"name"`
+	Type        string             `json:"type" bson:"type"`
+	Description string             `json:"description" bson:"description"`
+	Host        string             `json:"host" bson:"host"`
+	Port        string             `json:"port" bson:"port"`
+	Url         string             `json:"url" bson:"url"`
+	Hosts       []string           `json:"hosts" bson:"hosts"`
+	Database    string             `json:"database" bson:"database"`
+	Username    string             `json:"username" bson:"username"`
+	Password    string             `json:"password,omitempty" bson:"-"`
+	ConnectType string             `json:"connect_type" bson:"connect_type"`
+	Status      string             `json:"status" bson:"status"`
+	Error       string             `json:"error" bson:"error"`
+	Extra       map[string]string  `json:"extra,omitempty" bson:"extra,omitempty"`
+}
+
+func (ds *DataSource) GetId() (id primitive.ObjectID) {
+	return ds.Id
+}
+
+func (ds *DataSource) SetId(id primitive.ObjectID) {
+	ds.Id = id
+}
+
+type DataSourceList []DataSource
+
+func (l *DataSourceList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/data_source_v2.go

@@ -0,0 +1,20 @@
+package models
+
+type DataSourceV2 struct {
+	any                     `collection:"data_sources"`
+	BaseModelV2[DataSource] `bson:",inline"`
+	Name                    string            `json:"name" bson:"name"`
+	Type                    string            `json:"type" bson:"type"`
+	Description             string            `json:"description" bson:"description"`
+	Host                    string            `json:"host" bson:"host"`
+	Port                    string            `json:"port" bson:"port"`
+	Url                     string            `json:"url" bson:"url"`
+	Hosts                   []string          `json:"hosts" bson:"hosts"`
+	Database                string            `json:"database" bson:"database"`
+	Username                string            `json:"username" bson:"username"`
+	Password                string            `json:"password,omitempty" bson:"-"`
+	ConnectType             string            `json:"connect_type" bson:"connect_type"`
+	Status                  string            `json:"status" bson:"status"`
+	Error                   string            `json:"error" bson:"error"`
+	Extra                   map[string]string `json:"extra,omitempty" bson:"extra,omitempty"`
+}

core/models/models/dependency_setting.go

@@ -0,0 +1,36 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type DependencySetting struct {
+	Id           primitive.ObjectID `json:"_id" bson:"_id"`
+	Key          string             `json:"key" bson:"key"`
+	Name         string             `json:"name" bson:"name"`
+	Description  string             `json:"description" bson:"description"`
+	Enabled      bool               `json:"enabled" bson:"enabled"`
+	Cmd          string             `json:"cmd" bson:"cmd"`
+	Proxy        string             `json:"proxy" bson:"proxy"`
+	LastUpdateTs time.Time          `json:"last_update_ts" bson:"last_update_ts"`
+}
+
+func (j *DependencySetting) GetId() (id primitive.ObjectID) {
+	return j.Id
+}
+
+func (j *DependencySetting) SetId(id primitive.ObjectID) {
+	j.Id = id
+}
+
+type DependencySettingList []DependencySetting
+
+func (l *DependencySettingList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/dependency_setting_v2.go

@@ -0,0 +1,17 @@
+package models
+
+import (
+	"time"
+)
+
+type DependencySettingV2 struct {
+	any                            `collection:"dependency_settings"`
+	BaseModelV2[DependencySetting] `bson:",inline"`
+	Key                            string    `json:"key" bson:"key"`
+	Name                           string    `json:"name" bson:"name"`
+	Description                    string    `json:"description" bson:"description"`
+	Enabled                        bool      `json:"enabled" bson:"enabled"`
+	Cmd                            string    `json:"cmd" bson:"cmd"`
+	Proxy                          string    `json:"proxy" bson:"proxy"`
+	LastUpdateTs                   time.Time `json:"last_update_ts" bson:"last_update_ts"`
+}

core/models/models/environment.go

@@ -0,0 +1,46 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Environment struct {
+	Id    primitive.ObjectID `json:"_id" bson:"_id"`
+	Key   string             `json:"key" bson:"key"`
+	Value string             `json:"value" bson:"value"`
+}
+
+func (e *Environment) GetId() (id primitive.ObjectID) {
+	return e.Id
+}
+
+func (e *Environment) SetId(id primitive.ObjectID) {
+	e.Id = id
+}
+
+func (e *Environment) GetKey() (key string) {
+	return e.Key
+}
+
+func (e *Environment) SetKey(key string) {
+	e.Key = key
+}
+
+func (e *Environment) GetValue() (value string) {
+	return e.Value
+}
+
+func (e *Environment) SetValue(value string) {
+	e.Value = value
+}
+
+type EnvironmentList []Environment
+
+func (l *EnvironmentList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/environment_v2.go

@@ -0,0 +1,8 @@
+package models
+
+type EnvironmentV2 struct {
+	any                        `collection:"environments"`
+	BaseModelV2[EnvironmentV2] `bson:",inline"`
+	Key                        string `json:"key" bson:"key"`
+	Value                      string `json:"value" bson:"value"`
+}

core/models/models/extra_value.go

@@ -0,0 +1,64 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type ExtraValue struct {
+	Id       primitive.ObjectID `json:"_id" bson:"_id"`
+	ObjectId primitive.ObjectID `json:"oid" bson:"oid"`
+	Model    string             `json:"model" bson:"m"`
+	Type     string             `json:"type" bson:"t"`
+	Value    interface{}        `json:"value" bson:"v"`
+}
+
+func (ev *ExtraValue) GetId() (id primitive.ObjectID) {
+	return ev.Id
+}
+
+func (ev *ExtraValue) SetId(id primitive.ObjectID) {
+	ev.Id = id
+}
+
+func (ev *ExtraValue) GetValue() (v interface{}) {
+	return ev.Value
+}
+
+func (ev *ExtraValue) SetValue(v interface{}) {
+	ev.Value = v
+}
+
+func (ev *ExtraValue) GetObjectId() (oid primitive.ObjectID) {
+	return ev.ObjectId
+}
+
+func (ev *ExtraValue) SetObjectId(oid primitive.ObjectID) {
+	ev.ObjectId = oid
+}
+
+func (ev *ExtraValue) GetModel() (m string) {
+	return ev.Model
+}
+
+func (ev *ExtraValue) SetModel(m string) {
+	ev.Model = m
+}
+
+func (ev *ExtraValue) GetType() (t string) {
+	return ev.Type
+}
+
+func (ev *ExtraValue) SetType(t string) {
+	ev.Type = t
+}
+
+type ExtraValueList []ExtraValue
+
+func (l *ExtraValueList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/git.go

@@ -0,0 +1,82 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Git struct {
+	Id            primitive.ObjectID `json:"_id" bson:"_id"`
+	Url           string             `json:"url" bson:"url"`
+	AuthType      string             `json:"auth_type" bson:"auth_type"`
+	Username      string             `json:"username" bson:"username"`
+	Password      string             `json:"password" bson:"password"`
+	CurrentBranch string             `json:"current_branch" bson:"current_branch"`
+	AutoPull      bool               `json:"auto_pull" bson:"auto_pull"`
+}
+
+func (g *Git) GetId() (id primitive.ObjectID) {
+	return g.Id
+}
+
+func (g *Git) SetId(id primitive.ObjectID) {
+	g.Id = id
+}
+
+func (g *Git) GetUrl() (url string) {
+	return g.Url
+}
+
+func (g *Git) SetUrl(url string) {
+	g.Url = url
+}
+
+func (g *Git) GetAuthType() (authType string) {
+	return g.AuthType
+}
+
+func (g *Git) SetAuthType(authType string) {
+	g.AuthType = authType
+}
+
+func (g *Git) GetUsername() (username string) {
+	return g.Username
+}
+
+func (g *Git) SetUsername(username string) {
+	g.Username = username
+}
+
+func (g *Git) GetPassword() (password string) {
+	return g.Password
+}
+
+func (g *Git) SetPassword(password string) {
+	g.Password = password
+}
+
+func (g *Git) GetCurrentBranch() (currentBranch string) {
+	return g.CurrentBranch
+}
+
+func (g *Git) SetCurrentBranch(currentBranch string) {
+	g.CurrentBranch = currentBranch
+}
+
+func (g *Git) GetAutoPull() (autoPull bool) {
+	return g.AutoPull
+}
+
+func (g *Git) SetAutoPull(autoPull bool) {
+	g.AutoPull = autoPull
+}
+
+type GitList []Git
+
+func (l *GitList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/git_v2.go

@@ -0,0 +1,12 @@
+package models
+
+type GitV2 struct {
+	any                `collection:"gits"`
+	BaseModelV2[GitV2] `bson:",inline"`
+	Url                string `json:"url" bson:"url"`
+	AuthType           string `json:"auth_type" bson:"auth_type"`
+	Username           string `json:"username" bson:"username"`
+	Password           string `json:"password" bson:"password"`
+	CurrentBranch      string `json:"current_branch" bson:"current_branch"`
+	AutoPull           bool   `json:"auto_pull" bson:"auto_pull"`
+}

core/models/models/job.go

@@ -0,0 +1,29 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Job struct {
+	Id     primitive.ObjectID `bson:"_id" json:"_id"`
+	TaskId primitive.ObjectID `bson:"task_id" json:"task_id"`
+}
+
+func (j *Job) GetId() (id primitive.ObjectID) {
+	return j.Id
+}
+
+func (j *Job) SetId(id primitive.ObjectID) {
+	j.Id = id
+}
+
+type JobList []Job
+
+func (l *JobList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/node.go

@@ -0,0 +1,119 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type Node struct {
+	Id               primitive.ObjectID `json:"_id" bson:"_id"`
+	Key              string             `json:"key" bson:"key"`
+	Name             string             `json:"name" bson:"name"`
+	Ip               string             `json:"ip" bson:"ip"`
+	Port             string             `json:"port" bson:"port"`
+	Mac              string             `json:"mac" bson:"mac"`
+	Hostname         string             `json:"hostname" bson:"hostname"`
+	Description      string             `json:"description" bson:"description"`
+	IsMaster         bool               `json:"is_master" bson:"is_master"`
+	Status           string             `json:"status" bson:"status"`
+	Enabled          bool               `json:"enabled" bson:"enabled"`
+	Active           bool               `json:"active" bson:"active"`
+	ActiveTs         time.Time          `json:"active_ts" bson:"active_ts"`
+	AvailableRunners int                `json:"available_runners" bson:"available_runners"`
+	MaxRunners       int                `json:"max_runners" bson:"max_runners"`
+}
+
+func (n *Node) GetId() (id primitive.ObjectID) {
+	return n.Id
+}
+
+func (n *Node) SetId(id primitive.ObjectID) {
+	n.Id = id
+}
+
+func (n *Node) GetName() (name string) {
+	return n.Name
+}
+
+func (n *Node) SetName(name string) {
+	n.Name = name
+}
+
+func (n *Node) GetDescription() (description string) {
+	return n.Description
+}
+
+func (n *Node) SetDescription(description string) {
+	n.Description = description
+}
+
+func (n *Node) GetKey() (key string) {
+	return n.Key
+}
+
+func (n *Node) GetIsMaster() (ok bool) {
+	return n.IsMaster
+}
+
+func (n *Node) GetActive() (active bool) {
+	return n.Active
+}
+
+func (n *Node) SetActive(active bool) {
+	n.Active = active
+}
+
+func (n *Node) SetActiveTs(activeTs time.Time) {
+	n.ActiveTs = activeTs
+}
+
+func (n *Node) GetStatus() (status string) {
+	return n.Status
+}
+
+func (n *Node) SetStatus(status string) {
+	n.Status = status
+}
+
+func (n *Node) GetEnabled() (enabled bool) {
+	return n.Enabled
+}
+
+func (n *Node) SetEnabled(enabled bool) {
+	n.Enabled = enabled
+}
+
+func (n *Node) GetAvailableRunners() (runners int) {
+	return n.AvailableRunners
+}
+
+func (n *Node) SetAvailableRunners(runners int) {
+	n.AvailableRunners = runners
+}
+
+func (n *Node) GetMaxRunners() (runners int) {
+	return n.MaxRunners
+}
+
+func (n *Node) SetMaxRunners(runners int) {
+	n.MaxRunners = runners
+}
+
+func (n *Node) IncrementAvailableRunners() {
+	n.AvailableRunners++
+}
+
+func (n *Node) DecrementAvailableRunners() {
+	n.AvailableRunners--
+}
+
+type NodeList []Node
+
+func (l *NodeList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/node_v2.go

@@ -0,0 +1,24 @@
+package models
+
+import (
+	"time"
+)
+
+type NodeV2 struct {
+	any                 `collection:"nodes"`
+	BaseModelV2[NodeV2] `bson:",inline"`
+	Key                 string    `json:"key" bson:"key"`
+	Name                string    `json:"name" bson:"name"`
+	Ip                  string    `json:"ip" bson:"ip"`
+	Port                string    `json:"port" bson:"port"`
+	Mac                 string    `json:"mac" bson:"mac"`
+	Hostname            string    `json:"hostname" bson:"hostname"`
+	Description         string    `json:"description" bson:"description"`
+	IsMaster            bool      `json:"is_master" bson:"is_master"`
+	Status              string    `json:"status" bson:"status"`
+	Enabled             bool      `json:"enabled" bson:"enabled"`
+	Active              bool      `json:"active" bson:"active"`
+	ActiveAt            time.Time `json:"active_at" bson:"active_ts"`
+	AvailableRunners    int       `json:"available_runners" bson:"available_runners"`
+	MaxRunners          int       `json:"max_runners" bson:"max_runners"`
+}

core/models/models/notification_setting_v2.go

@@ -0,0 +1,32 @@
+package models
+
+import "go.mongodb.org/mongo-driver/bson/primitive"
+
+type NotificationSettingV2 struct {
+	Id          primitive.ObjectID        `json:"_id" bson:"_id"`
+	Type        string                    `json:"type" bson:"type"`
+	Name        string                    `json:"name" bson:"name"`
+	Description string                    `json:"description" bson:"description"`
+	Enabled     bool                      `json:"enabled" bson:"enabled"`
+	Global      bool                      `json:"global" bson:"global"`
+	Title       string                    `json:"title,omitempty" bson:"title,omitempty"`
+	Template    string                    `json:"template,omitempty" bson:"template,omitempty"`
+	TaskTrigger string                    `json:"task_trigger" bson:"task_trigger"`
+	Mail        NotificationSettingMail   `json:"mail,omitempty" bson:"mail,omitempty"`
+	Mobile      NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"`
+}
+
+type NotificationSettingMail struct {
+	Server         string `json:"server" bson:"server"`
+	Port           string `json:"port,omitempty" bson:"port,omitempty"`
+	User           string `json:"user,omitempty" bson:"user,omitempty"`
+	Password       string `json:"password,omitempty" bson:"password,omitempty"`
+	SenderEmail    string `json:"sender_email,omitempty" bson:"sender_email,omitempty"`
+	SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"`
+	To             string `json:"to,omitempty" bson:"to,omitempty"`
+	Cc             string `json:"cc,omitempty" bson:"cc,omitempty"`
+}
+
+type NotificationSettingMobile struct {
+	Webhook string `json:"webhook" bson:"webhook"`
+}

core/models/models/password.go

@@ -0,0 +1,29 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Password struct {
+	Id       primitive.ObjectID `json:"_id" bson:"_id"`
+	Password string             `json:"password" bson:"p"`
+}
+
+func (p *Password) GetId() (id primitive.ObjectID) {
+	return p.Id
+}
+
+func (p *Password) SetId(id primitive.ObjectID) {
+	p.Id = id
+}
+
+type PasswordList []Password
+
+func (l *PasswordList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/permission.go

@@ -0,0 +1,91 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Permission struct {
+	Id          primitive.ObjectID `json:"_id" bson:"_id"`
+	Key         string             `json:"key" bson:"key"`
+	Name        string             `json:"name" bson:"name"`
+	Description string             `json:"description" bson:"description"`
+	Type        string             `json:"type" bson:"type"`
+	Target      []string           `json:"target" bson:"target"`
+	Allow       []string           `json:"allow" bson:"allow"`
+	Deny        []string           `json:"deny" bson:"deny"`
+}
+
+func (p *Permission) GetId() (id primitive.ObjectID) {
+	return p.Id
+}
+
+func (p *Permission) SetId(id primitive.ObjectID) {
+	p.Id = id
+}
+
+func (p *Permission) GetKey() (key string) {
+	return p.Key
+}
+
+func (p *Permission) SetKey(key string) {
+	p.Key = key
+}
+
+func (p *Permission) GetName() (name string) {
+	return p.Name
+}
+
+func (p *Permission) SetName(name string) {
+	p.Name = name
+}
+
+func (p *Permission) GetDescription() (description string) {
+	return p.Description
+}
+
+func (p *Permission) SetDescription(description string) {
+	p.Description = description
+}
+
+func (p *Permission) GetType() (t string) {
+	return p.Type
+}
+
+func (p *Permission) SetType(t string) {
+	p.Type = t
+}
+
+func (p *Permission) GetTarget() (target []string) {
+	return p.Target
+}
+
+func (p *Permission) SetTarget(target []string) {
+	p.Target = target
+}
+
+func (p *Permission) GetAllow() (include []string) {
+	return p.Allow
+}
+
+func (p *Permission) SetAllow(include []string) {
+	p.Allow = include
+}
+
+func (p *Permission) GetDeny() (exclude []string) {
+	return p.Deny
+}
+
+func (p *Permission) SetDeny(exclude []string) {
+	p.Deny = exclude
+}
+
+type PermissionList []Permission
+
+func (l *PermissionList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/permission_v2.go

@@ -0,0 +1,13 @@
+package models
+
+type PermissionV2 struct {
+	any                       `collection:"permissions"`
+	BaseModelV2[PermissionV2] `bson:",inline"`
+	Key                       string   `json:"key" bson:"key"`
+	Name                      string   `json:"name" bson:"name"`
+	Description               string   `json:"description" bson:"description"`
+	Type                      string   `json:"type" bson:"type"`
+	Target                    []string `json:"target" bson:"target"`
+	Allow                     []string `json:"allow" bson:"allow"`
+	Deny                      []string `json:"deny" bson:"deny"`
+}

core/models/models/project.go

@@ -0,0 +1,47 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Project struct {
+	Id          primitive.ObjectID `json:"_id" bson:"_id"`
+	Name        string             `json:"name" bson:"name"`
+	Description string             `json:"description" bson:"description"`
+	Spiders     int                `json:"spiders" bson:"-"`
+}
+
+func (p *Project) GetId() (id primitive.ObjectID) {
+	return p.Id
+}
+
+func (p *Project) SetId(id primitive.ObjectID) {
+	p.Id = id
+}
+
+func (p *Project) GetName() (name string) {
+	return p.Name
+}
+
+func (p *Project) SetName(name string) {
+	p.Name = name
+}
+
+func (p *Project) GetDescription() (description string) {
+	return p.Description
+}
+
+func (p *Project) SetDescription(description string) {
+	p.Description = description
+}
+
+type ProjectList []Project
+
+func (l *ProjectList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/project_v2.go

@@ -0,0 +1,9 @@
+package models
+
+type ProjectV2 struct {
+	any                    `collection:"projects"`
+	BaseModelV2[ProjectV2] `bson:",inline"`
+	Name                   string `json:"name" bson:"name"`
+	Description            string `json:"description" bson:"description"`
+	Spiders                int    `json:"spiders" bson:"-"`
+}

core/models/models/result.go

@@ -0,0 +1,60 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Result bson.M
+
+func (r *Result) GetId() (id primitive.ObjectID) {
+	res, ok := r.Value()["_id"]
+	if ok {
+		id, ok = res.(primitive.ObjectID)
+		if ok {
+			return id
+		}
+	}
+	return id
+}
+
+func (r *Result) SetId(id primitive.ObjectID) {
+	(*r)["_id"] = id
+}
+
+func (r *Result) Value() map[string]interface{} {
+	return *r
+}
+
+func (r *Result) SetValue(key string, value interface{}) {
+	(*r)[key] = value
+}
+
+func (r *Result) GetValue(key string) (value interface{}) {
+	return (*r)[key]
+}
+
+func (r *Result) GetTaskId() (id primitive.ObjectID) {
+	res := r.GetValue(constants.TaskKey)
+	if res == nil {
+		return id
+	}
+	id, _ = res.(primitive.ObjectID)
+	return id
+}
+
+func (r *Result) SetTaskId(id primitive.ObjectID) {
+	r.SetValue(constants.TaskKey, id)
+}
+
+type ResultList []Result
+
+func (l *ResultList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/role.go

@@ -0,0 +1,55 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Role struct {
+	Id          primitive.ObjectID `json:"_id" bson:"_id"`
+	Key         string             `json:"key" bson:"key"`
+	Name        string             `json:"name" bson:"name"`
+	Description string             `json:"description" bson:"description"`
+}
+
+func (r *Role) GetId() (id primitive.ObjectID) {
+	return r.Id
+}
+
+func (r *Role) SetId(id primitive.ObjectID) {
+	r.Id = id
+}
+
+func (r *Role) GetKey() (key string) {
+	return r.Key
+}
+
+func (r *Role) SetKey(key string) {
+	r.Key = key
+}
+
+func (r *Role) GetName() (name string) {
+	return r.Name
+}
+
+func (r *Role) SetName(name string) {
+	r.Name = name
+}
+
+func (r *Role) GetDescription() (description string) {
+	return r.Description
+}
+
+func (r *Role) SetDescription(description string) {
+	r.Description = description
+}
+
+type RoleList []Role
+
+func (l *RoleList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/role_permission.go

@@ -0,0 +1,30 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type RolePermission struct {
+	Id           primitive.ObjectID `json:"_id" bson:"_id"`
+	RoleId       primitive.ObjectID `json:"role_id" bson:"role_id"`
+	PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"`
+}
+
+func (ur *RolePermission) GetId() (id primitive.ObjectID) {
+	return ur.Id
+}
+
+func (ur *RolePermission) SetId(id primitive.ObjectID) {
+	ur.Id = id
+}
+
+type RolePermissionList []RolePermission
+
+func (l *RolePermissionList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/role_permission_v2.go

@@ -0,0 +1,12 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type RolePermissionV2 struct {
+	any                           `collection:"role_permissions"`
+	BaseModelV2[RolePermissionV2] `bson:",inline"`
+	RoleId                        primitive.ObjectID `json:"role_id" bson:"role_id"`
+	PermissionId                  primitive.ObjectID `json:"permission_id" bson:"permission_id"`
+}

core/models/models/role_v2.go

@@ -0,0 +1,9 @@
+package models
+
+type RoleV2 struct {
+	any                 `collection:"roles"`
+	BaseModelV2[RoleV2] `bson:",inline"`
+	Key                 string `json:"key" bson:"key"`
+	Name                string `json:"name" bson:"name"`
+	Description         string `json:"description" bson:"description"`
+}

core/models/models/schedule.go

@@ -0,0 +1,113 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/robfig/cron/v3"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Schedule struct {
+	Id          primitive.ObjectID   `json:"_id" bson:"_id"`
+	Name        string               `json:"name" bson:"name"`
+	Description string               `json:"description" bson:"description"`
+	SpiderId    primitive.ObjectID   `json:"spider_id" bson:"spider_id"`
+	Cron        string               `json:"cron" bson:"cron"`
+	EntryId     cron.EntryID         `json:"entry_id" bson:"entry_id"`
+	Cmd         string               `json:"cmd" bson:"cmd"`
+	Param       string               `json:"param" bson:"param"`
+	Mode        string               `json:"mode" bson:"mode"`
+	NodeIds     []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
+	Priority    int                  `json:"priority" bson:"priority"`
+	Enabled     bool                 `json:"enabled" bson:"enabled"`
+	UserId      primitive.ObjectID   `json:"user_id" bson:"user_id"`
+}
+
+func (s *Schedule) GetId() (id primitive.ObjectID) {
+	return s.Id
+}
+
+func (s *Schedule) SetId(id primitive.ObjectID) {
+	s.Id = id
+}
+
+func (s *Schedule) GetEnabled() (enabled bool) {
+	return s.Enabled
+}
+
+func (s *Schedule) SetEnabled(enabled bool) {
+	s.Enabled = enabled
+}
+
+func (s *Schedule) GetEntryId() (id cron.EntryID) {
+	return s.EntryId
+}
+
+func (s *Schedule) SetEntryId(id cron.EntryID) {
+	s.EntryId = id
+}
+
+func (s *Schedule) GetCron() (c string) {
+	return s.Cron
+}
+
+func (s *Schedule) SetCron(c string) {
+	s.Cron = c
+}
+
+func (s *Schedule) GetSpiderId() (id primitive.ObjectID) {
+	return s.SpiderId
+}
+
+func (s *Schedule) SetSpiderId(id primitive.ObjectID) {
+	s.SpiderId = id
+}
+
+func (s *Schedule) GetMode() (mode string) {
+	return s.Mode
+}
+
+func (s *Schedule) SetMode(mode string) {
+	s.Mode = mode
+}
+
+func (s *Schedule) GetNodeIds() (ids []primitive.ObjectID) {
+	return s.NodeIds
+}
+
+func (s *Schedule) SetNodeIds(ids []primitive.ObjectID) {
+	s.NodeIds = ids
+}
+
+func (s *Schedule) GetCmd() (cmd string) {
+	return s.Cmd
+}
+
+func (s *Schedule) SetCmd(cmd string) {
+	s.Cmd = cmd
+}
+
+func (s *Schedule) GetParam() (param string) {
+	return s.Param
+}
+
+func (s *Schedule) SetParam(param string) {
+	s.Param = param
+}
+
+func (s *Schedule) GetPriority() (p int) {
+	return s.Priority
+}
+
+func (s *Schedule) SetPriority(p int) {
+	s.Priority = p
+}
+
+type ScheduleList []Schedule
+
+func (l *ScheduleList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/schedule_v2.go

@@ -0,0 +1,23 @@
+package models
+
+import (
+	"github.com/robfig/cron/v3"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type ScheduleV2 struct {
+	any                     `collection:"schedules"`
+	BaseModelV2[ScheduleV2] `bson:",inline"`
+	Name                    string               `json:"name" bson:"name"`
+	Description             string               `json:"description" bson:"description"`
+	SpiderId                primitive.ObjectID   `json:"spider_id" bson:"spider_id"`
+	Cron                    string               `json:"cron" bson:"cron"`
+	EntryId                 cron.EntryID         `json:"entry_id" bson:"entry_id"`
+	Cmd                     string               `json:"cmd" bson:"cmd"`
+	Param                   string               `json:"param" bson:"param"`
+	Mode                    string               `json:"mode" bson:"mode"`
+	NodeIds                 []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
+	Priority                int                  `json:"priority" bson:"priority"`
+	Enabled                 bool                 `json:"enabled" bson:"enabled"`
+	UserId                  primitive.ObjectID   `json:"user_id" bson:"user_id"`
+}

core/models/models/setting.go

@@ -0,0 +1,31 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Setting struct {
+	Id    primitive.ObjectID `json:"_id" bson:"_id"`
+	Key   string             `json:"key" bson:"key"`
+	Value bson.M             `json:"value" bson:"value"`
+}
+
+func (s *Setting) GetId() (id primitive.ObjectID) {
+	return s.Id
+}
+
+func (s *Setting) SetId(id primitive.ObjectID) {
+	s.Id = id
+}
+
+type SettingList []Setting
+
+func (l *SettingList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/setting_v2.go

@@ -0,0 +1,12 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+type SettingV2 struct {
+	any                    `collection:"settings"`
+	BaseModelV2[SettingV2] `bson:",inline"`
+	Key                    string `json:"key" bson:"key"`
+	Value                  bson.M `json:"value" bson:"value"`
+}

core/models/models/spider.go

@@ -0,0 +1,137 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Env struct {
+	Name  string `json:"name" bson:"name"`
+	Value string `json:"value" bson:"value"`
+}
+
+type Spider struct {
+	Id           primitive.ObjectID   `json:"_id" bson:"_id"`                       // spider id
+	Name         string               `json:"name" bson:"name"`                     // spider name
+	Type         string               `json:"type" bson:"type"`                     // spider type
+	ColId        primitive.ObjectID   `json:"col_id" bson:"col_id"`                 // data collection id
+	ColName      string               `json:"col_name,omitempty" bson:"-"`          // data collection name
+	DataSourceId primitive.ObjectID   `json:"data_source_id" bson:"data_source_id"` // data source id
+	DataSource   *DataSource          `json:"data_source,omitempty" bson:"-"`       // data source
+	Description  string               `json:"description" bson:"description"`       // description
+	ProjectId    primitive.ObjectID   `json:"project_id" bson:"project_id"`         // Project.Id
+	Mode         string               `json:"mode" bson:"mode"`                     // default Task.Mode
+	NodeIds      []primitive.ObjectID `json:"node_ids" bson:"node_ids"`             // default Task.NodeIds
+	Stat         *SpiderStat          `json:"stat,omitempty" bson:"-"`
+
+	// execution
+	Cmd         string `json:"cmd" bson:"cmd"`     // execute command
+	Param       string `json:"param" bson:"param"` // default task param
+	Priority    int    `json:"priority" bson:"priority"`
+	AutoInstall bool   `json:"auto_install" bson:"auto_install"`
+
+	// settings
+	IncrementalSync bool `json:"incremental_sync" bson:"incremental_sync"` // whether to incrementally sync files
+}
+
+func (s *Spider) GetId() (id primitive.ObjectID) {
+	return s.Id
+}
+
+func (s *Spider) SetId(id primitive.ObjectID) {
+	s.Id = id
+}
+
+func (s *Spider) GetName() (name string) {
+	return s.Name
+}
+
+func (s *Spider) SetName(name string) {
+	s.Name = name
+}
+
+func (s *Spider) GetDescription() (description string) {
+	return s.Description
+}
+
+func (s *Spider) SetDescription(description string) {
+	s.Description = description
+}
+
+func (s *Spider) GetType() (ty string) {
+	return s.Type
+}
+
+func (s *Spider) GetMode() (mode string) {
+	return s.Mode
+}
+
+func (s *Spider) SetMode(mode string) {
+	s.Mode = mode
+}
+
+func (s *Spider) GetNodeIds() (ids []primitive.ObjectID) {
+	return s.NodeIds
+}
+
+func (s *Spider) SetNodeIds(ids []primitive.ObjectID) {
+	s.NodeIds = ids
+}
+
+func (s *Spider) GetCmd() (cmd string) {
+	return s.Cmd
+}
+
+func (s *Spider) SetCmd(cmd string) {
+	s.Cmd = cmd
+}
+
+func (s *Spider) GetParam() (param string) {
+	return s.Param
+}
+
+func (s *Spider) SetParam(param string) {
+	s.Param = param
+}
+
+func (s *Spider) GetPriority() (p int) {
+	return s.Priority
+}
+
+func (s *Spider) SetPriority(p int) {
+	s.Priority = p
+}
+
+func (s *Spider) GetColId() (id primitive.ObjectID) {
+	return s.ColId
+}
+
+func (s *Spider) SetColId(id primitive.ObjectID) {
+	s.ColId = id
+}
+
+func (s *Spider) GetIncrementalSync() (incrementalSync bool) {
+	return s.IncrementalSync
+}
+
+func (s *Spider) SetIncrementalSync(incrementalSync bool) {
+	s.IncrementalSync = incrementalSync
+}
+
+func (s *Spider) GetAutoInstall() (autoInstall bool) {
+	return s.AutoInstall
+}
+
+func (s *Spider) SetAutoInstall(autoInstall bool) {
+	s.AutoInstall = autoInstall
+}
+
+type SpiderList []Spider
+
+func (l *SpiderList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/spider_stat.go

@@ -0,0 +1,38 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type SpiderStat struct {
+	Id                     primitive.ObjectID `json:"_id" bson:"_id"`
+	LastTaskId             primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"`
+	LastTask               *Task              `json:"last_task,omitempty" bson:"-"`
+	Tasks                  int                `json:"tasks" bson:"tasks"`
+	Results                int                `json:"results" bson:"results"`
+	WaitDuration           int64              `json:"wait_duration" bson:"wait_duration,omitempty"`       // in second
+	RuntimeDuration        int64              `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second
+	TotalDuration          int64              `json:"total_duration" bson:"total_duration,omitempty"`     // in second
+	AverageWaitDuration    int64              `json:"average_wait_duration" bson:"-"`                     // in second
+	AverageRuntimeDuration int64              `json:"average_runtime_duration" bson:"-"`                  // in second
+	AverageTotalDuration   int64              `json:"average_total_duration" bson:"-"`                    // in second
+}
+
+func (s *SpiderStat) GetId() (id primitive.ObjectID) {
+	return s.Id
+}
+
+func (s *SpiderStat) SetId(id primitive.ObjectID) {
+	s.Id = id
+}
+
+type SpiderStatList []SpiderStat
+
+func (l *SpiderStatList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/spider_stat_v2.go

@@ -0,0 +1,20 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type SpiderStatV2 struct {
+	any                       `collection:"spider_stats"`
+	BaseModelV2[SpiderStatV2] `bson:",inline"`
+	LastTaskId                primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"`
+	LastTask                  *TaskV2            `json:"last_task,omitempty" bson:"-"`
+	Tasks                     int                `json:"tasks" bson:"tasks"`
+	Results                   int                `json:"results" bson:"results"`
+	WaitDuration              int64              `json:"wait_duration" bson:"wait_duration,omitempty"`       // in second
+	RuntimeDuration           int64              `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second
+	TotalDuration             int64              `json:"total_duration" bson:"total_duration,omitempty"`     // in second
+	AverageWaitDuration       int64              `json:"average_wait_duration" bson:"-"`                     // in second
+	AverageRuntimeDuration    int64              `json:"average_runtime_duration" bson:"-"`                  // in second
+	AverageTotalDuration      int64              `json:"average_total_duration" bson:"-"`                    // in second
+}

core/models/models/spider_v2.go

@@ -0,0 +1,30 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type SpiderV2 struct {
+	any                   `collection:"spiders"` // spider id
+	BaseModelV2[SpiderV2] `bson:",inline"`
+	Name                  string               `json:"name" bson:"name"`                     // spider name
+	Type                  string               `json:"type" bson:"type"`                     // spider type
+	ColId                 primitive.ObjectID   `json:"col_id" bson:"col_id"`                 // data collection id
+	ColName               string               `json:"col_name,omitempty" bson:"-"`          // data collection name
+	DataSourceId          primitive.ObjectID   `json:"data_source_id" bson:"data_source_id"` // data source id
+	DataSource            *DataSourceV2        `json:"data_source,omitempty" bson:"-"`       // data source
+	Description           string               `json:"description" bson:"description"`       // description
+	ProjectId             primitive.ObjectID   `json:"project_id" bson:"project_id"`         // Project.Id
+	Mode                  string               `json:"mode" bson:"mode"`                     // default Task.Mode
+	NodeIds               []primitive.ObjectID `json:"node_ids" bson:"node_ids"`             // default Task.NodeIds
+	Stat                  *SpiderStatV2        `json:"stat,omitempty" bson:"-"`
+
+	// execution
+	Cmd         string `json:"cmd" bson:"cmd"`     // execute command
+	Param       string `json:"param" bson:"param"` // default task param
+	Priority    int    `json:"priority" bson:"priority"`
+	AutoInstall bool   `json:"auto_install" bson:"auto_install"`
+
+	// settings
+	IncrementalSync bool `json:"incremental_sync" bson:"incremental_sync"` // whether to incrementally sync files
+}

core/models/models/tag.go

@@ -0,0 +1,44 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Tag struct {
+	Id          primitive.ObjectID `json:"_id" bson:"_id"`
+	Name        string             `json:"name" bson:"name"`
+	Color       string             `json:"color" bson:"color"`
+	Description string             `json:"description" bson:"description"`
+	Col         string             `json:"col" bson:"col"`
+}
+
+func (t *Tag) GetId() (id primitive.ObjectID) {
+	return t.Id
+}
+
+func (t *Tag) SetId(id primitive.ObjectID) {
+	t.Id = id
+}
+
+func (t *Tag) GetName() (res string) {
+	return t.Name
+}
+
+func (t *Tag) GetColor() (res string) {
+	return t.Color
+}
+
+func (t *Tag) SetCol(col string) {
+	t.Col = col
+}
+
+type TagList []Tag
+
+func (l *TagList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/task.go

@@ -0,0 +1,118 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type Task struct {
+	Id         primitive.ObjectID   `json:"_id" bson:"_id"`
+	SpiderId   primitive.ObjectID   `json:"spider_id" bson:"spider_id"`
+	Status     string               `json:"status" bson:"status"`
+	NodeId     primitive.ObjectID   `json:"node_id" bson:"node_id"`
+	Cmd        string               `json:"cmd" bson:"cmd"`
+	Param      string               `json:"param" bson:"param"`
+	Error      string               `json:"error" bson:"error"`
+	Pid        int                  `json:"pid" bson:"pid"`
+	ScheduleId primitive.ObjectID   `json:"schedule_id" bson:"schedule_id"` // Schedule.Id
+	Type       string               `json:"type" bson:"type"`
+	Mode       string               `json:"mode" bson:"mode"`           // running mode of Task
+	NodeIds    []primitive.ObjectID `json:"node_ids" bson:"node_ids"`   // list of Node.Id
+	ParentId   primitive.ObjectID   `json:"parent_id" bson:"parent_id"` // parent Task.Id if it'Spider a sub-task
+	Priority   int                  `json:"priority" bson:"priority"`
+	Stat       *TaskStat            `json:"stat,omitempty" bson:"-"`
+	HasSub     bool                 `json:"has_sub" json:"has_sub"` // whether to have sub-tasks
+	SubTasks   []Task               `json:"sub_tasks,omitempty" bson:"-"`
+	Spider     *Spider              `json:"spider,omitempty" bson:"-"`
+	UserId     primitive.ObjectID   `json:"-" bson:"-"`
+	CreateTs   time.Time            `json:"create_ts" bson:"create_ts"`
+}
+
+func (t *Task) GetId() (id primitive.ObjectID) {
+	return t.Id
+}
+
+func (t *Task) SetId(id primitive.ObjectID) {
+	t.Id = id
+}
+
+func (t *Task) GetNodeId() (id primitive.ObjectID) {
+	return t.NodeId
+}
+
+func (t *Task) SetNodeId(id primitive.ObjectID) {
+	t.NodeId = id
+}
+
+func (t *Task) GetNodeIds() (ids []primitive.ObjectID) {
+	return t.NodeIds
+}
+
+func (t *Task) GetStatus() (status string) {
+	return t.Status
+}
+
+func (t *Task) SetStatus(status string) {
+	t.Status = status
+}
+
+func (t *Task) GetError() (error string) {
+	return t.Error
+}
+
+func (t *Task) SetError(error string) {
+	t.Error = error
+}
+
+func (t *Task) GetPid() (pid int) {
+	return t.Pid
+}
+
+func (t *Task) SetPid(pid int) {
+	t.Pid = pid
+}
+
+func (t *Task) GetSpiderId() (id primitive.ObjectID) {
+	return t.SpiderId
+}
+
+func (t *Task) GetType() (ty string) {
+	return t.Type
+}
+
+func (t *Task) GetCmd() (cmd string) {
+	return t.Cmd
+}
+
+func (t *Task) GetParam() (param string) {
+	return t.Param
+}
+
+func (t *Task) GetPriority() (p int) {
+	return t.Priority
+}
+
+func (t *Task) GetUserId() (id primitive.ObjectID) {
+	return t.UserId
+}
+
+func (t *Task) SetUserId(id primitive.ObjectID) {
+	t.UserId = id
+}
+
+type TaskList []Task
+
+func (l *TaskList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}
+
+type TaskDailyItem struct {
+	Date               string  `json:"date" bson:"_id"`
+	TaskCount          int     `json:"task_count" bson:"task_count"`
+	AvgRuntimeDuration float64 `json:"avg_runtime_duration" bson:"avg_runtime_duration"`
+}

core/models/models/task_queue_item.go

@@ -0,0 +1,30 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type TaskQueueItem struct {
+	Id       primitive.ObjectID `json:"_id" bson:"_id"`
+	Priority int                `json:"p" bson:"p"`
+	NodeId   primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"`
+}
+
+func (t *TaskQueueItem) GetId() (id primitive.ObjectID) {
+	return t.Id
+}
+
+func (t *TaskQueueItem) SetId(id primitive.ObjectID) {
+	t.Id = id
+}
+
+type TaskQueueItemList []TaskQueueItem
+
+func (l *TaskQueueItemList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/task_queue_item_v2.go

@@ -0,0 +1,12 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type TaskQueueItemV2 struct {
+	any                          `collection:"task_queue"`
+	BaseModelV2[TaskQueueItemV2] `bson:",inline"`
+	Priority                     int                `json:"p" bson:"p"`
+	NodeId                       primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"`
+}

core/models/models/task_stat.go

@@ -0,0 +1,101 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type TaskStat struct {
+	Id              primitive.ObjectID `json:"_id" bson:"_id"`
+	CreateTs        time.Time          `json:"create_ts" bson:"create_ts,omitempty"`
+	StartTs         time.Time          `json:"start_ts" bson:"start_ts,omitempty"`
+	EndTs           time.Time          `json:"end_ts" bson:"end_ts,omitempty"`
+	WaitDuration    int64              `json:"wait_duration" bson:"wait_duration,omitempty"`       // in millisecond
+	RuntimeDuration int64              `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond
+	TotalDuration   int64              `json:"total_duration" bson:"total_duration,omitempty"`     // in millisecond
+	ResultCount     int64              `json:"result_count" bson:"result_count"`
+	ErrorLogCount   int64              `json:"error_log_count" bson:"error_log_count"`
+}
+
+func (s *TaskStat) GetId() (id primitive.ObjectID) {
+	return s.Id
+}
+
+func (s *TaskStat) SetId(id primitive.ObjectID) {
+	s.Id = id
+}
+
+func (s *TaskStat) GetCreateTs() (ts time.Time) {
+	return s.CreateTs
+}
+
+func (s *TaskStat) SetCreateTs(ts time.Time) {
+	s.CreateTs = ts
+}
+
+func (s *TaskStat) GetStartTs() (ts time.Time) {
+	return s.StartTs
+}
+
+func (s *TaskStat) SetStartTs(ts time.Time) {
+	s.StartTs = ts
+}
+
+func (s *TaskStat) GetEndTs() (ts time.Time) {
+	return s.EndTs
+}
+
+func (s *TaskStat) SetEndTs(ts time.Time) {
+	s.EndTs = ts
+}
+
+func (s *TaskStat) GetWaitDuration() (d int64) {
+	return s.WaitDuration
+}
+
+func (s *TaskStat) SetWaitDuration(d int64) {
+	s.WaitDuration = d
+}
+
+func (s *TaskStat) GetRuntimeDuration() (d int64) {
+	return s.RuntimeDuration
+}
+
+func (s *TaskStat) SetRuntimeDuration(d int64) {
+	s.RuntimeDuration = d
+}
+
+func (s *TaskStat) GetTotalDuration() (d int64) {
+	return s.WaitDuration + s.RuntimeDuration
+}
+
+func (s *TaskStat) SetTotalDuration(d int64) {
+	s.TotalDuration = d
+}
+
+func (s *TaskStat) GetResultCount() (c int64) {
+	return s.ResultCount
+}
+
+func (s *TaskStat) SetResultCount(c int64) {
+	s.ResultCount = c
+}
+
+func (s *TaskStat) GetErrorLogCount() (c int64) {
+	return s.ErrorLogCount
+}
+
+func (s *TaskStat) SetErrorLogCount(c int64) {
+	s.ErrorLogCount = c
+}
+
+type TaskStatList []TaskStat
+
+func (l *TaskStatList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/task_stat_v2.go

@@ -0,0 +1,18 @@
+package models
+
+import (
+	"time"
+)
+
+type TaskStatV2 struct {
+	any                     `collection:"task_stats"`
+	BaseModelV2[TaskStatV2] `bson:",inline"`
+	CreateTs                time.Time `json:"create_ts" bson:"create_ts,omitempty"`
+	StartTs                 time.Time `json:"start_ts" bson:"start_ts,omitempty"`
+	EndTs                   time.Time `json:"end_ts" bson:"end_ts,omitempty"`
+	WaitDuration            int64     `json:"wait_duration" bson:"wait_duration,omitempty"`       // in millisecond
+	RuntimeDuration         int64     `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond
+	TotalDuration           int64     `json:"total_duration" bson:"total_duration,omitempty"`     // in millisecond
+	ResultCount             int64     `json:"result_count" bson:"result_count"`
+	ErrorLogCount           int64     `json:"error_log_count" bson:"error_log_count"`
+}

core/models/models/task_v2.go

@@ -0,0 +1,30 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type TaskV2 struct {
+	any                 `collection:"tasks"`
+	BaseModelV2[TaskV2] `bson:",inline"`
+	SpiderId            primitive.ObjectID   `json:"spider_id" bson:"spider_id"`
+	Status              string               `json:"status" bson:"status"`
+	NodeId              primitive.ObjectID   `json:"node_id" bson:"node_id"`
+	Cmd                 string               `json:"cmd" bson:"cmd"`
+	Param               string               `json:"param" bson:"param"`
+	Error               string               `json:"error" bson:"error"`
+	Pid                 int                  `json:"pid" bson:"pid"`
+	ScheduleId          primitive.ObjectID   `json:"schedule_id" bson:"schedule_id"`
+	Type                string               `json:"type" bson:"type"`
+	Mode                string               `json:"mode" bson:"mode"`
+	NodeIds             []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
+	ParentId            primitive.ObjectID   `json:"parent_id" bson:"parent_id"`
+	Priority            int                  `json:"priority" bson:"priority"`
+	Stat                *TaskStatV2          `json:"stat,omitempty" bson:"-"`
+	HasSub              bool                 `json:"has_sub" json:"has_sub"`
+	SubTasks            []TaskV2             `json:"sub_tasks,omitempty" bson:"-"`
+	Spider              *SpiderV2            `json:"spider,omitempty" bson:"-"`
+	UserId              primitive.ObjectID   `json:"-" bson:"-"`
+	CreateTs            time.Time            `json:"create_ts" bson:"create_ts"`
+}

core/models/models/test.go

@@ -0,0 +1,7 @@
+package models
+
+type TestModel struct {
+	any                    `collection:"testmodels"`
+	BaseModelV2[TestModel] `bson:",inline"`
+	Name                   string `json:"name" bson:"name"`
+}

core/models/models/token.go

@@ -0,0 +1,30 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Token struct {
+	Id    primitive.ObjectID `json:"_id" bson:"_id"`
+	Name  string             `json:"name" bson:"name"`
+	Token string             `json:"token" bson:"token"`
+}
+
+func (t *Token) GetId() (id primitive.ObjectID) {
+	return t.Id
+}
+
+func (t *Token) SetId(id primitive.ObjectID) {
+	t.Id = id
+}
+
+type TokenList []Token
+
+func (l *TokenList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/token_v2.go

@@ -0,0 +1,8 @@
+package models
+
+type TokenV2 struct {
+	any                  `collection:"tokens"`
+	BaseModelV2[TokenV2] `bson:",inline"`
+	Name                 string `json:"name" bson:"name"`
+	Token                string `json:"token" bson:"token"`
+}

core/models/models/user.go

@@ -0,0 +1,59 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type User struct {
+	Id       primitive.ObjectID `json:"_id" bson:"_id"`
+	Username string             `json:"username" bson:"username"`
+	Password string             `json:"password,omitempty" bson:"-"`
+	Role     string             `json:"role" bson:"role"`
+	Email    string             `json:"email" bson:"email"`
+	//Setting  UserSetting        `json:"setting" bson:"setting"`
+}
+
+func (u *User) GetId() (id primitive.ObjectID) {
+	return u.Id
+}
+
+func (u *User) SetId(id primitive.ObjectID) {
+	u.Id = id
+}
+
+func (u *User) GetUsername() (name string) {
+	return u.Username
+}
+
+func (u *User) GetPassword() (p string) {
+	return u.Password
+}
+
+func (u *User) GetRole() (r string) {
+	return u.Role
+}
+
+func (u *User) GetEmail() (email string) {
+	return u.Email
+}
+
+//type UserSetting struct {
+//	NotificationTrigger  string   `json:"notification_trigger" bson:"notification_trigger"`
+//	DingTalkRobotWebhook string   `json:"ding_talk_robot_webhook" bson:"ding_talk_robot_webhook"`
+//	WechatRobotWebhook   string   `json:"wechat_robot_webhook" bson:"wechat_robot_webhook"`
+//	EnabledNotifications []string `json:"enabled_notifications" bson:"enabled_notifications"`
+//	ErrorRegexPattern    string   `json:"error_regex_pattern" bson:"error_regex_pattern"`
+//	MaxErrorLog          int      `json:"max_error_log" bson:"max_error_log"`
+//	LogExpireDuration    int64    `json:"log_expire_duration" bson:"log_expire_duration"`
+//}
+
+type UserList []User
+
+func (l *UserList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/user_role.go

@@ -0,0 +1,30 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type UserRole struct {
+	Id     primitive.ObjectID `json:"_id" bson:"_id"`
+	RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
+	UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
+}
+
+func (ur *UserRole) GetId() (id primitive.ObjectID) {
+	return ur.Id
+}
+
+func (ur *UserRole) SetId(id primitive.ObjectID) {
+	ur.Id = id
+}
+
+type UserRoleList []UserRole
+
+func (l *UserRoleList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/user_role_v2.go

@@ -0,0 +1,12 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type UserRoleV2 struct {
+	any                     `collection:"user_roles"`
+	BaseModelV2[UserRoleV2] `bson:",inline"`
+	RoleId                  primitive.ObjectID `json:"role_id" bson:"role_id"`
+	UserId                  primitive.ObjectID `json:"user_id" bson:"user_id"`
+}

core/models/models/user_v2.go

@@ -0,0 +1,10 @@
+package models
+
+type UserV2 struct {
+	any                 `collection:"users"`
+	BaseModelV2[UserV2] `bson:",inline"`
+	Username            string `json:"username" bson:"username"`
+	Password            string `json:"-,omitempty" bson:"password"`
+	Role                string `json:"role" bson:"role"`
+	Email               string `json:"email" bson:"email"`
+}

core/models/models/utils_binder_legacy.go

@@ -0,0 +1,96 @@
+package models
+
+//func AssignFields(d interface{}, fieldIds ...interfaces.ModelId) (res interface{}, err error) {
+//	return assignFields(d, fieldIds...)
+//}
+//
+//func AssignListFields(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
+//	return assignListFields(list, fieldIds...)
+//}
+//
+//func AssignListFieldsAsPtr(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
+//	return assignListFieldsAsPtr(list, fieldIds...)
+//}
+//
+//func assignFields(d interface{}, fieldIds ...interfaces.ModelId) (res interface{}, err error) {
+//	doc, ok := d.(interfaces.Model)
+//	if !ok {
+//		return nil, errors.ErrorModelInvalidType
+//	}
+//	if len(fieldIds) == 0 {
+//		return doc, nil
+//	}
+//	for _, fid := range fieldIds {
+//		switch fid {
+//		case interfaces.ModelIdTag:
+//			// convert interface
+//			d, ok := doc.(interfaces.ModelWithTags)
+//			if !ok {
+//				return nil, errors.ErrorModelInvalidType
+//			}
+//
+//			// attempt to get artifact
+//			a, err := doc.GetArtifact()
+//			if err != nil {
+//				return nil, err
+//			}
+//
+//			// skip if no artifact found
+//			if a == nil {
+//				return d, nil
+//			}
+//
+//			// assign tags
+//			tags, err := a.GetTags()
+//			if err != nil {
+//				return nil, err
+//			}
+//			d.SetTags(tags)
+//
+//			return d, nil
+//		}
+//	}
+//	return doc, nil
+//}
+//
+//func _assignListFields(asPtr bool, list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
+//	vList := reflect.ValueOf(list)
+//	if vList.Kind() != reflect.Array &&
+//		vList.Kind() != reflect.Slice {
+//		return res, errors.ErrorModelInvalidType
+//	}
+//	for i := 0; i < vList.Len(); i++ {
+//		vItem := vList.Index(i)
+//		var item interface{}
+//		if vItem.CanAddr() {
+//			item = vItem.Addr().Interface()
+//		} else {
+//			item = vItem.Interface()
+//		}
+//		doc, ok := item.(interfaces.Model)
+//		if !ok {
+//			return res, errors.ErrorModelInvalidType
+//		}
+//		ptr, err := assignFields(doc, fieldIds...)
+//		if err != nil {
+//			return res, err
+//		}
+//		v := reflect.ValueOf(ptr)
+//		if !asPtr {
+//			// non-pointer item
+//			res.Add(v.Elem().Interface())
+//		} else {
+//			// pointer item
+//			res.Add(v.Interface())
+//		}
+//	}
+//	return res, nil
+//}
+//
+//func assignListFields(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
+//	return _assignListFields(false, list, fieldIds...)
+//}
+//
+//func assignListFieldsAsPtr(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
+//	return _assignListFields(true, list, fieldIds...)
+//}

core/models/models/utils_col.go

@@ -0,0 +1,10 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/utils/binders"
+)
+
+func GetModelColName(id interfaces.ModelId) (colName string) {
+	return binders.NewColNameBinder(id).MustBindString()
+}

core/models/models/utils_model_map.go

@@ -0,0 +1,95 @@
+package models
+
+type ModelMap struct {
+	Artifact          Artifact
+	Tag               Tag
+	Node              Node
+	Project           Project
+	Spider            Spider
+	Task              Task
+	Job               Job
+	Schedule          Schedule
+	User              User
+	Setting           Setting
+	Token             Token
+	Variable          Variable
+	TaskQueueItem     TaskQueueItem
+	TaskStat          TaskStat
+	SpiderStat        SpiderStat
+	DataSource        DataSource
+	DataCollection    DataCollection
+	Result            Result
+	Password          Password
+	ExtraValue        ExtraValue
+	Git               Git
+	Role              Role
+	UserRole          UserRole
+	Permission        Permission
+	RolePermission    RolePermission
+	Environment       Environment
+	DependencySetting DependencySetting
+}
+
+type ModelListMap struct {
+	Artifacts          ArtifactList
+	Tags               TagList
+	Nodes              NodeList
+	Projects           ProjectList
+	Spiders            SpiderList
+	Tasks              TaskList
+	Jobs               JobList
+	Schedules          ScheduleList
+	Users              UserList
+	Settings           SettingList
+	Tokens             TokenList
+	Variables          VariableList
+	TaskQueueItems     TaskQueueItemList
+	TaskStats          TaskStatList
+	SpiderStats        SpiderStatList
+	DataSources        DataSourceList
+	DataCollections    DataCollectionList
+	Results            ResultList
+	Passwords          PasswordList
+	ExtraValues        ExtraValueList
+	Gits               GitList
+	Roles              RoleList
+	UserRoles          UserRoleList
+	PermissionList     PermissionList
+	RolePermissionList RolePermissionList
+	Environments       EnvironmentList
+	DependencySettings DependencySettingList
+}
+
+func NewModelMap() (m *ModelMap) {
+	return &ModelMap{}
+}
+
+func NewModelListMap() (m *ModelListMap) {
+	return &ModelListMap{
+		Artifacts:          ArtifactList{},
+		Tags:               TagList{},
+		Nodes:              NodeList{},
+		Projects:           ProjectList{},
+		Spiders:            SpiderList{},
+		Tasks:              TaskList{},
+		Jobs:               JobList{},
+		Schedules:          ScheduleList{},
+		Users:              UserList{},
+		Settings:           SettingList{},
+		Tokens:             TokenList{},
+		Variables:          VariableList{},
+		TaskQueueItems:     TaskQueueItemList{},
+		TaskStats:          TaskStatList{},
+		SpiderStats:        SpiderStatList{},
+		DataSources:        DataSourceList{},
+		DataCollections:    DataCollectionList{},
+		Results:            ResultList{},
+		Passwords:          PasswordList{},
+		ExtraValues:        ExtraValueList{},
+		Gits:               GitList{},
+		Roles:              RoleList{},
+		PermissionList:     PermissionList{},
+		RolePermissionList: RolePermissionList{},
+		Environments:       EnvironmentList{},
+	}
+}

core/models/models/utils_tag.go

@@ -0,0 +1,34 @@
+package models
+
+import (
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/go-trace"
+)
+
+func convertInterfacesToTags(tags []interfaces.Tag) (res []Tag) {
+	if tags == nil {
+		return nil
+	}
+	for _, t := range tags {
+		tag, ok := t.(*Tag)
+		if !ok {
+			log.Warnf("%v: cannot convert tag", trace.TraceError(errors.ErrorModelInvalidType))
+			return nil
+		}
+		if tag == nil {
+			log.Warnf("%v: cannot convert tag", trace.TraceError(errors.ErrorModelInvalidType))
+			return nil
+		}
+		res = append(res, *tag)
+	}
+	return res
+}
+
+func convertTagsToInterfaces(tags []Tag) (res []interfaces.Tag) {
+	for _, t := range tags {
+		res = append(res, &t)
+	}
+	return res
+}

core/models/models/variable.go

@@ -0,0 +1,31 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type Variable struct {
+	Id     primitive.ObjectID `json:"_id" bson:"_id"`
+	Key    string             `json:"key" bson:"key"`
+	Value  string             `json:"value" bson:"value"`
+	Remark string             `json:"remark" bson:"remark"`
+}
+
+func (v *Variable) GetId() (id primitive.ObjectID) {
+	return v.Id
+}
+
+func (v *Variable) SetId(id primitive.ObjectID) {
+	v.Id = id
+}
+
+type VariableList []Variable
+
+func (l *VariableList) GetModels() (res []interfaces.Model) {
+	for i := range *l {
+		d := (*l)[i]
+		res = append(res, &d)
+	}
+	return res
+}

core/models/models/variable_v2.go

@@ -0,0 +1,9 @@
+package models
+
+type VariableV2 struct {
+	any                     `collection:"variables"`
+	BaseModelV2[VariableV2] `bson:",inline"`
+	Key                     string `json:"key" bson:"key"`
+	Value                   string `json:"value" bson:"value"`
+	Remark                  string `json:"remark" bson:"remark"`
+}