refactor: removed unnecessary code

core/models/models/git_v2.go

@@ -1,26 +0,0 @@
-package models
-
-import (
-	"github.com/crawlab-team/crawlab/vcs"
-	"time"
-)
-
-type GitV2 struct {
-	any                `collection:"gits"`
-	BaseModelV2[GitV2] `bson:",inline"`
-	Url                string       `json:"url" bson:"url"`
-	Name               string       `json:"name" bson:"name"`
-	AuthType           string       `json:"auth_type" bson:"auth_type"`
-	Username           string       `json:"username" bson:"username"`
-	Password           string       `json:"password" bson:"password"`
-	CurrentBranch      string       `json:"current_branch" bson:"current_branch"`
-	Status             string       `json:"status" bson:"status"`
-	Error              string       `json:"error" bson:"error"`
-	Spiders            []SpiderV2   `json:"spiders,omitempty" bson:"-"`
-	Refs               []vcs.GitRef `json:"refs" bson:"refs"`
-	RefsUpdatedAt      time.Time    `json:"refs_updated_at" bson:"refs_updated_at"`
-	CloneLogs          []string     `json:"clone_logs,omitempty" bson:"clone_logs"`
-
-	// settings
-	AutoPull bool `json:"auto_pull" bson:"auto_pull"`
-}

core/models/models/metric_v2.go

@@ -1,23 +0,0 @@
-package models
-
-import "go.mongodb.org/mongo-driver/bson/primitive"
-
-type MetricV2 struct {
-	any                   `collection:"metrics"`
-	BaseModelV2[MetricV2] `bson:",inline"`
-	Type                  string             `json:"type" bson:"type"`
-	NodeId                primitive.ObjectID `json:"node_id" bson:"node_id"`
-	CpuUsagePercent       float32            `json:"cpu_usage_percent" bson:"cpu_usage_percent"`
-	TotalMemory           uint64             `json:"total_memory" bson:"total_memory"`
-	AvailableMemory       uint64             `json:"available_memory" bson:"available_memory"`
-	UsedMemory            uint64             `json:"used_memory" bson:"used_memory"`
-	UsedMemoryPercent     float32            `json:"used_memory_percent" bson:"used_memory_percent"`
-	TotalDisk             uint64             `json:"total_disk" bson:"total_disk"`
-	AvailableDisk         uint64             `json:"available_disk" bson:"available_disk"`
-	UsedDisk              uint64             `json:"used_disk" bson:"used_disk"`
-	UsedDiskPercent       float32            `json:"used_disk_percent" bson:"used_disk_percent"`
-	DiskReadBytesRate     float32            `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"`
-	DiskWriteBytesRate    float32            `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"`
-	NetworkBytesSentRate  float32            `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"`
-	NetworkBytesRecvRate  float32            `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"`
-}

core/models/models/node_v2.go

@@ -1,24 +0,0 @@
-package models
-
-import (
-	"time"
-)
-
-type NodeV2 struct {
-	any                 `collection:"nodes"`
-	BaseModelV2[NodeV2] `bson:",inline"`
-	Key                 string    `json:"key" bson:"key"`
-	Name                string    `json:"name" bson:"name"`
-	Ip                  string    `json:"ip" bson:"ip"`
-	Port                string    `json:"port" bson:"port"`
-	Mac                 string    `json:"mac" bson:"mac"`
-	Hostname            string    `json:"hostname" bson:"hostname"`
-	Description         string    `json:"description" bson:"description"`
-	IsMaster            bool      `json:"is_master" bson:"is_master"`
-	Status              string    `json:"status" bson:"status"`
-	Enabled             bool      `json:"enabled" bson:"enabled"`
-	Active              bool      `json:"active" bson:"active"`
-	ActiveAt            time.Time `json:"active_at" bson:"active_ts"`
-	AvailableRunners    int       `json:"available_runners" bson:"available_runners"`
-	MaxRunners          int       `json:"max_runners" bson:"max_runners"`
-}

core/models/models/notification_setting_v2.go

@@ -1,31 +0,0 @@
-package models
-
-type NotificationSettingV2 struct {
-	any                                `collection:"notification_settings"`
-	BaseModelV2[NotificationSettingV2] `bson:",inline"`
-	Type                               string                    `json:"type" bson:"type"`
-	Name                               string                    `json:"name" bson:"name"`
-	Description                        string                    `json:"description" bson:"description"`
-	Enabled                            bool                      `json:"enabled" bson:"enabled"`
-	Global                             bool                      `json:"global" bson:"global"`
-	Title                              string                    `json:"title,omitempty" bson:"title,omitempty"`
-	Template                           string                    `json:"template,omitempty" bson:"template,omitempty"`
-	TaskTrigger                        string                    `json:"task_trigger" bson:"task_trigger"`
-	Mail                               NotificationSettingMail   `json:"mail,omitempty" bson:"mail,omitempty"`
-	Mobile                             NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"`
-}
-
-type NotificationSettingMail struct {
-	Server         string `json:"server" bson:"server"`
-	Port           string `json:"port,omitempty" bson:"port,omitempty"`
-	User           string `json:"user,omitempty" bson:"user,omitempty"`
-	Password       string `json:"password,omitempty" bson:"password,omitempty"`
-	SenderEmail    string `json:"sender_email,omitempty" bson:"sender_email,omitempty"`
-	SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"`
-	To             string `json:"to,omitempty" bson:"to,omitempty"`
-	Cc             string `json:"cc,omitempty" bson:"cc,omitempty"`
-}
-
-type NotificationSettingMobile struct {
-	Webhook string `json:"webhook" bson:"webhook"`
-}

core/models/models/permission_v2.go

@@ -1,13 +0,0 @@
-package models
-
-type PermissionV2 struct {
-	any                       `collection:"permissions"`
-	BaseModelV2[PermissionV2] `bson:",inline"`
-	Key                       string   `json:"key" bson:"key"`
-	Name                      string   `json:"name" bson:"name"`
-	Description               string   `json:"description" bson:"description"`
-	Type                      string   `json:"type" bson:"type"`
-	Target                    []string `json:"target" bson:"target"`
-	Allow                     []string `json:"allow" bson:"allow"`
-	Deny                      []string `json:"deny" bson:"deny"`
-}

core/models/models/project_v2.go

@@ -1,9 +0,0 @@
-package models
-
-type ProjectV2 struct {
-	any                    `collection:"projects"`
-	BaseModelV2[ProjectV2] `bson:",inline"`
-	Name                   string `json:"name" bson:"name"`
-	Description            string `json:"description" bson:"description"`
-	Spiders                int    `json:"spiders" bson:"-"`
-}

core/models/models/role_permission_v2.go

@@ -1,12 +0,0 @@
-package models
-
-import (
-	"go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-type RolePermissionV2 struct {
-	any                           `collection:"role_permissions"`
-	BaseModelV2[RolePermissionV2] `bson:",inline"`
-	RoleId                        primitive.ObjectID `json:"role_id" bson:"role_id"`
-	PermissionId                  primitive.ObjectID `json:"permission_id" bson:"permission_id"`
-}

core/models/models/role_v2.go

@@ -1,9 +0,0 @@
-package models
-
-type RoleV2 struct {
-	any                 `collection:"roles"`
-	BaseModelV2[RoleV2] `bson:",inline"`
-	Key                 string `json:"key" bson:"key"`
-	Name                string `json:"name" bson:"name"`
-	Description         string `json:"description" bson:"description"`
-}

core/models/models/schedule_v2.go

@@ -1,23 +0,0 @@
-package models
-
-import (
-	"github.com/robfig/cron/v3"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-type ScheduleV2 struct {
-	any                     `collection:"schedules"`
-	BaseModelV2[ScheduleV2] `bson:",inline"`
-	Name                    string               `json:"name" bson:"name"`
-	Description             string               `json:"description" bson:"description"`
-	SpiderId                primitive.ObjectID   `json:"spider_id" bson:"spider_id"`
-	Cron                    string               `json:"cron" bson:"cron"`
-	EntryId                 cron.EntryID         `json:"entry_id" bson:"entry_id"`
-	Cmd                     string               `json:"cmd" bson:"cmd"`
-	Param                   string               `json:"param" bson:"param"`
-	Mode                    string               `json:"mode" bson:"mode"`
-	NodeIds                 []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
-	Priority                int                  `json:"priority" bson:"priority"`
-	Enabled                 bool                 `json:"enabled" bson:"enabled"`
-	UserId                  primitive.ObjectID   `json:"user_id" bson:"user_id"`
-}

core/models/models/setting_v2.go

@@ -1,12 +0,0 @@
-package models
-
-import (
-	"go.mongodb.org/mongo-driver/bson"
-)
-
-type SettingV2 struct {
-	any                    `collection:"settings"`
-	BaseModelV2[SettingV2] `bson:",inline"`
-	Key                    string `json:"key" bson:"key"`
-	Value                  bson.M `json:"value" bson:"value"`
-}

core/models/models/spider_stat_v2.go

@@ -1,20 +0,0 @@
-package models
-
-import (
-	"go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-type SpiderStatV2 struct {
-	any                       `collection:"spider_stats"`
-	BaseModelV2[SpiderStatV2] `bson:",inline"`
-	LastTaskId                primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"`
-	LastTask                  *TaskV2            `json:"last_task,omitempty" bson:"-"`
-	Tasks                     int                `json:"tasks" bson:"tasks"`
-	Results                   int                `json:"results" bson:"results"`
-	WaitDuration              int64              `json:"wait_duration" bson:"wait_duration,omitempty"`       // in second
-	RuntimeDuration           int64              `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second
-	TotalDuration             int64              `json:"total_duration" bson:"total_duration,omitempty"`     // in second
-	AverageWaitDuration       int64              `json:"average_wait_duration" bson:"-"`                     // in second
-	AverageRuntimeDuration    int64              `json:"average_runtime_duration" bson:"-"`                  // in second
-	AverageTotalDuration      int64              `json:"average_total_duration" bson:"-"`                    // in second
-}

core/models/models/spider_v2.go

@@ -1,32 +0,0 @@
-package models
-
-import (
-	"go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-type SpiderV2 struct {
-	any                   `collection:"spiders"`
-	BaseModelV2[SpiderV2] `bson:",inline"`
-	Name                  string               `json:"name" bson:"name"`                     // spider name
-	Type                  string               `json:"type" bson:"type"`                     // spider type
-	ColId                 primitive.ObjectID   `json:"col_id" bson:"col_id"`                 // data collection id
-	ColName               string               `json:"col_name,omitempty" bson:"-"`          // data collection name
-	DataSourceId          primitive.ObjectID   `json:"data_source_id" bson:"data_source_id"` // data source id
-	DataSource            *DataSourceV2        `json:"data_source,omitempty" bson:"-"`       // data source
-	Description           string               `json:"description" bson:"description"`       // description
-	ProjectId             primitive.ObjectID   `json:"project_id" bson:"project_id"`         // Project.Id
-	Mode                  string               `json:"mode" bson:"mode"`                     // default Task.Mode
-	NodeIds               []primitive.ObjectID `json:"node_ids" bson:"node_ids"`             // default Task.NodeIds
-	GitId                 primitive.ObjectID   `json:"git_id" bson:"git_id"`                 // related Git.Id
-	GitRootPath           string               `json:"git_root_path" bson:"git_root_path"`
-	Git                   *GitV2               `json:"git,omitempty" bson:"-"`
-
-	// stats
-	Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"`
-
-	// execution
-	Cmd         string `json:"cmd" bson:"cmd"`     // execute command
-	Param       string `json:"param" bson:"param"` // default task param
-	Priority    int    `json:"priority" bson:"priority"`
-	AutoInstall bool   `json:"auto_install" bson:"auto_install"`
-}

core/models/models/task_queue_item_v2.go

@@ -1,12 +0,0 @@
-package models
-
-import (
-	"go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-type TaskQueueItemV2 struct {
-	any                          `collection:"task_queue"`
-	BaseModelV2[TaskQueueItemV2] `bson:",inline"`
-	Priority                     int                `json:"p" bson:"p"`
-	NodeId                       primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"`
-}

core/models/models/task_stat_v2.go

@@ -1,18 +0,0 @@
-package models
-
-import (
-	"time"
-)
-
-type TaskStatV2 struct {
-	any                     `collection:"task_stats"`
-	BaseModelV2[TaskStatV2] `bson:",inline"`
-	CreateTs                time.Time `json:"create_ts" bson:"create_ts,omitempty"`
-	StartTs                 time.Time `json:"start_ts" bson:"start_ts,omitempty"`
-	EndTs                   time.Time `json:"end_ts" bson:"end_ts,omitempty"`
-	WaitDuration            int64     `json:"wait_duration" bson:"wait_duration,omitempty"`       // in millisecond
-	RuntimeDuration         int64     `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond
-	TotalDuration           int64     `json:"total_duration" bson:"total_duration,omitempty"`     // in millisecond
-	ResultCount             int64     `json:"result_count" bson:"result_count"`
-	ErrorLogCount           int64     `json:"error_log_count" bson:"error_log_count"`
-}

core/models/models/task_v2.go

@@ -1,30 +0,0 @@
-package models
-
-import (
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"time"
-)
-
-type TaskV2 struct {
-	any                 `collection:"tasks"`
-	BaseModelV2[TaskV2] `bson:",inline"`
-	SpiderId            primitive.ObjectID   `json:"spider_id" bson:"spider_id"`
-	Status              string               `json:"status" bson:"status"`
-	NodeId              primitive.ObjectID   `json:"node_id" bson:"node_id"`
-	Cmd                 string               `json:"cmd" bson:"cmd"`
-	Param               string               `json:"param" bson:"param"`
-	Error               string               `json:"error" bson:"error"`
-	Pid                 int                  `json:"pid" bson:"pid"`
-	ScheduleId          primitive.ObjectID   `json:"schedule_id" bson:"schedule_id"`
-	Type                string               `json:"type" bson:"type"`
-	Mode                string               `json:"mode" bson:"mode"`
-	NodeIds             []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
-	ParentId            primitive.ObjectID   `json:"parent_id" bson:"parent_id"`
-	Priority            int                  `json:"priority" bson:"priority"`
-	Stat                *TaskStatV2          `json:"stat,omitempty" bson:"-"`
-	HasSub              bool                 `json:"has_sub" json:"has_sub"`
-	SubTasks            []TaskV2             `json:"sub_tasks,omitempty" bson:"-"`
-	Spider              *SpiderV2            `json:"spider,omitempty" bson:"-"`
-	UserId              primitive.ObjectID   `json:"-" bson:"-"`
-	CreateTs            time.Time            `json:"create_ts" bson:"create_ts"`
-}

core/models/models/token_v2.go

@@ -1,8 +0,0 @@
-package models
-
-type TokenV2 struct {
-	any                  `collection:"tokens"`
-	BaseModelV2[TokenV2] `bson:",inline"`
-	Name                 string `json:"name" bson:"name"`
-	Token                string `json:"token" bson:"token"`
-}

core/models/models/user_role_v2.go

@@ -1,12 +0,0 @@
-package models
-
-import (
-	"go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-type UserRoleV2 struct {
-	any                     `collection:"user_roles"`
-	BaseModelV2[UserRoleV2] `bson:",inline"`
-	RoleId                  primitive.ObjectID `json:"role_id" bson:"role_id"`
-	UserId                  primitive.ObjectID `json:"user_id" bson:"user_id"`
-}

core/models/models/user_v2.go

@@ -1,10 +0,0 @@
-package models
-
-type UserV2 struct {
-	any                 `collection:"users"`
-	BaseModelV2[UserV2] `bson:",inline"`
-	Username            string `json:"username" bson:"username"`
-	Password            string `json:"-,omitempty" bson:"password"`
-	Role                string `json:"role" bson:"role"`
-	Email               string `json:"email" bson:"email"`
-}

core/models/models/v2/git_v2.go

@@ -0,0 +1,27 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models/v2"
+	"github.com/crawlab-team/crawlab/vcs"
+	"time"
+)
+
+type GitV2 struct {
+	any                       `collection:"gits"`
+	models.BaseModelV2[GitV2] `bson:",inline"`
+	Url                       string       `json:"url" bson:"url"`
+	Name                      string       `json:"name" bson:"name"`
+	AuthType                  string       `json:"auth_type" bson:"auth_type"`
+	Username                  string       `json:"username" bson:"username"`
+	Password                  string       `json:"password" bson:"password"`
+	CurrentBranch             string       `json:"current_branch" bson:"current_branch"`
+	Status                    string       `json:"status" bson:"status"`
+	Error                     string       `json:"error" bson:"error"`
+	Spiders                   []SpiderV2   `json:"spiders,omitempty" bson:"-"`
+	Refs                      []vcs.GitRef `json:"refs" bson:"refs"`
+	RefsUpdatedAt             time.Time    `json:"refs_updated_at" bson:"refs_updated_at"`
+	CloneLogs                 []string     `json:"clone_logs,omitempty" bson:"clone_logs"`
+
+	// settings
+	AutoPull bool `json:"auto_pull" bson:"auto_pull"`
+}

core/models/models/v2/metric_v2.go

@@ -0,0 +1,26 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models/v2"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type MetricV2 struct {
+	any                          `collection:"metrics"`
+	models.BaseModelV2[MetricV2] `bson:",inline"`
+	Type                         string             `json:"type" bson:"type"`
+	NodeId                       primitive.ObjectID `json:"node_id" bson:"node_id"`
+	CpuUsagePercent              float32            `json:"cpu_usage_percent" bson:"cpu_usage_percent"`
+	TotalMemory                  uint64             `json:"total_memory" bson:"total_memory"`
+	AvailableMemory              uint64             `json:"available_memory" bson:"available_memory"`
+	UsedMemory                   uint64             `json:"used_memory" bson:"used_memory"`
+	UsedMemoryPercent            float32            `json:"used_memory_percent" bson:"used_memory_percent"`
+	TotalDisk                    uint64             `json:"total_disk" bson:"total_disk"`
+	AvailableDisk                uint64             `json:"available_disk" bson:"available_disk"`
+	UsedDisk                     uint64             `json:"used_disk" bson:"used_disk"`
+	UsedDiskPercent              float32            `json:"used_disk_percent" bson:"used_disk_percent"`
+	DiskReadBytesRate            float32            `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"`
+	DiskWriteBytesRate           float32            `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"`
+	NetworkBytesSentRate         float32            `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"`
+	NetworkBytesRecvRate         float32            `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"`
+}

core/models/models/v2/node_v2.go

@@ -0,0 +1,25 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models/v2"
+	"time"
+)
+
+type NodeV2 struct {
+	any                        `collection:"nodes"`
+	models.BaseModelV2[NodeV2] `bson:",inline"`
+	Key                        string    `json:"key" bson:"key"`
+	Name                       string    `json:"name" bson:"name"`
+	Ip                         string    `json:"ip" bson:"ip"`
+	Port                       string    `json:"port" bson:"port"`
+	Mac                        string    `json:"mac" bson:"mac"`
+	Hostname                   string    `json:"hostname" bson:"hostname"`
+	Description                string    `json:"description" bson:"description"`
+	IsMaster                   bool      `json:"is_master" bson:"is_master"`
+	Status                     string    `json:"status" bson:"status"`
+	Enabled                    bool      `json:"enabled" bson:"enabled"`
+	Active                     bool      `json:"active" bson:"active"`
+	ActiveAt                   time.Time `json:"active_at" bson:"active_ts"`
+	AvailableRunners           int       `json:"available_runners" bson:"available_runners"`
+	MaxRunners                 int       `json:"max_runners" bson:"max_runners"`
+}

core/models/models/v2/notification_channel_v2.go

@@ -0,0 +1 @@
+package models

core/models/models/v2/notification_setting_v2.go

@@ -0,0 +1,33 @@
+package models
+
+import "github.com/crawlab-team/crawlab/core/models/models/v2"
+
+type NotificationSettingV2 struct {
+	any                                       `collection:"notification_settings"`
+	models.BaseModelV2[NotificationSettingV2] `bson:",inline"`
+	Type                                      string                    `json:"type" bson:"type"`
+	Name                                      string                    `json:"name" bson:"name"`
+	Description                               string                    `json:"description" bson:"description"`
+	Enabled                                   bool                      `json:"enabled" bson:"enabled"`
+	Global                                    bool                      `json:"global" bson:"global"`
+	Title                                     string                    `json:"title,omitempty" bson:"title,omitempty"`
+	Template                                  string                    `json:"template,omitempty" bson:"template,omitempty"`
+	TaskTrigger                               string                    `json:"task_trigger" bson:"task_trigger"`
+	Mail                                      NotificationSettingMail   `json:"mail,omitempty" bson:"mail,omitempty"`
+	Mobile                                    NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"`
+}
+
+type NotificationSettingMail struct {
+	Server         string `json:"server" bson:"server"`
+	Port           string `json:"port,omitempty" bson:"port,omitempty"`
+	User           string `json:"user,omitempty" bson:"user,omitempty"`
+	Password       string `json:"password,omitempty" bson:"password,omitempty"`
+	SenderEmail    string `json:"sender_email,omitempty" bson:"sender_email,omitempty"`
+	SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"`
+	To             string `json:"to,omitempty" bson:"to,omitempty"`
+	Cc             string `json:"cc,omitempty" bson:"cc,omitempty"`
+}
+
+type NotificationSettingMobile struct {
+	Webhook string `json:"webhook" bson:"webhook"`
+}

core/models/models/v2/permission_v2.go

@@ -0,0 +1,15 @@
+package models
+
+import "github.com/crawlab-team/crawlab/core/models/models/v2"
+
+type PermissionV2 struct {
+	any                              `collection:"permissions"`
+	models.BaseModelV2[PermissionV2] `bson:",inline"`
+	Key                              string   `json:"key" bson:"key"`
+	Name                             string   `json:"name" bson:"name"`
+	Description                      string   `json:"description" bson:"description"`
+	Type                             string   `json:"type" bson:"type"`
+	Target                           []string `json:"target" bson:"target"`
+	Allow                            []string `json:"allow" bson:"allow"`
+	Deny                             []string `json:"deny" bson:"deny"`
+}

core/models/models/v2/project_v2.go

@@ -0,0 +1,11 @@
+package models
+
+import "github.com/crawlab-team/crawlab/core/models/models/v2"
+
+type ProjectV2 struct {
+	any                           `collection:"projects"`
+	models.BaseModelV2[ProjectV2] `bson:",inline"`
+	Name                          string `json:"name" bson:"name"`
+	Description                   string `json:"description" bson:"description"`
+	Spiders                       int    `json:"spiders" bson:"-"`
+}

core/models/models/v2/role_permission_v2.go

@@ -0,0 +1,13 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models/v2"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type RolePermissionV2 struct {
+	any                                  `collection:"role_permissions"`
+	models.BaseModelV2[RolePermissionV2] `bson:",inline"`
+	RoleId                               primitive.ObjectID `json:"role_id" bson:"role_id"`
+	PermissionId                         primitive.ObjectID `json:"permission_id" bson:"permission_id"`
+}

core/models/models/v2/role_v2.go

@@ -0,0 +1,11 @@
+package models
+
+import "github.com/crawlab-team/crawlab/core/models/models/v2"
+
+type RoleV2 struct {
+	any                        `collection:"roles"`
+	models.BaseModelV2[RoleV2] `bson:",inline"`
+	Key                        string `json:"key" bson:"key"`
+	Name                       string `json:"name" bson:"name"`
+	Description                string `json:"description" bson:"description"`
+}

core/models/models/v2/schedule_v2.go

@@ -0,0 +1,24 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models/v2"
+	"github.com/robfig/cron/v3"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type ScheduleV2 struct {
+	any                            `collection:"schedules"`
+	models.BaseModelV2[ScheduleV2] `bson:",inline"`
+	Name                           string               `json:"name" bson:"name"`
+	Description                    string               `json:"description" bson:"description"`
+	SpiderId                       primitive.ObjectID   `json:"spider_id" bson:"spider_id"`
+	Cron                           string               `json:"cron" bson:"cron"`
+	EntryId                        cron.EntryID         `json:"entry_id" bson:"entry_id"`
+	Cmd                            string               `json:"cmd" bson:"cmd"`
+	Param                          string               `json:"param" bson:"param"`
+	Mode                           string               `json:"mode" bson:"mode"`
+	NodeIds                        []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
+	Priority                       int                  `json:"priority" bson:"priority"`
+	Enabled                        bool                 `json:"enabled" bson:"enabled"`
+	UserId                         primitive.ObjectID   `json:"user_id" bson:"user_id"`
+}

core/models/models/v2/setting_v2.go

@@ -0,0 +1,13 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models/v2"
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+type SettingV2 struct {
+	any                           `collection:"settings"`
+	models.BaseModelV2[SettingV2] `bson:",inline"`
+	Key                           string `json:"key" bson:"key"`
+	Value                         bson.M `json:"value" bson:"value"`
+}

core/models/models/v2/spider_stat_v2.go

@@ -0,0 +1,21 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models/v2"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type SpiderStatV2 struct {
+	any                              `collection:"spider_stats"`
+	models.BaseModelV2[SpiderStatV2] `bson:",inline"`
+	LastTaskId                       primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"`
+	LastTask                         *TaskV2            `json:"last_task,omitempty" bson:"-"`
+	Tasks                            int                `json:"tasks" bson:"tasks"`
+	Results                          int                `json:"results" bson:"results"`
+	WaitDuration                     int64              `json:"wait_duration" bson:"wait_duration,omitempty"`       // in second
+	RuntimeDuration                  int64              `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second
+	TotalDuration                    int64              `json:"total_duration" bson:"total_duration,omitempty"`     // in second
+	AverageWaitDuration              int64              `json:"average_wait_duration" bson:"-"`                     // in second
+	AverageRuntimeDuration           int64              `json:"average_runtime_duration" bson:"-"`                  // in second
+	AverageTotalDuration             int64              `json:"average_total_duration" bson:"-"`                    // in second
+}

core/models/models/v2/spider_v2.go

@@ -0,0 +1,33 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models/v2"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type SpiderV2 struct {
+	any                          `collection:"spiders"`
+	models.BaseModelV2[SpiderV2] `bson:",inline"`
+	Name                         string               `json:"name" bson:"name"`                     // spider name
+	Type                         string               `json:"type" bson:"type"`                     // spider type
+	ColId                        primitive.ObjectID   `json:"col_id" bson:"col_id"`                 // data collection id
+	ColName                      string               `json:"col_name,omitempty" bson:"-"`          // data collection name
+	DataSourceId                 primitive.ObjectID   `json:"data_source_id" bson:"data_source_id"` // data source id
+	DataSource                   *models.DataSourceV2 `json:"data_source,omitempty" bson:"-"`       // data source
+	Description                  string               `json:"description" bson:"description"`       // description
+	ProjectId                    primitive.ObjectID   `json:"project_id" bson:"project_id"`         // Project.Id
+	Mode                         string               `json:"mode" bson:"mode"`                     // default Task.Mode
+	NodeIds                      []primitive.ObjectID `json:"node_ids" bson:"node_ids"`             // default Task.NodeIds
+	GitId                        primitive.ObjectID   `json:"git_id" bson:"git_id"`                 // related Git.Id
+	GitRootPath                  string               `json:"git_root_path" bson:"git_root_path"`
+	Git                          *GitV2               `json:"git,omitempty" bson:"-"`
+
+	// stats
+	Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"`
+
+	// execution
+	Cmd         string `json:"cmd" bson:"cmd"`     // execute command
+	Param       string `json:"param" bson:"param"` // default task param
+	Priority    int    `json:"priority" bson:"priority"`
+	AutoInstall bool   `json:"auto_install" bson:"auto_install"`
+}

core/models/models/v2/task_queue_item_v2.go

@@ -0,0 +1,13 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models/v2"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type TaskQueueItemV2 struct {
+	any                                 `collection:"task_queue"`
+	models.BaseModelV2[TaskQueueItemV2] `bson:",inline"`
+	Priority                            int                `json:"p" bson:"p"`
+	NodeId                              primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"`
+}

core/models/models/v2/task_stat_v2.go

@@ -0,0 +1,19 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models/v2"
+	"time"
+)
+
+type TaskStatV2 struct {
+	any                            `collection:"task_stats"`
+	models.BaseModelV2[TaskStatV2] `bson:",inline"`
+	CreateTs                       time.Time `json:"create_ts" bson:"create_ts,omitempty"`
+	StartTs                        time.Time `json:"start_ts" bson:"start_ts,omitempty"`
+	EndTs                          time.Time `json:"end_ts" bson:"end_ts,omitempty"`
+	WaitDuration                   int64     `json:"wait_duration" bson:"wait_duration,omitempty"`       // in millisecond
+	RuntimeDuration                int64     `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond
+	TotalDuration                  int64     `json:"total_duration" bson:"total_duration,omitempty"`     // in millisecond
+	ResultCount                    int64     `json:"result_count" bson:"result_count"`
+	ErrorLogCount                  int64     `json:"error_log_count" bson:"error_log_count"`
+}

core/models/models/v2/task_v2.go

@@ -0,0 +1,31 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models/v2"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type TaskV2 struct {
+	any                        `collection:"tasks"`
+	models.BaseModelV2[TaskV2] `bson:",inline"`
+	SpiderId                   primitive.ObjectID   `json:"spider_id" bson:"spider_id"`
+	Status                     string               `json:"status" bson:"status"`
+	NodeId                     primitive.ObjectID   `json:"node_id" bson:"node_id"`
+	Cmd                        string               `json:"cmd" bson:"cmd"`
+	Param                      string               `json:"param" bson:"param"`
+	Error                      string               `json:"error" bson:"error"`
+	Pid                        int                  `json:"pid" bson:"pid"`
+	ScheduleId                 primitive.ObjectID   `json:"schedule_id" bson:"schedule_id"`
+	Type                       string               `json:"type" bson:"type"`
+	Mode                       string               `json:"mode" bson:"mode"`
+	NodeIds                    []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
+	ParentId                   primitive.ObjectID   `json:"parent_id" bson:"parent_id"`
+	Priority                   int                  `json:"priority" bson:"priority"`
+	Stat                       *TaskStatV2          `json:"stat,omitempty" bson:"-"`
+	HasSub                     bool                 `json:"has_sub" json:"has_sub"`
+	SubTasks                   []TaskV2             `json:"sub_tasks,omitempty" bson:"-"`
+	Spider                     *SpiderV2            `json:"spider,omitempty" bson:"-"`
+	UserId                     primitive.ObjectID   `json:"-" bson:"-"`
+	CreateTs                   time.Time            `json:"create_ts" bson:"create_ts"`
+}

core/models/models/v2/token_v2.go

@@ -0,0 +1,10 @@
+package models
+
+import "github.com/crawlab-team/crawlab/core/models/models/v2"
+
+type TokenV2 struct {
+	any                         `collection:"tokens"`
+	models.BaseModelV2[TokenV2] `bson:",inline"`
+	Name                        string `json:"name" bson:"name"`
+	Token                       string `json:"token" bson:"token"`
+}

core/models/models/v2/user_role_v2.go

@@ -0,0 +1,13 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models/v2"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type UserRoleV2 struct {
+	any                            `collection:"user_roles"`
+	models.BaseModelV2[UserRoleV2] `bson:",inline"`
+	RoleId                         primitive.ObjectID `json:"role_id" bson:"role_id"`
+	UserId                         primitive.ObjectID `json:"user_id" bson:"user_id"`
+}

core/models/models/v2/user_v2.go

@@ -0,0 +1,12 @@
+package models
+
+import "github.com/crawlab-team/crawlab/core/models/models/v2"
+
+type UserV2 struct {
+	any                        `collection:"users"`
+	models.BaseModelV2[UserV2] `bson:",inline"`
+	Username                   string `json:"username" bson:"username"`
+	Password                   string `json:"-,omitempty" bson:"password"`
+	Role                       string `json:"role" bson:"role"`
+	Email                      string `json:"email" bson:"email"`
+}

core/models/models/v2/variable_v2.go

@@ -0,0 +1,11 @@
+package models
+
+import "github.com/crawlab-team/crawlab/core/models/models/v2"
+
+type VariableV2 struct {
+	any                            `collection:"variables"`
+	models.BaseModelV2[VariableV2] `bson:",inline"`
+	Key                            string `json:"key" bson:"key"`
+	Value                          string `json:"value" bson:"value"`
+	Remark                         string `json:"remark" bson:"remark"`
+}

core/models/models/variable_v2.go

@@ -1,9 +0,0 @@
-package models
-
-type VariableV2 struct {
-	any                     `collection:"variables"`
-	BaseModelV2[VariableV2] `bson:",inline"`
-	Key                     string `json:"key" bson:"key"`
-	Value                   string `json:"value" bson:"value"`
-	Remark                  string `json:"remark" bson:"remark"`
-}