git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: removed unnecessary code

Browse Source
This commit is contained in:
Marvin Zhang
2024-07-11 12:45:29 +08:00
parent f2f298d8f8
commit aca0c0ebce
57 changed files with 423 additions and 430 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
core/models/client/model_service_v2_test.go
View File

@@ -4,7 +4,7 @@ import (
"context"
"github.com/crawlab-team/crawlab/core/grpc/server"
"github.com/crawlab-team/crawlab/core/models/client"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/spf13/viper"
@@ -17,7 +17,7 @@ import (
"time"
)
type TestModel models.TestModel
type TestModel models.TestModelV2
func setupTestDB() {
viper.Set("mongo.db", "testdb")

44
core/models/common/index_service_v2.go
View File

@@ -1,7 +1,7 @@
package common
import (
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
@@ -11,7 +11,7 @@ import (
func CreateIndexesV2() {
// nodes
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.NodeV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NodeV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"key": 1}}, // key
{Keys: bson.M{"name": 1}}, // name
{Keys: bson.M{"is_master": 1}}, // is_master
@@ -21,12 +21,12 @@ func CreateIndexesV2() {
})
// projects
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.ProjectV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ProjectV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// spiders
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.SpiderV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SpiderV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
{Keys: bson.M{"type": 1}},
{Keys: bson.M{"col_id": 1}},
@@ -34,7 +34,7 @@ func CreateIndexesV2() {
})
// tasks
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"spider_id": 1}},
{Keys: bson.M{"status": 1}},
{Keys: bson.M{"node_id": 1}},
@@ -48,73 +48,73 @@ func CreateIndexesV2() {
})
// task stats
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TaskStatV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskStatV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"create_ts": 1}},
})
// schedules
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.ScheduleV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ScheduleV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
{Keys: bson.M{"spider_id": 1}},
{Keys: bson.M{"enabled": 1}},
})
// users
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.UserV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"username": 1}},
{Keys: bson.M{"role": 1}},
{Keys: bson.M{"email": 1}},
})
// settings
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"key": 1}},
})
// tokens
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TokenV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TokenV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// variables
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.VariableV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.VariableV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"key": 1}},
})
// data sources
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DataSourceV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataSourceV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// data collections
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DataCollectionV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataCollectionV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// roles
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.RoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)},
})
// user role relations
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.UserRoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserRoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.D{{"user_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)},
{Keys: bson.D{{"role_id", 1}, {"user_id", 1}}, Options: options.Index().SetUnique(true)},
})
// permissions
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.PermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.PermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)},
})
// role permission relations
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.RolePermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RolePermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.D{{"role_id", 1}, {"permission_id", 1}}, Options: options.Index().SetUnique(true)},
{Keys: bson.D{{"permission_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)},
})
// dependencies
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{
{"type", 1},
@@ -126,7 +126,7 @@ func CreateIndexesV2() {
})
// dependency settings
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencySettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencySettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{
{"type", 1},
@@ -138,7 +138,7 @@ func CreateIndexesV2() {
})
// dependency logs
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyLogV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyLogV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{{"task_id", 1}},
},
@@ -149,7 +149,7 @@ func CreateIndexesV2() {
})
// dependency tasks
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyTaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyTaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{
{"update_ts", 1},
@@ -159,7 +159,7 @@ func CreateIndexesV2() {
})
// metrics
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.MetricV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.MetricV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{
{"created_ts", -1},

10
core/models/models/v2/data_collection_v2.go
View File

@@ -5,11 +5,11 @@ import (
)
type DataCollectionV2 struct {
any `collection:"data_collections"`
BaseModelV2[DataCollection] `bson:",inline"`
Name string `json:"name" bson:"name"`
Fields []entity.DataField `json:"fields" bson:"fields"`
Dedup struct {
any `collection:"data_collections"`
BaseModelV2[DataCollectionV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Fields []entity.DataField `json:"fields" bson:"fields"`
Dedup struct {
Enabled bool `json:"enabled" bson:"enabled"`
Keys []string `json:"keys" bson:"keys"`
Type string `json:"type" bson:"type"`

32
core/models/models/v2/data_source_v2.go
View File

@@ -1,20 +1,20 @@
package models
type DataSourceV2 struct {
any `collection:"data_sources"`
BaseModelV2[DataSource] `bson:",inline"`
Name string `json:"name" bson:"name"`
Type string `json:"type" bson:"type"`
Description string `json:"description" bson:"description"`
Host string `json:"host" bson:"host"`
Port string `json:"port" bson:"port"`
Url string `json:"url" bson:"url"`
Hosts []string `json:"hosts" bson:"hosts"`
Database string `json:"database" bson:"database"`
Username string `json:"username" bson:"username"`
Password string `json:"-,omitempty" bson:"password"`
ConnectType string `json:"connect_type" bson:"connect_type"`
Status string `json:"status" bson:"status"`
Error string `json:"error" bson:"error"`
Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"`
any `collection:"data_sources"`
BaseModelV2[DataSourceV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Type string `json:"type" bson:"type"`
Description string `json:"description" bson:"description"`
Host string `json:"host" bson:"host"`
Port string `json:"port" bson:"port"`
Url string `json:"url" bson:"url"`
Hosts []string `json:"hosts" bson:"hosts"`
Database string `json:"database" bson:"database"`
Username string `json:"username" bson:"username"`
Password string `json:"-,omitempty" bson:"password"`
ConnectType string `json:"connect_type" bson:"connect_type"`
Status string `json:"status" bson:"status"`
Error string `json:"error" bson:"error"`
Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"`
}

18
core/models/models/v2/dependency_setting_v2.go
View File

@@ -5,13 +5,13 @@ import (
)
type DependencySettingV2 struct {
any `collection:"dependency_settings"`
BaseModelV2[DependencySetting] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Enabled bool `json:"enabled" bson:"enabled"`
Cmd string `json:"cmd" bson:"cmd"`
Proxy string `json:"proxy" bson:"proxy"`
LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"`
any `collection:"dependency_settings"`
BaseModelV2[DependencySettingV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Enabled bool `json:"enabled" bson:"enabled"`
Cmd string `json:"cmd" bson:"cmd"`
Proxy string `json:"proxy" bson:"proxy"`
LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"`
}

29
core/models/models/v2/git_v2.go
View File

@@ -1,26 +1,25 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/vcs"
"time"
)
type GitV2 struct {
any `collection:"gits"`
models.BaseModelV2[GitV2] `bson:",inline"`
Url string `json:"url" bson:"url"`
Name string `json:"name" bson:"name"`
AuthType string `json:"auth_type" bson:"auth_type"`
Username string `json:"username" bson:"username"`
Password string `json:"password" bson:"password"`
CurrentBranch string `json:"current_branch" bson:"current_branch"`
Status string `json:"status" bson:"status"`
Error string `json:"error" bson:"error"`
Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"`
Refs []vcs.GitRef `json:"refs" bson:"refs"`
RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"`
CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"`
any `collection:"gits"`
BaseModelV2[GitV2] `bson:",inline"`
Url string `json:"url" bson:"url"`
Name string `json:"name" bson:"name"`
AuthType string `json:"auth_type" bson:"auth_type"`
Username string `json:"username" bson:"username"`
Password string `json:"password" bson:"password"`
CurrentBranch string `json:"current_branch" bson:"current_branch"`
Status string `json:"status" bson:"status"`
Error string `json:"error" bson:"error"`
Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"`
Refs []vcs.GitRef `json:"refs" bson:"refs"`
RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"`
CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"`
// settings
AutoPull bool `json:"auto_pull" bson:"auto_pull"`

35
core/models/models/v2/metric_v2.go
View File

@@ -1,26 +1,25 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type MetricV2 struct {
any `collection:"metrics"`
models.BaseModelV2[MetricV2] `bson:",inline"`
Type string `json:"type" bson:"type"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"`
TotalMemory uint64 `json:"total_memory" bson:"total_memory"`
AvailableMemory uint64 `json:"available_memory" bson:"available_memory"`
UsedMemory uint64 `json:"used_memory" bson:"used_memory"`
UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"`
TotalDisk uint64 `json:"total_disk" bson:"total_disk"`
AvailableDisk uint64 `json:"available_disk" bson:"available_disk"`
UsedDisk uint64 `json:"used_disk" bson:"used_disk"`
UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"`
DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"`
DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"`
NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"`
NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"`
any `collection:"metrics"`
BaseModelV2[MetricV2] `bson:",inline"`
Type string `json:"type" bson:"type"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"`
TotalMemory uint64 `json:"total_memory" bson:"total_memory"`
AvailableMemory uint64 `json:"available_memory" bson:"available_memory"`
UsedMemory uint64 `json:"used_memory" bson:"used_memory"`
UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"`
TotalDisk uint64 `json:"total_disk" bson:"total_disk"`
AvailableDisk uint64 `json:"available_disk" bson:"available_disk"`
UsedDisk uint64 `json:"used_disk" bson:"used_disk"`
UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"`
DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"`
DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"`
NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"`
NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"`
}

33
core/models/models/v2/node_v2.go
View File

@@ -1,25 +1,24 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"time"
)
type NodeV2 struct {
any `collection:"nodes"`
models.BaseModelV2[NodeV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Ip string `json:"ip" bson:"ip"`
Port string `json:"port" bson:"port"`
Mac string `json:"mac" bson:"mac"`
Hostname string `json:"hostname" bson:"hostname"`
Description string `json:"description" bson:"description"`
IsMaster bool `json:"is_master" bson:"is_master"`
Status string `json:"status" bson:"status"`
Enabled bool `json:"enabled" bson:"enabled"`
Active bool `json:"active" bson:"active"`
ActiveAt time.Time `json:"active_at" bson:"active_ts"`
AvailableRunners int `json:"available_runners" bson:"available_runners"`
MaxRunners int `json:"max_runners" bson:"max_runners"`
any `collection:"nodes"`
BaseModelV2[NodeV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Ip string `json:"ip" bson:"ip"`
Port string `json:"port" bson:"port"`
Mac string `json:"mac" bson:"mac"`
Hostname string `json:"hostname" bson:"hostname"`
Description string `json:"description" bson:"description"`
IsMaster bool `json:"is_master" bson:"is_master"`
Status string `json:"status" bson:"status"`
Enabled bool `json:"enabled" bson:"enabled"`
Active bool `json:"active" bson:"active"`
ActiveAt time.Time `json:"active_at" bson:"active_ts"`
AvailableRunners int `json:"available_runners" bson:"available_runners"`
MaxRunners int `json:"max_runners" bson:"max_runners"`
}

3
core/models/models/v2/notification_channel_v2.go
View File

@@ -1 +1,4 @@
package models
type NotificationChannel struct {
}

26
core/models/models/v2/notification_setting_v2.go
View File

@@ -1,20 +1,18 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type NotificationSettingV2 struct {
any `collection:"notification_settings"`
models.BaseModelV2[NotificationSettingV2] `bson:",inline"`
Type string `json:"type" bson:"type"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Enabled bool `json:"enabled" bson:"enabled"`
Global bool `json:"global" bson:"global"`
Title string `json:"title,omitempty" bson:"title,omitempty"`
Template string `json:"template,omitempty" bson:"template,omitempty"`
TaskTrigger string `json:"task_trigger" bson:"task_trigger"`
Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"`
Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"`
any `collection:"notification_settings"`
BaseModelV2[NotificationSettingV2] `bson:",inline"`
Type string `json:"type" bson:"type"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Enabled bool `json:"enabled" bson:"enabled"`
Global bool `json:"global" bson:"global"`
Title string `json:"title,omitempty" bson:"title,omitempty"`
Template string `json:"template,omitempty" bson:"template,omitempty"`
TaskTrigger string `json:"task_trigger" bson:"task_trigger"`
Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"`
Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"`
}
type NotificationSettingMail struct {

20
core/models/models/v2/permission_v2.go
View File

@@ -1,15 +1,13 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type PermissionV2 struct {
any `collection:"permissions"`
models.BaseModelV2[PermissionV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Type string `json:"type" bson:"type"`
Target []string `json:"target" bson:"target"`
Allow []string `json:"allow" bson:"allow"`
Deny []string `json:"deny" bson:"deny"`
any `collection:"permissions"`
BaseModelV2[PermissionV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Type string `json:"type" bson:"type"`
Target []string `json:"target" bson:"target"`
Allow []string `json:"allow" bson:"allow"`
Deny []string `json:"deny" bson:"deny"`
}

12
core/models/models/v2/project_v2.go
View File

@@ -1,11 +1,9 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type ProjectV2 struct {
any `collection:"projects"`
models.BaseModelV2[ProjectV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Spiders int `json:"spiders" bson:"-"`
any `collection:"projects"`
BaseModelV2[ProjectV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Spiders int `json:"spiders" bson:"-"`
}

9
core/models/models/v2/role_permission_v2.go
View File

@@ -1,13 +1,12 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type RolePermissionV2 struct {
any `collection:"role_permissions"`
models.BaseModelV2[RolePermissionV2] `bson:",inline"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"`
any `collection:"role_permissions"`
BaseModelV2[RolePermissionV2] `bson:",inline"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"`
}

12
core/models/models/v2/role_v2.go
View File

@@ -1,11 +1,9 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type RoleV2 struct {
any `collection:"roles"`
models.BaseModelV2[RoleV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
any `collection:"roles"`
BaseModelV2[RoleV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
}

29
core/models/models/v2/schedule_v2.go
View File

@@ -1,24 +1,23 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/robfig/cron/v3"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type ScheduleV2 struct {
any `collection:"schedules"`
models.BaseModelV2[ScheduleV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Cron string `json:"cron" bson:"cron"`
EntryId cron.EntryID `json:"entry_id" bson:"entry_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
Priority int `json:"priority" bson:"priority"`
Enabled bool `json:"enabled" bson:"enabled"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
any `collection:"schedules"`
BaseModelV2[ScheduleV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Cron string `json:"cron" bson:"cron"`
EntryId cron.EntryID `json:"entry_id" bson:"entry_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
Priority int `json:"priority" bson:"priority"`
Enabled bool `json:"enabled" bson:"enabled"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
}

9
core/models/models/v2/setting_v2.go
View File

@@ -1,13 +1,12 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson"
)
type SettingV2 struct {
any `collection:"settings"`
models.BaseModelV2[SettingV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Value bson.M `json:"value" bson:"value"`
any `collection:"settings"`
BaseModelV2[SettingV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Value bson.M `json:"value" bson:"value"`
}

25
core/models/models/v2/spider_stat_v2.go
View File

@@ -1,21 +1,20 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type SpiderStatV2 struct {
any `collection:"spider_stats"`
models.BaseModelV2[SpiderStatV2] `bson:",inline"`
LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"`
LastTask *TaskV2 `json:"last_task,omitempty" bson:"-"`
Tasks int `json:"tasks" bson:"tasks"`
Results int `json:"results" bson:"results"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second
AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second
AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second
AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second
any `collection:"spider_stats"`
BaseModelV2[SpiderStatV2] `bson:",inline"`
LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"`
LastTask *TaskV2 `json:"last_task,omitempty" bson:"-"`
Tasks int `json:"tasks" bson:"tasks"`
Results int `json:"results" bson:"results"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second
AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second
AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second
AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second
}

31
core/models/models/v2/spider_v2.go
View File

@@ -1,26 +1,25 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type SpiderV2 struct {
any `collection:"spiders"`
models.BaseModelV2[SpiderV2] `bson:",inline"`
Name string `json:"name" bson:"name"` // spider name
Type string `json:"type" bson:"type"` // spider type
ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id
ColName string `json:"col_name,omitempty" bson:"-"` // data collection name
DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id
DataSource *models.DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source
Description string `json:"description" bson:"description"` // description
ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id
Mode string `json:"mode" bson:"mode"` // default Task.Mode
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds
GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id
GitRootPath string `json:"git_root_path" bson:"git_root_path"`
Git *GitV2 `json:"git,omitempty" bson:"-"`
any `collection:"spiders"`
BaseModelV2[SpiderV2] `bson:",inline"`
Name string `json:"name" bson:"name"` // spider name
Type string `json:"type" bson:"type"` // spider type
ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id
ColName string `json:"col_name,omitempty" bson:"-"` // data collection name
DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id
DataSource *DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source
Description string `json:"description" bson:"description"` // description
ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id
Mode string `json:"mode" bson:"mode"` // default Task.Mode
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds
GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id
GitRootPath string `json:"git_root_path" bson:"git_root_path"`
Git *GitV2 `json:"git,omitempty" bson:"-"`
// stats
Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"`

9
core/models/models/v2/task_queue_item_v2.go
View File

@@ -1,13 +1,12 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type TaskQueueItemV2 struct {
any `collection:"task_queue"`
models.BaseModelV2[TaskQueueItemV2] `bson:",inline"`
Priority int `json:"p" bson:"p"`
NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"`
any `collection:"task_queue"`
BaseModelV2[TaskQueueItemV2] `bson:",inline"`
Priority int `json:"p" bson:"p"`
NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"`
}

21
core/models/models/v2/task_stat_v2.go
View File

@@ -1,19 +1,18 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"time"
)
type TaskStatV2 struct {
any `collection:"task_stats"`
models.BaseModelV2[TaskStatV2] `bson:",inline"`
CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"`
StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"`
EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond
ResultCount int64 `json:"result_count" bson:"result_count"`
ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"`
any `collection:"task_stats"`
BaseModelV2[TaskStatV2] `bson:",inline"`
CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"`
StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"`
EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond
ResultCount int64 `json:"result_count" bson:"result_count"`
ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"`
}

43
core/models/models/v2/task_v2.go
View File

@@ -1,31 +1,30 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
"time"
)
type TaskV2 struct {
any `collection:"tasks"`
models.BaseModelV2[TaskV2] `bson:",inline"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Status string `json:"status" bson:"status"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Error string `json:"error" bson:"error"`
Pid int `json:"pid" bson:"pid"`
ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"`
Type string `json:"type" bson:"type"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"`
Priority int `json:"priority" bson:"priority"`
Stat *TaskStatV2 `json:"stat,omitempty" bson:"-"`
HasSub bool `json:"has_sub" json:"has_sub"`
SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"`
Spider *SpiderV2 `json:"spider,omitempty" bson:"-"`
UserId primitive.ObjectID `json:"-" bson:"-"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
any `collection:"tasks"`
BaseModelV2[TaskV2] `bson:",inline"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Status string `json:"status" bson:"status"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Error string `json:"error" bson:"error"`
Pid int `json:"pid" bson:"pid"`
ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"`
Type string `json:"type" bson:"type"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"`
Priority int `json:"priority" bson:"priority"`
Stat *TaskStatV2 `json:"stat,omitempty" bson:"-"`
HasSub bool `json:"has_sub" json:"has_sub"`
SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"`
Spider *SpiderV2 `json:"spider,omitempty" bson:"-"`
UserId primitive.ObjectID `json:"-" bson:"-"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
}

8
core/models/models/v2/test_v2.go
View File

@@ -1,7 +1,7 @@
package models
type TestModel struct {
any `collection:"testmodels"`
BaseModelV2[TestModel] `bson:",inline"`
Name string `json:"name" bson:"name"`
type TestModelV2 struct {
any `collection:"testmodels"`
BaseModelV2[TestModelV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
}

10
core/models/models/v2/token_v2.go
View File

@@ -1,10 +1,8 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type TokenV2 struct {
any `collection:"tokens"`
models.BaseModelV2[TokenV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Token string `json:"token" bson:"token"`
any `collection:"tokens"`
BaseModelV2[TokenV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Token string `json:"token" bson:"token"`
}

9
core/models/models/v2/user_role_v2.go
View File

@@ -1,13 +1,12 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type UserRoleV2 struct {
any `collection:"user_roles"`
models.BaseModelV2[UserRoleV2] `bson:",inline"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
any `collection:"user_roles"`
BaseModelV2[UserRoleV2] `bson:",inline"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
}

14
core/models/models/v2/user_v2.go
View File

@@ -1,12 +1,10 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type UserV2 struct {
any `collection:"users"`
models.BaseModelV2[UserV2] `bson:",inline"`
Username string `json:"username" bson:"username"`
Password string `json:"-,omitempty" bson:"password"`
Role string `json:"role" bson:"role"`
Email string `json:"email" bson:"email"`
any `collection:"users"`
BaseModelV2[UserV2] `bson:",inline"`
Username string `json:"username" bson:"username"`
Password string `json:"-,omitempty" bson:"password"`
Role string `json:"role" bson:"role"`
Email string `json:"email" bson:"email"`
}

12
core/models/models/v2/variable_v2.go
View File

@@ -1,11 +1,9 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type VariableV2 struct {
any `collection:"variables"`
models.BaseModelV2[VariableV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Value string `json:"value" bson:"value"`
Remark string `json:"remark" bson:"remark"`
any `collection:"variables"`
BaseModelV2[VariableV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Value string `json:"value" bson:"value"`
Remark string `json:"remark" bson:"remark"`
}

2
core/models/service/base_service_v2_test.go
View File

@@ -2,7 +2,7 @@ package service_test
import (
"context"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/spf13/viper"