git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-29 18:00:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: removed unnecessary code

Browse Source
This commit is contained in:
Marvin Zhang
2024-07-11 12:45:29 +08:00
parent f2f298d8f8
commit aca0c0ebce
57 changed files with 423 additions and 430 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
core/models/common/index_service_v2.go
View File

@@ -1,7 +1,7 @@
package common
import (
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
@@ -11,7 +11,7 @@ import (
func CreateIndexesV2() {
// nodes
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.NodeV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NodeV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"key": 1}}, // key
{Keys: bson.M{"name": 1}}, // name
{Keys: bson.M{"is_master": 1}}, // is_master
@@ -21,12 +21,12 @@ func CreateIndexesV2() {
})
// projects
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.ProjectV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ProjectV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// spiders
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.SpiderV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SpiderV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
{Keys: bson.M{"type": 1}},
{Keys: bson.M{"col_id": 1}},
@@ -34,7 +34,7 @@ func CreateIndexesV2() {
})
// tasks
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"spider_id": 1}},
{Keys: bson.M{"status": 1}},
{Keys: bson.M{"node_id": 1}},
@@ -48,73 +48,73 @@ func CreateIndexesV2() {
})
// task stats
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TaskStatV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskStatV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"create_ts": 1}},
})
// schedules
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.ScheduleV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ScheduleV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
{Keys: bson.M{"spider_id": 1}},
{Keys: bson.M{"enabled": 1}},
})
// users
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.UserV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"username": 1}},
{Keys: bson.M{"role": 1}},
{Keys: bson.M{"email": 1}},
})
// settings
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"key": 1}},
})
// tokens
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TokenV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TokenV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// variables
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.VariableV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.VariableV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"key": 1}},
})
// data sources
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DataSourceV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataSourceV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// data collections
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DataCollectionV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataCollectionV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// roles
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.RoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)},
})
// user role relations
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.UserRoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserRoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.D{{"user_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)},
{Keys: bson.D{{"role_id", 1}, {"user_id", 1}}, Options: options.Index().SetUnique(true)},
})
// permissions
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.PermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.PermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)},
})
// role permission relations
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.RolePermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RolePermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.D{{"role_id", 1}, {"permission_id", 1}}, Options: options.Index().SetUnique(true)},
{Keys: bson.D{{"permission_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)},
})
// dependencies
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{
{"type", 1},
@@ -126,7 +126,7 @@ func CreateIndexesV2() {
})
// dependency settings
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencySettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencySettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{
{"type", 1},
@@ -138,7 +138,7 @@ func CreateIndexesV2() {
})
// dependency logs
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyLogV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyLogV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{{"task_id", 1}},
},
@@ -149,7 +149,7 @@ func CreateIndexesV2() {
})
// dependency tasks
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyTaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyTaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{
{"update_ts", 1},
@@ -159,7 +159,7 @@ func CreateIndexesV2() {
})
// metrics
mongo.GetMongoCol(service.GetCollectionNameByInstance(models.MetricV2{})).MustCreateIndexes([]mongo2.IndexModel{
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.MetricV2{})).MustCreateIndexes([]mongo2.IndexModel{
{
Keys: bson.D{
{"created_ts", -1},