diff --git a/core/apps/api_v2.go b/core/apps/api.go similarity index 77% rename from core/apps/api_v2.go rename to core/apps/api.go index 16fe102c..e73b7716 100644 --- a/core/apps/api_v2.go +++ b/core/apps/api.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/viper" "net" "net/http" + "sync" "time" ) @@ -24,7 +25,7 @@ func init() { } } -type ApiV2 struct { +type Api struct { // dependencies interfaces.WithConfigPath @@ -35,7 +36,7 @@ type ApiV2 struct { ready bool } -func (app *ApiV2) Init() { +func (app *Api) Init() { // initialize middlewares _ = app.initModuleWithApp("middlewares", middlewares.InitMiddlewares) @@ -43,7 +44,7 @@ func (app *ApiV2) Init() { _ = app.initModuleWithApp("routes", controllers.InitRoutes) } -func (app *ApiV2) Start() { +func (app *Api) Start() { // address host := viper.GetString("server.host") port := viper.GetString("server.port") @@ -73,11 +74,11 @@ func (app *ApiV2) Start() { } } -func (app *ApiV2) Wait() { +func (app *Api) Wait() { utils.DefaultWait() } -func (app *ApiV2) Stop() { +func (app *Api) Stop() { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -86,38 +87,38 @@ func (app *ApiV2) Stop() { } } -func (app *ApiV2) GetGinEngine() *gin.Engine { +func (app *Api) GetGinEngine() *gin.Engine { return app.app } -func (app *ApiV2) GetHttpServer() *http.Server { +func (app *Api) GetHttpServer() *http.Server { return app.srv } -func (app *ApiV2) Ready() (ok bool) { +func (app *Api) Ready() (ok bool) { return app.ready } -func (app *ApiV2) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) { +func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) { return initModule(name, func() error { return fn(app.app) }) } -func NewApiV2() *ApiV2 { - api := &ApiV2{ +func NewApi() *Api { + api := &Api{ app: gin.New(), } api.Init() return api } -var apiV2 *ApiV2 +var api *Api +var apiOnce sync.Once -func GetApiV2() *ApiV2 { - if apiV2 != nil { - return apiV2 - } - apiV2 = NewApiV2() - return apiV2 +func GetApi() *Api { + apiOnce.Do(func() { + api = NewApi() + }) + return api } diff --git a/core/apps/server_v2.go b/core/apps/server.go similarity index 66% rename from core/apps/server_v2.go rename to core/apps/server.go index 71fd6c5a..55ba51eb 100644 --- a/core/apps/server_v2.go +++ b/core/apps/server.go @@ -10,9 +10,10 @@ import ( "github.com/spf13/viper" "net/http" _ "net/http/pprof" + "sync" ) -type ServerV2 struct { +type Server struct { // settings grpcAddress interfaces.Address @@ -21,14 +22,14 @@ type ServerV2 struct { // modules nodeSvc interfaces.NodeService - api *ApiV2 + api *Api dck *Docker // internals quit chan int } -func (app *ServerV2) Init() { +func (app *Server) Init() { // log node info app.logNodeInfo() @@ -36,7 +37,7 @@ func (app *ServerV2) Init() { app.initPprof() } -func (app *ServerV2) Start() { +func (app *Server) Start() { if utils.IsMaster() { // start docker app if utils.IsDocker() { @@ -51,31 +52,31 @@ func (app *ServerV2) Start() { go app.nodeSvc.Start() } -func (app *ServerV2) Wait() { +func (app *Server) Wait() { <-app.quit } -func (app *ServerV2) Stop() { +func (app *Server) Stop() { app.api.Stop() app.quit <- 1 } -func (app *ServerV2) GetApi() ApiApp { +func (app *Server) GetApi() ApiApp { return app.api } -func (app *ServerV2) GetNodeService() interfaces.NodeService { +func (app *Server) GetNodeService() interfaces.NodeService { return app.nodeSvc } -func (app *ServerV2) logNodeInfo() { +func (app *Server) logNodeInfo() { log.Infof("current node type: %s", utils.GetNodeType()) if utils.IsDocker() { log.Infof("running in docker container") } } -func (app *ServerV2) initPprof() { +func (app *Server) initPprof() { if viper.GetBool("pprof") { go func() { fmt.Println(http.ListenAndServe("0.0.0.0:6060", nil)) @@ -83,9 +84,9 @@ func (app *ServerV2) initPprof() { } } -func NewServerV2() (app NodeApp) { +func newServer() NodeApp { // server - svr := &ServerV2{ + svr := &Server{ WithConfigPath: config.NewConfigPathService(), quit: make(chan int, 1), } @@ -93,7 +94,7 @@ func NewServerV2() (app NodeApp) { // master modules if utils.IsMaster() { // api - svr.api = GetApiV2() + svr.api = GetApi() // docker if utils.IsDocker() { @@ -102,25 +103,21 @@ func NewServerV2() (app NodeApp) { } // node service - var err error if utils.IsMaster() { - svr.nodeSvc, err = service.GetMasterService() + svr.nodeSvc = service.GetMasterService() } else { - svr.nodeSvc, err = service.GetWorkerService() - } - if err != nil { - panic(err) + svr.nodeSvc = service.GetWorkerService() } return svr } -var serverV2 NodeApp +var server NodeApp +var serverOnce sync.Once -func GetServerV2() NodeApp { - if serverV2 != nil { - return serverV2 - } - serverV2 = NewServerV2() - return serverV2 +func GetServer() NodeApp { + serverOnce.Do(func() { + server = newServer() + }) + return server } diff --git a/core/cmd/server.go b/core/cmd/server.go index afa7cd42..dfb274bc 100644 --- a/core/cmd/server.go +++ b/core/cmd/server.go @@ -16,7 +16,7 @@ var serverCmd = &cobra.Command{ Long: `Start Crawlab node server that can serve as API, task scheduler, task runner, etc.`, Run: func(cmd *cobra.Command, args []string) { // app - svr := apps.GetServerV2() + svr := apps.GetServer() // start apps.Start(svr) diff --git a/core/controllers/base_v2.go b/core/controllers/base.go similarity index 86% rename from core/controllers/base_v2.go rename to core/controllers/base.go index a01bacab..2e6c6324 100644 --- a/core/controllers/base_v2.go +++ b/core/controllers/base.go @@ -17,12 +17,12 @@ type Action struct { HandlerFunc gin.HandlerFunc } -type BaseControllerV2[T any] struct { +type BaseController[T any] struct { modelSvc *service.ModelService[T] actions []Action } -func (ctr *BaseControllerV2[T]) GetById(c *gin.Context) { +func (ctr *BaseController[T]) GetById(c *gin.Context) { id, err := primitive.ObjectIDFromHex(c.Param("id")) if err != nil { HandleErrorBadRequest(c, err) @@ -38,7 +38,7 @@ func (ctr *BaseControllerV2[T]) GetById(c *gin.Context) { HandleSuccessWithData(c, model) } -func (ctr *BaseControllerV2[T]) GetList(c *gin.Context) { +func (ctr *BaseController[T]) GetList(c *gin.Context) { // get all if query field "all" is set true all := MustGetFilterAll(c) if all { @@ -50,7 +50,7 @@ func (ctr *BaseControllerV2[T]) GetList(c *gin.Context) { ctr.getList(c) } -func (ctr *BaseControllerV2[T]) Post(c *gin.Context) { +func (ctr *BaseController[T]) Post(c *gin.Context) { var model T if err := c.ShouldBindJSON(&model); err != nil { HandleErrorBadRequest(c, err) @@ -77,7 +77,7 @@ func (ctr *BaseControllerV2[T]) Post(c *gin.Context) { HandleSuccessWithData(c, result) } -func (ctr *BaseControllerV2[T]) PutById(c *gin.Context) { +func (ctr *BaseController[T]) PutById(c *gin.Context) { id, err := primitive.ObjectIDFromHex(c.Param("id")) if err != nil { HandleErrorBadRequest(c, err) @@ -108,7 +108,7 @@ func (ctr *BaseControllerV2[T]) PutById(c *gin.Context) { HandleSuccessWithData(c, result) } -func (ctr *BaseControllerV2[T]) PatchList(c *gin.Context) { +func (ctr *BaseController[T]) PatchList(c *gin.Context) { type Payload struct { Ids []primitive.ObjectID `json:"ids"` Update bson.M `json:"update"` @@ -136,7 +136,7 @@ func (ctr *BaseControllerV2[T]) PatchList(c *gin.Context) { HandleSuccess(c) } -func (ctr *BaseControllerV2[T]) DeleteById(c *gin.Context) { +func (ctr *BaseController[T]) DeleteById(c *gin.Context) { id, err := primitive.ObjectIDFromHex(c.Param("id")) if err != nil { HandleErrorBadRequest(c, err) @@ -151,7 +151,7 @@ func (ctr *BaseControllerV2[T]) DeleteById(c *gin.Context) { HandleSuccess(c) } -func (ctr *BaseControllerV2[T]) DeleteList(c *gin.Context) { +func (ctr *BaseController[T]) DeleteList(c *gin.Context) { type Payload struct { Ids []primitive.ObjectID `json:"ids"` } @@ -174,7 +174,7 @@ func (ctr *BaseControllerV2[T]) DeleteList(c *gin.Context) { HandleSuccess(c) } -func (ctr *BaseControllerV2[T]) getAll(c *gin.Context) { +func (ctr *BaseController[T]) getAll(c *gin.Context) { query := MustGetFilterQuery(c) sort := MustGetSortOption(c) if sort == nil { @@ -195,7 +195,7 @@ func (ctr *BaseControllerV2[T]) getAll(c *gin.Context) { HandleSuccessWithListData(c, models, total) } -func (ctr *BaseControllerV2[T]) getList(c *gin.Context) { +func (ctr *BaseController[T]) getList(c *gin.Context) { // params pagination := MustGetPagination(c) query := MustGetFilterQuery(c) @@ -227,8 +227,8 @@ func (ctr *BaseControllerV2[T]) getList(c *gin.Context) { HandleSuccessWithListData(c, models, total) } -func NewControllerV2[T any](actions ...Action) *BaseControllerV2[T] { - ctr := &BaseControllerV2[T]{ +func NewController[T any](actions ...Action) *BaseController[T] { + ctr := &BaseController[T]{ modelSvc: service.NewModelService[T](), actions: actions, } diff --git a/core/controllers/base_file_v2.go b/core/controllers/base_file.go similarity index 98% rename from core/controllers/base_file_v2.go rename to core/controllers/base_file.go index 7fd671b7..6e2186ef 100644 --- a/core/controllers/base_file_v2.go +++ b/core/controllers/base_file.go @@ -297,7 +297,7 @@ func GetBaseFileFsSvc(rootPath string) (svc interfaces.FsService, err error) { func getBaseFileFsSvc(rootPath string) (svc interfaces.FsService, err error) { workspacePath := viper.GetString("workspace") - fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, rootPath)) + fsSvc := fs.NewFsService(filepath.Join(workspacePath, rootPath)) return fsSvc, nil } diff --git a/core/controllers/base_v2_test.go b/core/controllers/base_test.go similarity index 84% rename from core/controllers/base_v2_test.go rename to core/controllers/base_test.go index f1c373f3..05197026 100644 --- a/core/controllers/base_v2_test.go +++ b/core/controllers/base_test.go @@ -6,7 +6,7 @@ import ( "encoding/json" "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/user" "github.com/spf13/viper" @@ -24,15 +24,15 @@ func init() { } // TestModel is a simple struct to be used as a model in tests -type TestModel models.TestModelV2 +type TestModel models.TestModel var TestToken string // SetupTestDB sets up the test database func SetupTestDB() { viper.Set("mongo.db", "testdb") - modelSvc := service.NewModelService[models.UserV2]() - u := models.UserV2{ + modelSvc := service.NewModelService[models.User]() + u := models.User{ Username: "admin", } id, err := modelSvc.InsertOne(u) @@ -41,7 +41,7 @@ func SetupTestDB() { } u.SetId(id) - userSvc, err := user.GetUserServiceV2() + userSvc, err := user.GetUserService() if err != nil { panic(err) } @@ -63,7 +63,7 @@ func CleanupTestDB() { mongo.GetMongoDb("testdb").Drop(context.Background()) } -func TestBaseControllerV2_GetById(t *testing.T) { +func TestBaseController_GetById(t *testing.T) { SetupTestDB() defer CleanupTestDB() @@ -72,11 +72,11 @@ func TestBaseControllerV2_GetById(t *testing.T) { assert.NoError(t, err) // Initialize the controller - ctr := controllers.NewControllerV2[TestModel]() + ctr := controllers.NewController[TestModel]() // Set up the router router := SetupRouter() - router.Use(middlewares.AuthorizationMiddlewareV2()) + router.Use(middlewares.AuthorizationMiddleware()) router.GET("/testmodels/:id", ctr.GetById) // Create a test request @@ -96,16 +96,16 @@ func TestBaseControllerV2_GetById(t *testing.T) { assert.Equal(t, "test", response.Data.Name) } -func TestBaseControllerV2_Post(t *testing.T) { +func TestBaseController_Post(t *testing.T) { SetupTestDB() defer CleanupTestDB() // Initialize the controller - ctr := controllers.NewControllerV2[TestModel]() + ctr := controllers.NewController[TestModel]() // Set up the router router := SetupRouter() - router.Use(middlewares.AuthorizationMiddlewareV2()) + router.Use(middlewares.AuthorizationMiddleware()) router.POST("/testmodels", ctr.Post) // Create a test request @@ -132,7 +132,7 @@ func TestBaseControllerV2_Post(t *testing.T) { assert.Equal(t, "test", result.Name) } -func TestBaseControllerV2_DeleteById(t *testing.T) { +func TestBaseController_DeleteById(t *testing.T) { SetupTestDB() defer CleanupTestDB() @@ -141,11 +141,11 @@ func TestBaseControllerV2_DeleteById(t *testing.T) { assert.NoError(t, err) // Initialize the controller - ctr := controllers.NewControllerV2[TestModel]() + ctr := controllers.NewController[TestModel]() // Set up the router router := SetupRouter() - router.Use(middlewares.AuthorizationMiddlewareV2()) + router.Use(middlewares.AuthorizationMiddleware()) router.DELETE("/testmodels/:id", ctr.DeleteById) // Create a test request diff --git a/core/controllers/export_v2.go b/core/controllers/export.go similarity index 100% rename from core/controllers/export_v2.go rename to core/controllers/export.go diff --git a/core/controllers/filter_v2.go b/core/controllers/filter.go similarity index 100% rename from core/controllers/filter_v2.go rename to core/controllers/filter.go diff --git a/core/controllers/login_v2.go b/core/controllers/login.go similarity index 95% rename from core/controllers/login_v2.go rename to core/controllers/login.go index 2bb08cef..082ff200 100644 --- a/core/controllers/login_v2.go +++ b/core/controllers/login.go @@ -16,7 +16,7 @@ func PostLogin(c *gin.Context) { HandleErrorBadRequest(c, err) return } - userSvc, err := user.GetUserServiceV2() + userSvc, err := user.GetUserService() if err != nil { HandleErrorInternalServerError(c, err) return diff --git a/core/controllers/project_v2.go b/core/controllers/project.go similarity index 77% rename from core/controllers/project_v2.go rename to core/controllers/project.go index a2947c09..fa1b2870 100644 --- a/core/controllers/project_v2.go +++ b/core/controllers/project.go @@ -2,7 +2,7 @@ package controllers import ( "errors" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "github.com/gin-gonic/gin" @@ -15,7 +15,7 @@ func GetProjectList(c *gin.Context) { // get all list all := MustGetFilterAll(c) if all { - NewControllerV2[models2.ProjectV2]().getAll(c) + NewController[models.Project]().getAll(c) return } @@ -25,7 +25,7 @@ func GetProjectList(c *gin.Context) { sort := MustGetSortOption(c) // get list - projects, err := service.NewModelService[models2.ProjectV2]().GetMany(query, &mongo.FindOptions{ + projects, err := service.NewModelService[models.Project]().GetMany(query, &mongo.FindOptions{ Sort: sort, Skip: pagination.Size * (pagination.Page - 1), Limit: pagination.Size, @@ -37,12 +37,12 @@ func GetProjectList(c *gin.Context) { return } if len(projects) == 0 { - HandleSuccessWithListData(c, []models2.ProjectV2{}, 0) + HandleSuccessWithListData(c, []models.Project{}, 0) return } // total count - total, err := service.NewModelService[models2.ProjectV2]().Count(query) + total, err := service.NewModelService[models.Project]().Count(query) if err != nil { HandleErrorInternalServerError(c, err) return @@ -61,7 +61,7 @@ func GetProjectList(c *gin.Context) { } // spiders - spiders, err := service.NewModelService[models2.SpiderV2]().GetMany(bson.M{ + spiders, err := service.NewModelService[models.Spider]().GetMany(bson.M{ "project_id": bson.M{ "$in": ids, }, @@ -80,7 +80,7 @@ func GetProjectList(c *gin.Context) { } // assign - var data []models2.ProjectV2 + var data []models.Project for _, p := range projects { p.Spiders = cache[p.Id] data = append(data, p) diff --git a/core/controllers/router_v2.go b/core/controllers/router.go similarity index 92% rename from core/controllers/router_v2.go rename to core/controllers/router.go index a295df50..3fe451c5 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router.go @@ -2,7 +2,7 @@ package controllers import ( "github.com/crawlab-team/crawlab/core/middlewares" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/gin-gonic/gin" "net/http" ) @@ -14,12 +14,12 @@ type RouterGroups struct { func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) { return &RouterGroups{ - AuthGroup: app.Group("/", middlewares.AuthorizationMiddlewareV2()), + AuthGroup: app.Group("/", middlewares.AuthorizationMiddleware()), AnonymousGroup: app.Group("/"), } } -func RegisterController[T any](group *gin.RouterGroup, basePath string, ctr *BaseControllerV2[T]) { +func RegisterController[T any](group *gin.RouterGroup, basePath string, ctr *BaseController[T]) { actionPaths := make(map[string]bool) for _, action := range ctr.actions { group.Handle(action.Method, basePath+action.Path, action.HandlerFunc) @@ -55,17 +55,17 @@ func InitRoutes(app *gin.Engine) (err error) { // routes groups groups := NewRouterGroups(app) - RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models2.DataCollectionV2]()) - RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models2.EnvironmentV2]()) - RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models2.NodeV2]()) - RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models2.ProjectV2]([]Action{ + RegisterController(groups.AuthGroup, "/data/collections", NewController[models.DataCollection]()) + RegisterController(groups.AuthGroup, "/environments", NewController[models.Environment]()) + RegisterController(groups.AuthGroup, "/nodes", NewController[models.Node]()) + RegisterController(groups.AuthGroup, "/projects", NewController[models.Project]([]Action{ { Method: http.MethodGet, Path: "", HandlerFunc: GetProjectList, }, }...)) - RegisterController(groups.AuthGroup, "/schedules", NewControllerV2[models2.ScheduleV2]([]Action{ + RegisterController(groups.AuthGroup, "/schedules", NewController[models.Schedule]([]Action{ { Method: http.MethodPost, Path: "", @@ -87,7 +87,7 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: PostScheduleDisable, }, }...)) - RegisterController(groups.AuthGroup, "/spiders", NewControllerV2[models2.SpiderV2]([]Action{ + RegisterController(groups.AuthGroup, "/spiders", NewController[models.Spider]([]Action{ { Method: http.MethodGet, Path: "/:id", @@ -179,7 +179,7 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetSpiderResults, }, }...)) - RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models2.TaskV2]([]Action{ + RegisterController(groups.AuthGroup, "/tasks", NewController[models.Task]([]Action{ { Method: http.MethodGet, Path: "/:id", @@ -221,14 +221,14 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetTaskLogs, }, }...)) - RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models2.TokenV2]([]Action{ + RegisterController(groups.AuthGroup, "/tokens", NewController[models.Token]([]Action{ { Method: http.MethodPost, Path: "", HandlerFunc: PostToken, }, }...)) - RegisterController(groups.AuthGroup, "/users", NewControllerV2[models2.UserV2]([]Action{ + RegisterController(groups.AuthGroup, "/users", NewController[models.User]([]Action{ { Method: http.MethodPost, Path: "", diff --git a/core/controllers/router_v2_test.go b/core/controllers/router_test.go similarity index 95% rename from core/controllers/router_v2_test.go rename to core/controllers/router_test.go index e0e6f530..e222ce8c 100644 --- a/core/controllers/router_v2_test.go +++ b/core/controllers/router_test.go @@ -2,7 +2,7 @@ package controllers_test import ( "github.com/crawlab-team/crawlab/core/controllers" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" "testing" @@ -28,7 +28,7 @@ func TestRouterGroups(t *testing.T) { func TestRegisterController_Routes(t *testing.T) { router := gin.Default() groups := controllers.NewRouterGroups(router) - ctr := controllers.NewControllerV2[models.TestModelV2]() + ctr := controllers.NewController[models.TestModel]() basePath := "/testmodels" controllers.RegisterController(groups.AuthGroup, basePath, ctr) diff --git a/core/controllers/schedule_v2.go b/core/controllers/schedule.go similarity index 76% rename from core/controllers/schedule_v2.go rename to core/controllers/schedule.go index 4aa7bede..74b37aac 100644 --- a/core/controllers/schedule_v2.go +++ b/core/controllers/schedule.go @@ -2,7 +2,7 @@ package controllers import ( errors2 "errors" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/schedule" "github.com/gin-gonic/gin" @@ -10,7 +10,7 @@ import ( ) func PostSchedule(c *gin.Context) { - var s models.ScheduleV2 + var s models.Schedule if err := c.ShouldBindJSON(&s); err != nil { HandleErrorBadRequest(c, err) return @@ -18,7 +18,7 @@ func PostSchedule(c *gin.Context) { u := GetUserFromContext(c) - modelSvc := service.NewModelService[models.ScheduleV2]() + modelSvc := service.NewModelService[models.Schedule]() s.SetCreated(u.Id) s.SetUpdated(u.Id) @@ -30,11 +30,7 @@ func PostSchedule(c *gin.Context) { s.Id = id if s.Enabled { - scheduleSvc, err := schedule.GetScheduleServiceV2() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } + scheduleSvc := schedule.GetScheduleService() if err := scheduleSvc.Enable(s, u.Id); err != nil { HandleErrorInternalServerError(c, err) return @@ -50,7 +46,7 @@ func PutScheduleById(c *gin.Context) { HandleErrorBadRequest(c, err) return } - var s models.ScheduleV2 + var s models.Schedule if err := c.ShouldBindJSON(&s); err != nil { HandleErrorBadRequest(c, err) return @@ -60,18 +56,14 @@ func PutScheduleById(c *gin.Context) { return } - modelSvc := service.NewModelService[models.ScheduleV2]() + modelSvc := service.NewModelService[models.Schedule]() err = modelSvc.ReplaceById(id, s) if err != nil { HandleErrorInternalServerError(c, err) return } - scheduleSvc, err := schedule.GetScheduleServiceV2() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } + scheduleSvc := schedule.GetScheduleService() u := GetUserFromContext(c) @@ -105,12 +97,8 @@ func postScheduleEnableDisableFunc(isEnable bool) func(c *gin.Context) { HandleErrorBadRequest(c, err) return } - svc, err := schedule.GetScheduleServiceV2() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - s, err := service.NewModelService[models.ScheduleV2]().GetById(id) + svc := schedule.GetScheduleService() + s, err := service.NewModelService[models.Schedule]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return diff --git a/core/controllers/setting_v2.go b/core/controllers/setting.go similarity index 81% rename from core/controllers/setting_v2.go rename to core/controllers/setting.go index f9777f56..03b4b8bf 100644 --- a/core/controllers/setting_v2.go +++ b/core/controllers/setting.go @@ -2,7 +2,7 @@ package controllers import ( "errors" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" @@ -14,7 +14,7 @@ func GetSetting(c *gin.Context) { key := c.Param("id") // setting - s, err := service.NewModelService[models.SettingV2]().GetOne(bson.M{"key": key}, nil) + s, err := service.NewModelService[models.Setting]().GetOne(bson.M{"key": key}, nil) if err != nil { if errors.Is(err, mongo.ErrNoDocuments) { HandleSuccess(c) @@ -32,7 +32,7 @@ func PostSetting(c *gin.Context) { key := c.Param("id") // settings - var s models.SettingV2 + var s models.Setting if err := c.ShouldBindJSON(&s); err != nil { HandleErrorInternalServerError(c, err) return @@ -47,7 +47,7 @@ func PostSetting(c *gin.Context) { s.SetCreated(u.Id) s.SetUpdated(u.Id) - id, err := service.NewModelService[models.SettingV2]().InsertOne(s) + id, err := service.NewModelService[models.Setting]().InsertOne(s) if err != nil { HandleErrorInternalServerError(c, err) return @@ -62,13 +62,13 @@ func PutSetting(c *gin.Context) { key := c.Param("id") // settings - var s models.SettingV2 + var s models.Setting if err := c.ShouldBindJSON(&s); err != nil { HandleErrorInternalServerError(c, err) return } - modelSvc := service.NewModelService[models.SettingV2]() + modelSvc := service.NewModelService[models.Setting]() // setting _s, err := modelSvc.GetOne(bson.M{"key": key}, nil) diff --git a/core/controllers/spider_v2.go b/core/controllers/spider.go similarity index 80% rename from core/controllers/spider_v2.go rename to core/controllers/spider.go index 17eb9bb6..e93a26c7 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider.go @@ -2,6 +2,7 @@ package controllers import ( "errors" + "github.com/crawlab-team/crawlab/core/models/models" "math" "os" "path/filepath" @@ -10,7 +11,6 @@ import ( "github.com/apex/log" "github.com/crawlab-team/crawlab/core/fs" "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/utils" @@ -30,7 +30,7 @@ func GetSpiderById(c *gin.Context) { HandleErrorBadRequest(c, err) return } - s, err := service.NewModelService[models2.SpiderV2]().GetById(id) + s, err := service.NewModelService[models.Spider]().GetById(id) if errors.Is(err, mongo2.ErrNoDocuments) { HandleErrorNotFound(c, err) return @@ -41,7 +41,7 @@ func GetSpiderById(c *gin.Context) { } // stat - s.Stat, err = service.NewModelService[models2.SpiderStatV2]().GetById(s.Id) + s.Stat, err = service.NewModelService[models.SpiderStat]().GetById(s.Id) if err != nil { if !errors.Is(err, mongo2.ErrNoDocuments) { HandleErrorInternalServerError(c, err) @@ -51,7 +51,7 @@ func GetSpiderById(c *gin.Context) { // data collection (compatible to old version) # TODO: remove in the future if s.ColName == "" && !s.ColId.IsZero() { - col, err := service.NewModelService[models2.DataCollectionV2]().GetById(s.ColId) + col, err := service.NewModelService[models.DataCollection]().GetById(s.ColId) if err != nil { if !errors.Is(err, mongo2.ErrNoDocuments) { HandleErrorInternalServerError(c, err) @@ -64,7 +64,7 @@ func GetSpiderById(c *gin.Context) { // git if utils.IsPro() && !s.GitId.IsZero() { - s.Git, err = service.NewModelService[models2.GitV2]().GetById(s.GitId) + s.Git, err = service.NewModelService[models.Git]().GetById(s.GitId) if err != nil { if !errors.Is(err, mongo2.ErrNoDocuments) { HandleErrorInternalServerError(c, err) @@ -80,14 +80,14 @@ func GetSpiderList(c *gin.Context) { // get all list all := MustGetFilterAll(c) if all { - NewControllerV2[models2.SpiderV2]().getAll(c) + NewController[models.Spider]().getAll(c) return } // get list withStats := c.Query("stats") if withStats == "" { - NewControllerV2[models2.SpiderV2]().GetList(c) + NewController[models.Spider]().GetList(c) return } @@ -102,7 +102,7 @@ func getSpiderListWithStats(c *gin.Context) { sort := MustGetSortOption(c) // get list - spiders, err := service.NewModelService[models2.SpiderV2]().GetMany(query, &mongo.FindOptions{ + spiders, err := service.NewModelService[models.Spider]().GetMany(query, &mongo.FindOptions{ Sort: sort, Skip: pagination.Size * (pagination.Page - 1), Limit: pagination.Size, @@ -114,7 +114,7 @@ func getSpiderListWithStats(c *gin.Context) { return } if len(spiders) == 0 { - HandleSuccessWithListData(c, []models2.SpiderV2{}, 0) + HandleSuccessWithListData(c, []models.Spider{}, 0) return } @@ -129,21 +129,21 @@ func getSpiderListWithStats(c *gin.Context) { } // total count - total, err := service.NewModelService[models2.SpiderV2]().Count(query) + total, err := service.NewModelService[models.Spider]().Count(query) if err != nil { HandleErrorInternalServerError(c, err) return } // stat list - spiderStats, err := service.NewModelService[models2.SpiderStatV2]().GetMany(bson.M{"_id": bson.M{"$in": ids}}, nil) + spiderStats, err := service.NewModelService[models.SpiderStat]().GetMany(bson.M{"_id": bson.M{"$in": ids}}, nil) if err != nil { HandleErrorInternalServerError(c, err) return } // cache stat list to dict - dict := map[primitive.ObjectID]models2.SpiderStatV2{} + dict := map[primitive.ObjectID]models.SpiderStat{} var taskIds []primitive.ObjectID for _, st := range spiderStats { if st.Tasks > 0 { @@ -160,9 +160,9 @@ func getSpiderListWithStats(c *gin.Context) { } // task list and stats - var tasks []models2.TaskV2 - dictTask := map[primitive.ObjectID]models2.TaskV2{} - dictTaskStat := map[primitive.ObjectID]models2.TaskStatV2{} + var tasks []models.Task + dictTask := map[primitive.ObjectID]models.Task{} + dictTaskStat := map[primitive.ObjectID]models.TaskStat{} if len(taskIds) > 0 { // task list queryTask := bson.M{ @@ -170,14 +170,14 @@ func getSpiderListWithStats(c *gin.Context) { "$in": taskIds, }, } - tasks, err = service.NewModelService[models2.TaskV2]().GetMany(queryTask, nil) + tasks, err = service.NewModelService[models.Task]().GetMany(queryTask, nil) if err != nil { HandleErrorInternalServerError(c, err) return } // task stats list - taskStats, err := service.NewModelService[models2.TaskStatV2]().GetMany(queryTask, nil) + taskStats, err := service.NewModelService[models.TaskStat]().GetMany(queryTask, nil) if err != nil { HandleErrorInternalServerError(c, err) return @@ -199,9 +199,9 @@ func getSpiderListWithStats(c *gin.Context) { } // git list - var gits []models2.GitV2 + var gits []models.Git if len(gitIds) > 0 && utils.IsPro() { - gits, err = service.NewModelService[models2.GitV2]().GetMany(bson.M{"_id": bson.M{"$in": gitIds}}, nil) + gits, err = service.NewModelService[models.Git]().GetMany(bson.M{"_id": bson.M{"$in": gitIds}}, nil) if err != nil { HandleErrorInternalServerError(c, err) return @@ -209,13 +209,13 @@ func getSpiderListWithStats(c *gin.Context) { } // cache git list to dict - dictGit := map[primitive.ObjectID]models2.GitV2{} + dictGit := map[primitive.ObjectID]models.Git{} for _, g := range gits { dictGit[g.Id] = g } // iterate list again - var data []models2.SpiderV2 + var data []models.Spider for _, s := range spiders { // spider stat st, ok := dict[s.Id] @@ -247,7 +247,7 @@ func getSpiderListWithStats(c *gin.Context) { func PostSpider(c *gin.Context) { // bind - var s models2.SpiderV2 + var s models.Spider if err := c.ShouldBindJSON(&s); err != nil { HandleErrorBadRequest(c, err) return @@ -259,7 +259,7 @@ func PostSpider(c *gin.Context) { // add s.SetCreated(u.Id) s.SetUpdated(u.Id) - id, err := service.NewModelService[models2.SpiderV2]().InsertOne(s) + id, err := service.NewModelService[models.Spider]().InsertOne(s) if err != nil { HandleErrorInternalServerError(c, err) return @@ -267,11 +267,11 @@ func PostSpider(c *gin.Context) { s.SetId(id) // add stat - st := models2.SpiderStatV2{} + st := models.SpiderStat{} st.SetId(id) st.SetCreated(u.Id) st.SetUpdated(u.Id) - _, err = service.NewModelService[models2.SpiderStatV2]().InsertOne(st) + _, err = service.NewModelService[models.SpiderStat]().InsertOne(st) if err != nil { HandleErrorInternalServerError(c, err) return @@ -300,7 +300,7 @@ func PutSpiderById(c *gin.Context) { } // bind - var s models2.SpiderV2 + var s models.Spider if err := c.ShouldBindJSON(&s); err != nil { HandleErrorBadRequest(c, err) return @@ -308,7 +308,7 @@ func PutSpiderById(c *gin.Context) { u := GetUserFromContext(c) - modelSvc := service.NewModelService[models2.SpiderV2]() + modelSvc := service.NewModelService[models.Spider]() // save s.SetUpdated(u.Id) @@ -336,7 +336,7 @@ func DeleteSpiderById(c *gin.Context) { } // spider - s, err := service.NewModelService[models2.SpiderV2]().GetById(id) + s, err := service.NewModelService[models.Spider]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return @@ -344,19 +344,19 @@ func DeleteSpiderById(c *gin.Context) { if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { // delete spider - err = service.NewModelService[models2.SpiderV2]().DeleteById(id) + err = service.NewModelService[models.Spider]().DeleteById(id) if err != nil { return err } // delete spider stat - err = service.NewModelService[models2.SpiderStatV2]().DeleteById(id) + err = service.NewModelService[models.SpiderStat]().DeleteById(id) if err != nil { return err } // related tasks - tasks, err := service.NewModelService[models2.TaskV2]().GetMany(bson.M{"spider_id": id}, nil) + tasks, err := service.NewModelService[models.Task]().GetMany(bson.M{"spider_id": id}, nil) if err != nil { return err } @@ -372,13 +372,13 @@ func DeleteSpiderById(c *gin.Context) { } // delete related tasks - err = service.NewModelService[models2.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}) + err = service.NewModelService[models.Task]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}) if err != nil { return err } // delete related task stats - err = service.NewModelService[models2.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}) + err = service.NewModelService[models.TaskStat]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}) if err != nil { return err } @@ -433,7 +433,7 @@ func DeleteSpiderList(c *gin.Context) { } // Fetch spiders before deletion - spiders, err := service.NewModelService[models2.SpiderV2]().GetMany(bson.M{ + spiders, err := service.NewModelService[models.Spider]().GetMany(bson.M{ "_id": bson.M{ "$in": payload.Ids, }, @@ -445,7 +445,7 @@ func DeleteSpiderList(c *gin.Context) { if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { // delete spiders - if err := service.NewModelService[models2.SpiderV2]().DeleteMany(bson.M{ + if err := service.NewModelService[models.Spider]().DeleteMany(bson.M{ "_id": bson.M{ "$in": payload.Ids, }, @@ -454,7 +454,7 @@ func DeleteSpiderList(c *gin.Context) { } // delete spider stats - if err := service.NewModelService[models2.SpiderStatV2]().DeleteMany(bson.M{ + if err := service.NewModelService[models.SpiderStat]().DeleteMany(bson.M{ "_id": bson.M{ "$in": payload.Ids, }, @@ -463,7 +463,7 @@ func DeleteSpiderList(c *gin.Context) { } // related tasks - tasks, err := service.NewModelService[models2.TaskV2]().GetMany(bson.M{"spider_id": bson.M{"$in": payload.Ids}}, nil) + tasks, err := service.NewModelService[models.Task]().GetMany(bson.M{"spider_id": bson.M{"$in": payload.Ids}}, nil) if err != nil { return err } @@ -479,12 +479,12 @@ func DeleteSpiderList(c *gin.Context) { } // delete related tasks - if err := service.NewModelService[models2.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { + if err := service.NewModelService[models.Task]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { return err } // delete related task stats - if err := service.NewModelService[models2.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { + if err := service.NewModelService[models.TaskStat]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { return err } @@ -514,7 +514,7 @@ func DeleteSpiderList(c *gin.Context) { wg := sync.WaitGroup{} wg.Add(len(spiders)) for i := range spiders { - go func(s *models2.SpiderV2) { + go func(s *models.Spider) { defer wg.Done() // Skip spider with git @@ -653,11 +653,7 @@ func PostSpiderRun(c *gin.Context) { opts.UserId = u.GetId() } - adminSvc, err := admin.GetSpiderAdminService() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } + adminSvc := admin.GetSpiderAdminService() // schedule taskIds, err := adminSvc.Schedule(id, &opts) @@ -676,7 +672,7 @@ func GetSpiderResults(c *gin.Context) { return } - s, err := service.NewModelService[models2.SpiderV2]().GetById(id) + s, err := service.NewModelService[models.Spider]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return @@ -708,15 +704,15 @@ func GetSpiderResults(c *gin.Context) { HandleSuccessWithListData(c, results, total) } -func getSpiderFsSvc(s *models2.SpiderV2) (svc interfaces.FsService, err error) { +func getSpiderFsSvc(s *models.Spider) (svc interfaces.FsService, err error) { workspacePath := viper.GetString("workspace") - fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, s.Id.Hex())) + fsSvc := fs.NewFsService(filepath.Join(workspacePath, s.Id.Hex())) return fsSvc, nil } func getSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsService, err error) { - s, err := service.NewModelService[models2.SpiderV2]().GetById(id) + s, err := service.NewModelService[models.Spider]().GetById(id) if err != nil { log.Errorf("failed to get spider: %s", err.Error()) trace.PrintError(err) @@ -733,7 +729,7 @@ func getSpiderRootPath(c *gin.Context) (rootPath string, err error) { } // spider - s, err := service.NewModelService[models2.SpiderV2]().GetById(id) + s, err := service.NewModelService[models.Spider]().GetById(id) if err != nil { return "", err } diff --git a/core/controllers/spider_v2_test.go b/core/controllers/spider_test.go similarity index 65% rename from core/controllers/spider_v2_test.go rename to core/controllers/spider_test.go index da17cacc..10c2f4e0 100644 --- a/core/controllers/spider_v2_test.go +++ b/core/controllers/spider_test.go @@ -3,13 +3,13 @@ package controllers_test import ( "bytes" "encoding/json" + "github.com/crawlab-team/crawlab/core/models/models" "net/http" "net/http/httptest" "testing" "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" @@ -25,10 +25,10 @@ func TestCreateSpider(t *testing.T) { gin.SetMode(gin.TestMode) router := gin.Default() - router.Use(middlewares.AuthorizationMiddlewareV2()) + router.Use(middlewares.AuthorizationMiddleware()) router.POST("/spiders", controllers.PostSpider) - payload := models.SpiderV2{ + payload := models.Spider{ Name: "Test Spider", ColName: "test_spiders", } @@ -41,7 +41,7 @@ func TestCreateSpider(t *testing.T) { assert.Equal(t, http.StatusOK, resp.Code) - var response controllers.Response[models.SpiderV2] + var response controllers.Response[models.Spider] err := json.Unmarshal(resp.Body.Bytes(), &response) require.Nil(t, err) assert.False(t, response.Data.Id.IsZero()) @@ -55,18 +55,18 @@ func TestGetSpiderById(t *testing.T) { gin.SetMode(gin.TestMode) router := gin.Default() - router.Use(middlewares.AuthorizationMiddlewareV2()) + router.Use(middlewares.AuthorizationMiddleware()) router.GET("/spiders/:id", controllers.GetSpiderById) - model := models.SpiderV2{ + model := models.Spider{ Name: "Test Spider", ColName: "test_spiders", } - id, err := service.NewModelService[models.SpiderV2]().InsertOne(model) + id, err := service.NewModelService[models.Spider]().InsertOne(model) require.Nil(t, err) - ts := models.SpiderStatV2{} + ts := models.SpiderStat{} ts.SetId(id) - _, err = service.NewModelService[models.SpiderStatV2]().InsertOne(ts) + _, err = service.NewModelService[models.SpiderStat]().InsertOne(ts) require.Nil(t, err) req, _ := http.NewRequest("GET", "/spiders/"+id.Hex(), nil) @@ -77,7 +77,7 @@ func TestGetSpiderById(t *testing.T) { assert.Equal(t, http.StatusOK, resp.Code) - var response controllers.Response[models.SpiderV2] + var response controllers.Response[models.Spider] err = json.Unmarshal(resp.Body.Bytes(), &response) require.Nil(t, err) assert.Equal(t, model.Name, response.Data.Name) @@ -90,22 +90,22 @@ func TestUpdateSpiderById(t *testing.T) { gin.SetMode(gin.TestMode) router := gin.Default() - router.Use(middlewares.AuthorizationMiddlewareV2()) + router.Use(middlewares.AuthorizationMiddleware()) router.PUT("/spiders/:id", controllers.PutSpiderById) - model := models.SpiderV2{ + model := models.Spider{ Name: "Test Spider", ColName: "test_spiders", } - id, err := service.NewModelService[models.SpiderV2]().InsertOne(model) + id, err := service.NewModelService[models.Spider]().InsertOne(model) require.Nil(t, err) - ts := models.SpiderStatV2{} + ts := models.SpiderStat{} ts.SetId(id) - _, err = service.NewModelService[models.SpiderStatV2]().InsertOne(ts) + _, err = service.NewModelService[models.SpiderStat]().InsertOne(ts) require.Nil(t, err) spiderId := id.Hex() - payload := models.SpiderV2{ + payload := models.Spider{ Name: "Updated Spider", ColName: "test_spider", } @@ -119,12 +119,12 @@ func TestUpdateSpiderById(t *testing.T) { assert.Equal(t, http.StatusOK, resp.Code) - var response controllers.Response[models.SpiderV2] + var response controllers.Response[models.Spider] err = json.Unmarshal(resp.Body.Bytes(), &response) require.Nil(t, err) assert.Equal(t, payload.Name, response.Data.Name) - svc := service.NewModelService[models.SpiderV2]() + svc := service.NewModelService[models.Spider]() resModel, err := svc.GetById(id) require.Nil(t, err) assert.Equal(t, payload.Name, resModel.Name) @@ -137,26 +137,26 @@ func TestDeleteSpiderById(t *testing.T) { gin.SetMode(gin.TestMode) router := gin.Default() - router.Use(middlewares.AuthorizationMiddlewareV2()) + router.Use(middlewares.AuthorizationMiddleware()) router.DELETE("/spiders/:id", controllers.DeleteSpiderById) - model := models.SpiderV2{ + model := models.Spider{ Name: "Test Spider", ColName: "test_spiders", } - id, err := service.NewModelService[models.SpiderV2]().InsertOne(model) + id, err := service.NewModelService[models.Spider]().InsertOne(model) require.Nil(t, err) - ts := models.SpiderStatV2{} + ts := models.SpiderStat{} ts.SetId(id) - _, err = service.NewModelService[models.SpiderStatV2]().InsertOne(ts) + _, err = service.NewModelService[models.SpiderStat]().InsertOne(ts) require.Nil(t, err) - task := models.TaskV2{} + task := models.Task{} task.SpiderId = id - taskId, err := service.NewModelService[models.TaskV2]().InsertOne(task) + taskId, err := service.NewModelService[models.Task]().InsertOne(task) require.Nil(t, err) - taskStat := models.TaskStatV2{} + taskStat := models.TaskStat{} taskStat.SetId(taskId) - _, err = service.NewModelService[models.TaskStatV2]().InsertOne(taskStat) + _, err = service.NewModelService[models.TaskStat]().InsertOne(taskStat) require.Nil(t, err) req, _ := http.NewRequest("DELETE", "/spiders/"+id.Hex(), nil) @@ -167,14 +167,14 @@ func TestDeleteSpiderById(t *testing.T) { assert.Equal(t, http.StatusOK, resp.Code) - _, err = service.NewModelService[models.SpiderV2]().GetById(id) + _, err = service.NewModelService[models.Spider]().GetById(id) assert.NotNil(t, err) - _, err = service.NewModelService[models.SpiderStatV2]().GetById(id) + _, err = service.NewModelService[models.SpiderStat]().GetById(id) assert.NotNil(t, err) - taskCount, err := service.NewModelService[models.TaskV2]().Count(bson.M{"spider_id": id}) + taskCount, err := service.NewModelService[models.Task]().Count(bson.M{"spider_id": id}) require.Nil(t, err) assert.Equal(t, 0, taskCount) - taskStatCount, err := service.NewModelService[models.TaskStatV2]().Count(bson.M{"_id": taskId}) + taskStatCount, err := service.NewModelService[models.TaskStat]().Count(bson.M{"_id": taskId}) require.Nil(t, err) assert.Equal(t, 0, taskStatCount) @@ -187,10 +187,10 @@ func TestDeleteSpiderList(t *testing.T) { gin.SetMode(gin.TestMode) router := gin.Default() - router.Use(middlewares.AuthorizationMiddlewareV2()) + router.Use(middlewares.AuthorizationMiddleware()) router.DELETE("/spiders", controllers.DeleteSpiderList) - modelList := []models.SpiderV2{ + modelList := []models.Spider{ { Name: "Test Name 1", ColName: "test_spiders", @@ -202,19 +202,19 @@ func TestDeleteSpiderList(t *testing.T) { var ids []primitive.ObjectID var taskIds []primitive.ObjectID for _, model := range modelList { - id, err := service.NewModelService[models.SpiderV2]().InsertOne(model) + id, err := service.NewModelService[models.Spider]().InsertOne(model) require.Nil(t, err) - ts := models.SpiderStatV2{} + ts := models.SpiderStat{} ts.SetId(id) - _, err = service.NewModelService[models.SpiderStatV2]().InsertOne(ts) + _, err = service.NewModelService[models.SpiderStat]().InsertOne(ts) require.Nil(t, err) - task := models.TaskV2{} + task := models.Task{} task.SpiderId = id - taskId, err := service.NewModelService[models.TaskV2]().InsertOne(task) + taskId, err := service.NewModelService[models.Task]().InsertOne(task) require.Nil(t, err) - taskStat := models.TaskStatV2{} + taskStat := models.TaskStat{} taskStat.SetId(taskId) - _, err = service.NewModelService[models.TaskStatV2]().InsertOne(taskStat) + _, err = service.NewModelService[models.TaskStat]().InsertOne(taskStat) require.Nil(t, err) ids = append(ids, id) taskIds = append(taskIds, taskId) @@ -234,16 +234,16 @@ func TestDeleteSpiderList(t *testing.T) { assert.Equal(t, http.StatusOK, resp.Code) - spiderCount, err := service.NewModelService[models.SpiderV2]().Count(bson.M{"_id": bson.M{"$in": ids}}) + spiderCount, err := service.NewModelService[models.Spider]().Count(bson.M{"_id": bson.M{"$in": ids}}) require.Nil(t, err) assert.Equal(t, 0, spiderCount) - spiderStatCount, err := service.NewModelService[models.SpiderStatV2]().Count(bson.M{"_id": bson.M{"$in": ids}}) + spiderStatCount, err := service.NewModelService[models.SpiderStat]().Count(bson.M{"_id": bson.M{"$in": ids}}) require.Nil(t, err) assert.Equal(t, 0, spiderStatCount) - taskCount, err := service.NewModelService[models.TaskV2]().Count(bson.M{"_id": bson.M{"$in": taskIds}}) + taskCount, err := service.NewModelService[models.Task]().Count(bson.M{"_id": bson.M{"$in": taskIds}}) require.Nil(t, err) assert.Equal(t, 0, taskCount) - taskStatCount, err := service.NewModelService[models.TaskStatV2]().Count(bson.M{"_id": bson.M{"$in": taskIds}}) + taskStatCount, err := service.NewModelService[models.TaskStat]().Count(bson.M{"_id": bson.M{"$in": taskIds}}) require.Nil(t, err) assert.Equal(t, 0, taskStatCount) } diff --git a/core/controllers/stats_v2.go b/core/controllers/stats.go similarity index 100% rename from core/controllers/stats_v2.go rename to core/controllers/stats.go diff --git a/core/controllers/sync_v2.go b/core/controllers/sync.go similarity index 100% rename from core/controllers/sync_v2.go rename to core/controllers/sync.go diff --git a/core/controllers/system_info_v2.go b/core/controllers/system_info.go similarity index 100% rename from core/controllers/system_info_v2.go rename to core/controllers/system_info.go diff --git a/core/controllers/task_v2.go b/core/controllers/task.go similarity index 78% rename from core/controllers/task_v2.go rename to core/controllers/task.go index 926e9c35..dfa681d0 100644 --- a/core/controllers/task_v2.go +++ b/core/controllers/task.go @@ -5,7 +5,7 @@ import ( log2 "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/task/log" @@ -32,7 +32,7 @@ func GetTaskById(c *gin.Context) { } // task - t, err := service.NewModelService[models.TaskV2]().GetById(id) + t, err := service.NewModelService[models.Task]().GetById(id) if errors.Is(err, mongo2.ErrNoDocuments) { HandleErrorNotFound(c, err) return @@ -50,21 +50,21 @@ func GetTaskById(c *gin.Context) { // spider if !t.SpiderId.IsZero() { - t.Spider, _ = service.NewModelService[models.SpiderV2]().GetById(t.SpiderId) + t.Spider, _ = service.NewModelService[models.Spider]().GetById(t.SpiderId) } // schedule if !t.ScheduleId.IsZero() { - t.Schedule, _ = service.NewModelService[models.ScheduleV2]().GetById(t.ScheduleId) + t.Schedule, _ = service.NewModelService[models.Schedule]().GetById(t.ScheduleId) } // node if !t.NodeId.IsZero() { - t.Node, _ = service.NewModelService[models.NodeV2]().GetById(t.NodeId) + t.Node, _ = service.NewModelService[models.Node]().GetById(t.NodeId) } // task stat - t.Stat, _ = service.NewModelService[models.TaskStatV2]().GetById(id) + t.Stat, _ = service.NewModelService[models.TaskStat]().GetById(id) HandleSuccessWithData(c, t) } @@ -72,7 +72,7 @@ func GetTaskById(c *gin.Context) { func GetTaskList(c *gin.Context) { withStats := c.Query("stats") if withStats == "" { - NewControllerV2[models.TaskV2]().GetList(c) + NewController[models.Task]().GetList(c) return } @@ -82,7 +82,7 @@ func GetTaskList(c *gin.Context) { sort := MustGetSortOption(c) // get tasks - tasks, err := service.NewModelService[models.TaskV2]().GetMany(query, &mongo.FindOptions{ + tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo.FindOptions{ Sort: sort, Skip: pagination.Size * (pagination.Page - 1), Limit: pagination.Size, @@ -111,14 +111,14 @@ func GetTaskList(c *gin.Context) { } // total count - total, err := service.NewModelService[models.TaskV2]().Count(query) + total, err := service.NewModelService[models.Task]().Count(query) if err != nil { HandleErrorInternalServerError(c, err) return } // stat list - stats, err := service.NewModelService[models.TaskStatV2]().GetMany(bson.M{ + stats, err := service.NewModelService[models.TaskStat]().GetMany(bson.M{ "_id": bson.M{ "$in": taskIds, }, @@ -129,13 +129,13 @@ func GetTaskList(c *gin.Context) { } // cache stat list to dict - statsDict := map[primitive.ObjectID]models.TaskStatV2{} + statsDict := map[primitive.ObjectID]models.TaskStat{} for _, s := range stats { statsDict[s.Id] = s } // spider list - spiders, err := service.NewModelService[models.SpiderV2]().GetMany(bson.M{ + spiders, err := service.NewModelService[models.Spider]().GetMany(bson.M{ "_id": bson.M{ "$in": spiderIds, }, @@ -146,7 +146,7 @@ func GetTaskList(c *gin.Context) { } // cache spider list to dict - spiderDict := map[primitive.ObjectID]models.SpiderV2{} + spiderDict := map[primitive.ObjectID]models.Spider{} for _, s := range spiders { spiderDict[s.Id] = s } @@ -180,22 +180,22 @@ func DeleteTaskById(c *gin.Context) { // delete in db if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { // delete task - _, err = service.NewModelService[models.TaskV2]().GetById(id) + _, err = service.NewModelService[models.Task]().GetById(id) if err != nil { return err } - err = service.NewModelService[models.TaskV2]().DeleteById(id) + err = service.NewModelService[models.Task]().DeleteById(id) if err != nil { return err } // delete task stat - _, err = service.NewModelService[models.TaskStatV2]().GetById(id) + _, err = service.NewModelService[models.TaskStat]().GetById(id) if err != nil { log2.Warnf("delete task stat error: %s", err.Error()) return nil } - err = service.NewModelService[models.TaskStatV2]().DeleteById(id) + err = service.NewModelService[models.TaskStat]().DeleteById(id) if err != nil { log2.Warnf("delete task stat error: %s", err.Error()) return nil @@ -227,7 +227,7 @@ func DeleteList(c *gin.Context) { if err := mongo.RunTransaction(func(context mongo2.SessionContext) error { // delete tasks - if err := service.NewModelService[models.TaskV2]().DeleteMany(bson.M{ + if err := service.NewModelService[models.Task]().DeleteMany(bson.M{ "_id": bson.M{ "$in": payload.Ids, }, @@ -236,7 +236,7 @@ func DeleteList(c *gin.Context) { } // delete task stats - if err := service.NewModelService[models.TaskV2]().DeleteMany(bson.M{ + if err := service.NewModelService[models.Task]().DeleteMany(bson.M{ "_id": bson.M{ "$in": payload.Ids, }, @@ -271,7 +271,7 @@ func DeleteList(c *gin.Context) { func PostTaskRun(c *gin.Context) { // task - var t models.TaskV2 + var t models.Task if err := c.ShouldBindJSON(&t); err != nil { HandleErrorBadRequest(c, err) return @@ -284,7 +284,7 @@ func PostTaskRun(c *gin.Context) { } // spider - s, err := service.NewModelService[models.SpiderV2]().GetById(t.SpiderId) + s, err := service.NewModelService[models.Spider]().GetById(t.SpiderId) if err != nil { HandleErrorInternalServerError(c, err) return @@ -305,11 +305,7 @@ func PostTaskRun(c *gin.Context) { } // run - adminSvc, err := admin.GetSpiderAdminService() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } + adminSvc := admin.GetSpiderAdminService() taskIds, err := adminSvc.Schedule(s.Id, opts) if err != nil { HandleErrorInternalServerError(c, err) @@ -329,7 +325,7 @@ func PostTaskRestart(c *gin.Context) { } // task - t, err := service.NewModelService[models.TaskV2]().GetById(id) + t, err := service.NewModelService[models.Task]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return @@ -350,11 +346,7 @@ func PostTaskRestart(c *gin.Context) { } // run - adminSvc, err := admin.GetSpiderAdminService() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } + adminSvc := admin.GetSpiderAdminService() taskIds, err := adminSvc.Schedule(t.SpiderId, opts) if err != nil { HandleErrorInternalServerError(c, err) @@ -384,7 +376,7 @@ func PostTaskCancel(c *gin.Context) { } // task - t, err := service.NewModelService[models.TaskV2]().GetById(id) + t, err := service.NewModelService[models.Task]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return @@ -399,15 +391,12 @@ func PostTaskCancel(c *gin.Context) { u := GetUserFromContext(c) // cancel - schedulerSvc, err := scheduler.GetTaskSchedulerService() + schedulerSvc := scheduler.GetTaskSchedulerService() + err = schedulerSvc.Cancel(id, u.Id, p.Force) if err != nil { HandleErrorInternalServerError(c, err) return } - if err := schedulerSvc.Cancel(id, u.Id, p.Force); err != nil { - HandleErrorInternalServerError(c, err) - return - } HandleSuccess(c) } @@ -428,11 +417,7 @@ func GetTaskLogs(c *gin.Context) { } // logs - logDriver, err := log.GetFileLogDriver() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } + logDriver := log.GetFileLogDriver() logs, err := logDriver.Find(id.Hex(), "", (p.Page-1)*p.Size, p.Size) if err != nil { if strings.HasSuffix(err.Error(), "Status:404 Not Found") { diff --git a/core/controllers/token_v2.go b/core/controllers/token.go similarity index 77% rename from core/controllers/token_v2.go rename to core/controllers/token.go index daefc7a0..6100e178 100644 --- a/core/controllers/token_v2.go +++ b/core/controllers/token.go @@ -1,19 +1,19 @@ package controllers import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/user" "github.com/gin-gonic/gin" ) func PostToken(c *gin.Context) { - var t models.TokenV2 + var t models.Token if err := c.ShouldBindJSON(&t); err != nil { HandleErrorBadRequest(c, err) return } - svc, err := user.GetUserServiceV2() + svc, err := user.GetUserService() if err != nil { HandleErrorInternalServerError(c, err) return @@ -26,7 +26,7 @@ func PostToken(c *gin.Context) { HandleErrorInternalServerError(c, err) return } - _, err = service.NewModelService[models.TokenV2]().InsertOne(t) + _, err = service.NewModelService[models.Token]().InsertOne(t) if err != nil { HandleErrorInternalServerError(c, err) return diff --git a/core/controllers/user_v2.go b/core/controllers/user.go similarity index 84% rename from core/controllers/user_v2.go rename to core/controllers/user.go index 97a3edc1..f1b7eb0b 100644 --- a/core/controllers/user_v2.go +++ b/core/controllers/user.go @@ -1,7 +1,7 @@ package controllers import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" "github.com/gin-gonic/gin" @@ -20,7 +20,7 @@ func PostUser(c *gin.Context) { return } u := GetUserFromContext(c) - model := models.UserV2{ + model := models.User{ Username: payload.Username, Password: utils.EncryptMd5(payload.Password), Role: payload.Role, @@ -28,13 +28,13 @@ func PostUser(c *gin.Context) { } model.SetCreated(u.Id) model.SetUpdated(u.Id) - id, err := service.NewModelService[models.UserV2]().InsertOne(model) + id, err := service.NewModelService[models.User]().InsertOne(model) if err != nil { HandleErrorInternalServerError(c, err) return } - result, err := service.NewModelService[models.UserV2]().GetById(id) + result, err := service.NewModelService[models.User]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return @@ -63,7 +63,7 @@ func PostUserChangePassword(c *gin.Context) { // get user u := GetUserFromContext(c) - modelSvc := service.NewModelService[models.UserV2]() + modelSvc := service.NewModelService[models.User]() // update password user, err := modelSvc.GetById(id) @@ -84,7 +84,7 @@ func PostUserChangePassword(c *gin.Context) { func GetUserMe(c *gin.Context) { u := GetUserFromContext(c) - _u, err := service.NewModelService[models.UserV2]().GetById(u.Id) + _u, err := service.NewModelService[models.User]().GetById(u.Id) if err != nil { HandleErrorInternalServerError(c, err) return @@ -94,7 +94,7 @@ func GetUserMe(c *gin.Context) { func PutUserById(c *gin.Context) { // get payload - var user models.UserV2 + var user models.User if err := c.ShouldBindJSON(&user); err != nil { HandleErrorBadRequest(c, err) return @@ -103,7 +103,7 @@ func PutUserById(c *gin.Context) { // get user u := GetUserFromContext(c) - modelSvc := service.NewModelService[models.UserV2]() + modelSvc := service.NewModelService[models.User]() // update user userDb, err := modelSvc.GetById(u.Id) diff --git a/core/controllers/user_v2_test.go b/core/controllers/user_test.go similarity index 82% rename from core/controllers/user_v2_test.go rename to core/controllers/user_test.go index 53a62f1e..c6c8a1ff 100644 --- a/core/controllers/user_v2_test.go +++ b/core/controllers/user_test.go @@ -3,7 +3,7 @@ package controllers_test import ( "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" @@ -17,14 +17,14 @@ func TestPostUserChangePassword_Success(t *testing.T) { SetupTestDB() defer CleanupTestDB() - modelSvc := service.NewModelService[models.UserV2]() - u := models.UserV2{} + modelSvc := service.NewModelService[models.User]() + u := models.User{} id, err := modelSvc.InsertOne(u) assert.Nil(t, err) u.SetId(id) router := gin.Default() - router.Use(middlewares.AuthorizationMiddlewareV2()) + router.Use(middlewares.AuthorizationMiddleware()) router.POST("/users/:id/change-password", controllers.PostUserChangePassword) password := "newPassword" @@ -43,14 +43,14 @@ func TestGetUserMe_Success(t *testing.T) { SetupTestDB() defer CleanupTestDB() - modelSvc := service.NewModelService[models.UserV2]() - u := models.UserV2{} + modelSvc := service.NewModelService[models.User]() + u := models.User{} id, err := modelSvc.InsertOne(u) assert.Nil(t, err) u.SetId(id) router := gin.Default() - router.Use(middlewares.AuthorizationMiddlewareV2()) + router.Use(middlewares.AuthorizationMiddleware()) router.GET("/users/me", controllers.GetUserMe) req, _ := http.NewRequest(http.MethodGet, "/users/me", nil) @@ -67,14 +67,14 @@ func TestPutUserById_Success(t *testing.T) { SetupTestDB() defer CleanupTestDB() - modelSvc := service.NewModelService[models.UserV2]() - u := models.UserV2{} + modelSvc := service.NewModelService[models.User]() + u := models.User{} id, err := modelSvc.InsertOne(u) assert.Nil(t, err) u.SetId(id) router := gin.Default() - router.Use(middlewares.AuthorizationMiddlewareV2()) + router.Use(middlewares.AuthorizationMiddleware()) router.PUT("/users/me", controllers.PutUserById) reqBody := strings.NewReader(`{"id":"` + id.Hex() + `","username":"newUsername","email":"newEmail@test.com"}`) diff --git a/core/controllers/utils_context.go b/core/controllers/utils_context.go index 08f55255..506c7549 100644 --- a/core/controllers/utils_context.go +++ b/core/controllers/utils_context.go @@ -2,16 +2,16 @@ package controllers import ( "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/gin-gonic/gin" ) -func GetUserFromContext(c *gin.Context) (u *models.UserV2) { +func GetUserFromContext(c *gin.Context) (u *models.User) { value, ok := c.Get(constants.UserContextKey) if !ok { return nil } - u, ok = value.(*models.UserV2) + u, ok = value.(*models.User) if !ok { return nil } diff --git a/core/database/interfaces/database_service.go b/core/database/interfaces/database_service.go index 53e9866d..122cbb22 100644 --- a/core/database/interfaces/database_service.go +++ b/core/database/interfaces/database_service.go @@ -2,7 +2,7 @@ package interfaces import ( "github.com/crawlab-team/crawlab/core/database/entity" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "go.mongodb.org/mongo-driver/bson/primitive" ) @@ -23,5 +23,5 @@ type DatabaseService interface { UpdateRow(id primitive.ObjectID, databaseName, tableName string, filter map[string]interface{}, update map[string]interface{}) error DeleteRow(id primitive.ObjectID, databaseName, tableName string, filter map[string]interface{}) error Query(id primitive.ObjectID, databaseName, query string) (results *entity.DatabaseQueryResults, err error) - GetCurrentMetric(id primitive.ObjectID) (m *models.DatabaseMetricV2, err error) + GetCurrentMetric(id primitive.ObjectID) (m *models.DatabaseMetric, err error) } diff --git a/core/fs/service_v2.go b/core/fs/service.go similarity index 82% rename from core/fs/service_v2.go rename to core/fs/service.go index efdd9a0b..2103fa4f 100644 --- a/core/fs/service_v2.go +++ b/core/fs/service.go @@ -11,13 +11,13 @@ import ( "path/filepath" ) -type ServiceV2 struct { +type Service struct { // settings rootPath string skipNames []string } -func (svc *ServiceV2) List(path string) (files []interfaces.FsFileInfo, err error) { +func (svc *Service) List(path string) (files []interfaces.FsFileInfo, err error) { // Normalize the provided path normPath := filepath.Clean(path) if normPath == "." { @@ -78,11 +78,11 @@ func (svc *ServiceV2) List(path string) (files []interfaces.FsFileInfo, err erro return files, err } -func (svc *ServiceV2) GetFile(path string) (data []byte, err error) { +func (svc *Service) GetFile(path string) (data []byte, err error) { return os.ReadFile(filepath.Join(svc.rootPath, path)) } -func (svc *ServiceV2) GetFileInfo(path string) (file interfaces.FsFileInfo, err error) { +func (svc *Service) GetFileInfo(path string) (file interfaces.FsFileInfo, err error) { f, err := os.Stat(filepath.Join(svc.rootPath, path)) if err != nil { return nil, err @@ -100,7 +100,7 @@ func (svc *ServiceV2) GetFileInfo(path string) (file interfaces.FsFileInfo, err }, nil } -func (svc *ServiceV2) Save(path string, data []byte) (err error) { +func (svc *Service) Save(path string, data []byte) (err error) { // Create directories if not exist dir := filepath.Dir(filepath.Join(svc.rootPath, path)) if _, err := os.Stat(dir); os.IsNotExist(err) { @@ -113,22 +113,22 @@ func (svc *ServiceV2) Save(path string, data []byte) (err error) { return os.WriteFile(filepath.Join(svc.rootPath, path), data, 0644) } -func (svc *ServiceV2) CreateDir(path string) (err error) { +func (svc *Service) CreateDir(path string) (err error) { return os.MkdirAll(filepath.Join(svc.rootPath, path), 0755) } -func (svc *ServiceV2) Rename(path, newPath string) (err error) { +func (svc *Service) Rename(path, newPath string) (err error) { oldPath := filepath.Join(svc.rootPath, path) newFullPath := filepath.Join(svc.rootPath, newPath) return os.Rename(oldPath, newFullPath) } -func (svc *ServiceV2) Delete(path string) (err error) { +func (svc *Service) Delete(path string) (err error) { fullPath := filepath.Join(svc.rootPath, path) return os.RemoveAll(fullPath) } -func (svc *ServiceV2) Copy(path, newPath string) (err error) { +func (svc *Service) Copy(path, newPath string) (err error) { srcPath := filepath.Join(svc.rootPath, path) destPath := filepath.Join(svc.rootPath, newPath) @@ -161,7 +161,7 @@ func (svc *ServiceV2) Copy(path, newPath string) (err error) { } } -func (svc *ServiceV2) Export() (resultPath string, err error) { +func (svc *Service) Export() (resultPath string, err error) { zipFilePath := filepath.Join(os.TempDir(), uuid.New().String()+".zip") if err := utils.ZipDirectory(svc.rootPath, zipFilePath); err != nil { return "", trace.TraceError(err) @@ -170,8 +170,8 @@ func (svc *ServiceV2) Export() (resultPath string, err error) { return zipFilePath, nil } -func NewFsServiceV2(path string) (svc interfaces.FsService) { - return &ServiceV2{ +func NewFsService(path string) (svc interfaces.FsService) { + return &Service{ rootPath: path, skipNames: []string{".git"}, } diff --git a/core/fs/service_v2_test.go b/core/fs/service_test.go similarity index 90% rename from core/fs/service_v2_test.go rename to core/fs/service_test.go index f3006f5d..06f4624b 100644 --- a/core/fs/service_v2_test.go +++ b/core/fs/service_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestServiceV2_List(t *testing.T) { +func TestService_List(t *testing.T) { rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) @@ -30,7 +30,7 @@ func TestServiceV2_List(t *testing.T) { _ = os.WriteFile(filepath.Join(subDir, "file3.txt"), []byte("subdir file"), 0644) _ = os.Mkdir(filepath.Join(testDir, "empty"), 0755) // explicitly testing empty dir inclusion - svc := NewFsServiceV2(rootDir) + svc := NewFsService(rootDir) files, err := svc.List("dir") if err != nil { @@ -60,7 +60,7 @@ func TestServiceV2_List(t *testing.T) { } } -func TestServiceV2_GetFile(t *testing.T) { +func TestService_GetFile(t *testing.T) { rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) @@ -75,7 +75,7 @@ func TestServiceV2_GetFile(t *testing.T) { expectedContent := []byte("hello world") _ = os.WriteFile(filepath.Join(rootDir, "file.txt"), expectedContent, 0644) - svc := NewFsServiceV2(rootDir) + svc := NewFsService(rootDir) content, err := svc.GetFile("file.txt") if err != nil { @@ -84,7 +84,7 @@ func TestServiceV2_GetFile(t *testing.T) { assert.Equal(t, expectedContent, content) } -func TestServiceV2_Delete(t *testing.T) { +func TestService_Delete(t *testing.T) { rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) @@ -99,7 +99,7 @@ func TestServiceV2_Delete(t *testing.T) { filePath := filepath.Join(rootDir, "file.txt") _ = os.WriteFile(filePath, []byte("hello world"), 0644) - svc := NewFsServiceV2(rootDir) + svc := NewFsService(rootDir) // Delete the file err = svc.Delete("file.txt") @@ -112,7 +112,7 @@ func TestServiceV2_Delete(t *testing.T) { assert.True(t, os.IsNotExist(err)) } -func TestServiceV2_CreateDir(t *testing.T) { +func TestService_CreateDir(t *testing.T) { rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) @@ -124,7 +124,7 @@ func TestServiceV2_CreateDir(t *testing.T) { } }() - svc := NewFsServiceV2(rootDir) + svc := NewFsService(rootDir) // Create a new directory err = svc.CreateDir("newDir") @@ -137,7 +137,7 @@ func TestServiceV2_CreateDir(t *testing.T) { assert.NoError(t, err) } -func TestServiceV2_Save(t *testing.T) { +func TestService_Save(t *testing.T) { rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) @@ -149,7 +149,7 @@ func TestServiceV2_Save(t *testing.T) { } }() - svc := NewFsServiceV2(rootDir) + svc := NewFsService(rootDir) // Save a new file err = svc.Save("newFile.txt", []byte("Hello, world!")) @@ -163,7 +163,7 @@ func TestServiceV2_Save(t *testing.T) { assert.Equal(t, "Hello, world!", string(data)) } -func TestServiceV2_Rename(t *testing.T) { +func TestService_Rename(t *testing.T) { rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) @@ -175,7 +175,7 @@ func TestServiceV2_Rename(t *testing.T) { } }() - svc := NewFsServiceV2(rootDir) + svc := NewFsService(rootDir) // Create a file to rename _ = os.WriteFile(filepath.Join(rootDir, "oldName.txt"), []byte("Hello, world!"), 0644) @@ -191,7 +191,7 @@ func TestServiceV2_Rename(t *testing.T) { assert.NoError(t, err) } -func TestServiceV2_RenameDir(t *testing.T) { +func TestService_RenameDir(t *testing.T) { rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) @@ -203,7 +203,7 @@ func TestServiceV2_RenameDir(t *testing.T) { } }() - svc := NewFsServiceV2(rootDir) + svc := NewFsService(rootDir) // Create a directory to rename _ = os.Mkdir(filepath.Join(rootDir, "oldName"), 0755) @@ -219,7 +219,7 @@ func TestServiceV2_RenameDir(t *testing.T) { assert.NoError(t, err) } -func TestServiceV2_Copy(t *testing.T) { +func TestService_Copy(t *testing.T) { rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) @@ -231,7 +231,7 @@ func TestServiceV2_Copy(t *testing.T) { } }() - svc := NewFsServiceV2(rootDir) + svc := NewFsService(rootDir) // Create a file to copy _ = os.WriteFile(filepath.Join(rootDir, "source.txt"), []byte("Hello, world!"), 0644) @@ -248,7 +248,7 @@ func TestServiceV2_Copy(t *testing.T) { assert.Equal(t, "Hello, world!", string(data)) } -func TestServiceV2_CopyDir(t *testing.T) { +func TestService_CopyDir(t *testing.T) { rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) @@ -260,7 +260,7 @@ func TestServiceV2_CopyDir(t *testing.T) { } }() - svc := NewFsServiceV2(rootDir) + svc := NewFsService(rootDir) // Create a directory to copy _ = os.Mkdir(filepath.Join(rootDir, "sourceDir"), 0755) diff --git a/core/grpc/client/client.go b/core/grpc/client/client.go index 15e99cec..2fdbf894 100644 --- a/core/grpc/client/client.go +++ b/core/grpc/client/client.go @@ -173,12 +173,12 @@ func newGrpcClient() (c *GrpcClient) { return client } -var clientV2 *GrpcClient -var clientV2Once sync.Once +var _client *GrpcClient +var _clientOnce sync.Once func GetGrpcClient() *GrpcClient { - clientV2Once.Do(func() { - clientV2 = newGrpcClient() + _clientOnce.Do(func() { + _client = newGrpcClient() }) - return clientV2 + return _client } diff --git a/core/grpc/server/dependency_service_server.go b/core/grpc/server/dependency_service_server.go index c5d1843f..baf06797 100644 --- a/core/grpc/server/dependency_service_server.go +++ b/core/grpc/server/dependency_service_server.go @@ -4,7 +4,7 @@ import ( "context" "errors" "github.com/apex/log" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" mongo2 "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/grpc" @@ -40,12 +40,12 @@ func (svr DependencyServiceServer) Connect(req *grpc.DependencyServiceConnectReq } func (svr DependencyServiceServer) Sync(ctx context.Context, request *grpc.DependencyServiceSyncRequest) (response *grpc.Response, err error) { - n, err := service.NewModelService[models2.NodeV2]().GetOne(bson.M{"key": request.NodeKey}, nil) + n, err := service.NewModelService[models.Node]().GetOne(bson.M{"key": request.NodeKey}, nil) if err != nil { return nil, err } - depsDb, err := service.NewModelService[models2.DependencyV2]().GetMany(bson.M{ + depsDb, err := service.NewModelService[models.Dependency]().GetMany(bson.M{ "node_id": n.Id, "type": request.Lang, }, nil) @@ -56,15 +56,15 @@ func (svr DependencyServiceServer) Sync(ctx context.Context, request *grpc.Depen } } - depsDbMap := make(map[string]*models2.DependencyV2) + depsDbMap := make(map[string]*models.Dependency) for _, d := range depsDb { depsDbMap[d.Name] = &d } - var depsToInsert []models2.DependencyV2 - depsMap := make(map[string]*models2.DependencyV2) + var depsToInsert []models.Dependency + depsMap := make(map[string]*models.Dependency) for _, dep := range request.Dependencies { - d := models2.DependencyV2{ + d := models.Dependency{ Name: dep.Name, NodeId: n.Id, Type: request.Lang, @@ -90,7 +90,7 @@ func (svr DependencyServiceServer) Sync(ctx context.Context, request *grpc.Depen err = mongo2.RunTransaction(func(ctx mongo.SessionContext) (err error) { if len(depIdsToDelete) > 0 { - err = service.NewModelService[models2.DependencyV2]().DeleteMany(bson.M{ + err = service.NewModelService[models.Dependency]().DeleteMany(bson.M{ "_id": bson.M{"$in": depIdsToDelete}, }) if err != nil { @@ -101,7 +101,7 @@ func (svr DependencyServiceServer) Sync(ctx context.Context, request *grpc.Depen } if len(depsToInsert) > 0 { - _, err = service.NewModelService[models2.DependencyV2]().InsertMany(depsToInsert) + _, err = service.NewModelService[models.Dependency]().InsertMany(depsToInsert) if err != nil { log.Errorf("[DependencyServiceServer] insert dependencies in db error: %v", err) trace.PrintError(err) @@ -116,8 +116,8 @@ func (svr DependencyServiceServer) Sync(ctx context.Context, request *grpc.Depen } func (svr DependencyServiceServer) UpdateLogs(stream grpc.DependencyService_UpdateLogsServer) (err error) { - var n *models2.NodeV2 - var dep *models2.DependencyV2 + var n *models.Node + var dep *models.Dependency for { // receive message req, err := stream.Recv() @@ -131,7 +131,7 @@ func (svr DependencyServiceServer) UpdateLogs(stream grpc.DependencyService_Upda // if node is nil, get node if n == nil { - n, err = service.NewModelService[models2.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil) + n, err = service.NewModelService[models.Node]().GetOne(bson.M{"key": req.NodeKey}, nil) if err != nil { log.Errorf("[DependencyServiceServer] get node error: %v", err) return err @@ -140,7 +140,7 @@ func (svr DependencyServiceServer) UpdateLogs(stream grpc.DependencyService_Upda // if dependency is nil, get dependency if dep == nil { - dep, err = service.NewModelService[models2.DependencyV2]().GetOne(bson.M{ + dep, err = service.NewModelService[models.Dependency]().GetOne(bson.M{ "node_id": n.Id, "name": req.Name, "type": req.Lang, @@ -151,14 +151,14 @@ func (svr DependencyServiceServer) UpdateLogs(stream grpc.DependencyService_Upda return err } // create dependency if not found - dep = &models2.DependencyV2{ + dep = &models.Dependency{ NodeId: n.Id, Name: req.Name, Type: req.Lang, } dep.SetCreatedAt(time.Now()) dep.SetUpdatedAt(time.Now()) - dep.Id, err = service.NewModelService[models2.DependencyV2]().InsertOne(*dep) + dep.Id, err = service.NewModelService[models.Dependency]().InsertOne(*dep) if err != nil { log.Errorf("[DependencyServiceServer] insert dependency error: %v", err) return err @@ -179,7 +179,7 @@ func (svr DependencyServiceServer) GetStream(key string) (stream *grpc.Dependenc return stream, nil } -func NewDependencyServerV2() *DependencyServiceServer { +func newDependencyServer() *DependencyServiceServer { return &DependencyServiceServer{ mu: new(sync.Mutex), streams: make(map[string]*grpc.DependencyService_ConnectServer), @@ -187,11 +187,11 @@ func NewDependencyServerV2() *DependencyServiceServer { } var depSvc *DependencyServiceServer +var depSvcOnce sync.Once -func GetDependencyServerV2() *DependencyServiceServer { - if depSvc != nil { - return depSvc - } - depSvc = NewDependencyServerV2() +func GetDependencyServer() *DependencyServiceServer { + depSvcOnce.Do(func() { + depSvc = newDependencyServer() + }) return depSvc } diff --git a/core/grpc/server/metric_service_server.go b/core/grpc/server/metric_service_server.go index 95ef78e1..7ffb2c34 100644 --- a/core/grpc/server/metric_service_server.go +++ b/core/grpc/server/metric_service_server.go @@ -3,7 +3,7 @@ package server import ( "context" "github.com/apex/log" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/grpc" "go.mongodb.org/mongo-driver/bson" @@ -17,12 +17,12 @@ type MetricServiceServer struct { func (svr MetricServiceServer) Send(_ context.Context, req *grpc.MetricServiceSendRequest) (res *grpc.Response, err error) { log.Info("[MetricServiceServer] received metric from node: " + req.NodeKey) - n, err := service.NewModelService[models2.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil) + n, err := service.NewModelService[models.Node]().GetOne(bson.M{"key": req.NodeKey}, nil) if err != nil { log.Errorf("[MetricServiceServer] error getting node: %v", err) return HandleError(err) } - metric := models2.MetricV2{ + metric := models.Metric{ Type: req.Type, NodeId: n.Id, CpuUsagePercent: req.CpuUsagePercent, @@ -40,7 +40,7 @@ func (svr MetricServiceServer) Send(_ context.Context, req *grpc.MetricServiceSe NetworkBytesRecvRate: req.NetworkBytesRecvRate, } metric.CreatedAt = time.Unix(req.Timestamp, 0) - _, err = service.NewModelService[models2.MetricV2]().InsertOne(metric) + _, err = service.NewModelService[models.Metric]().InsertOne(metric) if err != nil { log.Errorf("[MetricServiceServer] error inserting metric: %v", err) return HandleError(err) @@ -48,19 +48,16 @@ func (svr MetricServiceServer) Send(_ context.Context, req *grpc.MetricServiceSe return HandleSuccess() } -func newMetricsServerV2() *MetricServiceServer { +func newMetricsServer() *MetricServiceServer { return &MetricServiceServer{} } -var metricsServerV2 *MetricServiceServer -var metricsServerV2Once = &sync.Once{} +var metricsServer *MetricServiceServer +var metricsServerOnce = &sync.Once{} -func GetMetricsServerV2() *MetricServiceServer { - if metricsServerV2 != nil { - return metricsServerV2 - } - metricsServerV2Once.Do(func() { - metricsServerV2 = newMetricsServerV2() +func GetMetricsServer() *MetricServiceServer { + metricsServerOnce.Do(func() { + metricsServer = newMetricsServer() }) - return metricsServerV2 + return metricsServer } diff --git a/core/grpc/server/model_base_service_server.go b/core/grpc/server/model_base_service_server.go index 46882984..0e5a699d 100644 --- a/core/grpc/server/model_base_service_server.go +++ b/core/grpc/server/model_base_service_server.go @@ -4,8 +4,9 @@ import ( "context" "encoding/json" "reflect" + "sync" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/grpc" @@ -16,7 +17,7 @@ import ( var ( typeNameColNameMap = make(map[string]string) typeOneNameModelMap = make(map[string]any) - typeOneInstances = models2.GetModelInstances() + typeOneInstances = models.GetModelInstances() ) func init() { @@ -285,6 +286,16 @@ func GetModelService[T any](typeName string) *service.ModelService[T] { return service.NewModelServiceWithColName[T](typeNameColNameMap[typeName]) } -func NewModelBaseServiceServer() *ModelBaseServiceServer { +var modelBaseServiceServer *ModelBaseServiceServer +var modelBaseServiceServerOnce = &sync.Once{} + +func newModelBaseServiceServer() *ModelBaseServiceServer { return &ModelBaseServiceServer{} } + +func GetModelBaseServiceServer() *ModelBaseServiceServer { + modelBaseServiceServerOnce.Do(func() { + modelBaseServiceServer = newModelBaseServiceServer() + }) + return modelBaseServiceServer +} diff --git a/core/grpc/server/node_service_server.go b/core/grpc/server/node_service_server.go index d7e8dc3a..281893eb 100644 --- a/core/grpc/server/node_service_server.go +++ b/core/grpc/server/node_service_server.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/notification" @@ -40,21 +40,21 @@ func (svr NodeServiceServer) Register(_ context.Context, req *grpc.NodeServiceRe } // find in db - var node *models.NodeV2 - node, err = service.NewModelService[models.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil) + var node *models.Node + node, err = service.NewModelService[models.Node]().GetOne(bson.M{"key": req.NodeKey}, nil) if err == nil { // register existing node.Status = constants.NodeStatusOnline node.Active = true node.ActiveAt = time.Now() - err = service.NewModelService[models.NodeV2]().ReplaceById(node.Id, *node) + err = service.NewModelService[models.Node]().ReplaceById(node.Id, *node) if err != nil { return HandleError(err) } log.Infof("[NodeServiceServer] updated worker[%s] in db. id: %s", req.NodeKey, node.Id.Hex()) } else if errors2.Is(err, mongo.ErrNoDocuments) { // register new - node = &models.NodeV2{ + node = &models.Node{ Key: req.NodeKey, Name: req.NodeName, Status: constants.NodeStatusOnline, @@ -65,7 +65,7 @@ func (svr NodeServiceServer) Register(_ context.Context, req *grpc.NodeServiceRe } node.SetCreated(primitive.NilObjectID) node.SetUpdated(primitive.NilObjectID) - node.Id, err = service.NewModelService[models.NodeV2]().InsertOne(*node) + node.Id, err = service.NewModelService[models.Node]().InsertOne(*node) if err != nil { return HandleError(err) } @@ -83,7 +83,7 @@ func (svr NodeServiceServer) Register(_ context.Context, req *grpc.NodeServiceRe // SendHeartbeat from worker to master func (svr NodeServiceServer) SendHeartbeat(_ context.Context, req *grpc.NodeServiceSendHeartbeatRequest) (res *grpc.Response, err error) { // find in db - node, err := service.NewModelService[models.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil) + node, err := service.NewModelService[models.Node]().GetOne(bson.M{"key": req.NodeKey}, nil) if err != nil { if errors2.Is(err, mongo.ErrNoDocuments) { return HandleError(errors.ErrorNodeNotExists) @@ -96,7 +96,7 @@ func (svr NodeServiceServer) SendHeartbeat(_ context.Context, req *grpc.NodeServ node.Status = constants.NodeStatusOnline node.Active = true node.ActiveAt = time.Now() - err = service.NewModelService[models.NodeV2]().ReplaceById(node.Id, *node) + err = service.NewModelService[models.Node]().ReplaceById(node.Id, *node) if err != nil { return HandleError(err) } @@ -105,7 +105,7 @@ func (svr NodeServiceServer) SendHeartbeat(_ context.Context, req *grpc.NodeServ // send notification if status changed if utils.IsPro() { if oldStatus != newStatus { - go notification.GetNotificationServiceV2().SendNodeNotification(node) + go notification.GetNotificationService().SendNodeNotification(node) } } @@ -116,7 +116,7 @@ func (svr NodeServiceServer) Subscribe(request *grpc.NodeServiceSubscribeRequest log.Infof("[NodeServiceServer] master received subscribe request from node[%s]", request.NodeKey) // find in db - node, err := service.NewModelService[models.NodeV2]().GetOne(bson.M{"key": request.NodeKey}, nil) + node, err := service.NewModelService[models.Node]().GetOne(bson.M{"key": request.NodeKey}, nil) if err != nil { log.Errorf("[NodeServiceServer] error getting node: %v", err) return err @@ -148,18 +148,19 @@ func (svr NodeServiceServer) GetSubscribeStream(nodeId primitive.ObjectID) (stre return stream, ok } -var nodeSvr *NodeServiceServer -var nodeSvrOnce = new(sync.Once) - -func NewNodeServiceServer() *NodeServiceServer { - if nodeSvr != nil { - return nodeSvr +func newNodeServiceServer() *NodeServiceServer { + return &NodeServiceServer{ + cfgSvc: nodeconfig.GetNodeConfigService(), + subs: make(map[primitive.ObjectID]grpc.NodeService_SubscribeServer), } +} + +var nodeSvr *NodeServiceServer +var nodeSvrOnce sync.Once + +func GetNodeServiceServer() *NodeServiceServer { nodeSvrOnce.Do(func() { - nodeSvr = &NodeServiceServer{ - subs: make(map[primitive.ObjectID]grpc.NodeService_SubscribeServer), - } - nodeSvr.cfgSvc = nodeconfig.GetNodeConfigService() + nodeSvr = newNodeServiceServer() }) return nodeSvr } diff --git a/core/grpc/server/server.go b/core/grpc/server/server.go index ef58fc3b..404116c7 100644 --- a/core/grpc/server/server.go +++ b/core/grpc/server/server.go @@ -48,11 +48,8 @@ func (svr *GrpcServer) SetConfigPath(path string) { svr.cfgPath = path } -func (svr *GrpcServer) Init() (err error) { - // register +func (svr *GrpcServer) Init() { svr.register() - - return nil } func (svr *GrpcServer) Start() (err error) { @@ -103,14 +100,12 @@ func (svr *GrpcServer) Stop() (err error) { return nil } -func (svr *GrpcServer) register() (err error) { +func (svr *GrpcServer) register() { grpc2.RegisterNodeServiceServer(svr.svr, *svr.NodeSvr) grpc2.RegisterModelBaseServiceServer(svr.svr, *svr.ModelBaseServiceSvr) grpc2.RegisterTaskServiceServer(svr.svr, *svr.TaskSvr) grpc2.RegisterDependencyServiceServer(svr.svr, *svr.DependencySvr) grpc2.RegisterMetricServiceServer(svr.svr, *svr.MetricSvr) - - return nil } func (svr *GrpcServer) recoveryHandlerFunc(p interface{}) (err error) { @@ -118,9 +113,9 @@ func (svr *GrpcServer) recoveryHandlerFunc(p interface{}) (err error) { return fmt.Errorf("recovered from panic: %v", p) } -func NewGrpcServer() (svr *GrpcServer, err error) { +func newGrpcServer() *GrpcServer { // server - svr = &GrpcServer{ + svr := &GrpcServer{ address: entity.NewAddress(&entity.AddressOptions{ Host: constants.DefaultGrpcServerHost, Port: constants.DefaultGrpcServerPort, @@ -128,23 +123,23 @@ func NewGrpcServer() (svr *GrpcServer, err error) { } if viper.GetString("grpc.server.address") != "" { - svr.address, err = entity.NewAddressFromString(viper.GetString("grpc.server.address")) + address, err := entity.NewAddressFromString(viper.GetString("grpc.server.address")) if err != nil { - log.Fatalf("[NewGrpcServer] failed to parse grpc server address: %v", err) + log.Fatalf("[GrpcServer] failed to parse grpc server address: %v", err) panic(err) } + svr.address = address } + // node config service svr.nodeCfgSvc = nodeconfig.GetNodeConfigService() - svr.NodeSvr = NewNodeServiceServer() - svr.ModelBaseServiceSvr = NewModelBaseServiceServer() - svr.TaskSvr, err = NewTaskServiceServer() - if err != nil { - return nil, err - } - svr.DependencySvr = GetDependencyServerV2() - svr.MetricSvr = GetMetricsServerV2() + // servers + svr.NodeSvr = GetNodeServiceServer() + svr.ModelBaseServiceSvr = GetModelBaseServiceServer() + svr.TaskSvr = GetTaskServiceServer() + svr.DependencySvr = GetDependencyServer() + svr.MetricSvr = GetMetricsServer() // recovery options recoveryOpts := []grpcrecovery.Option{ @@ -166,18 +161,15 @@ func NewGrpcServer() (svr *GrpcServer, err error) { // initialize svr.Init() - return svr, nil + return svr } var _server *GrpcServer var _serverOnce sync.Once -func GetGrpcServer() (svr *GrpcServer, err error) { +func GetGrpcServer() *GrpcServer { _serverOnce.Do(func() { - _server, err = NewGrpcServer() + _server = newGrpcServer() }) - if err != nil { - return nil, err - } - return _server, nil + return _server } diff --git a/core/grpc/server/task_service_server.go b/core/grpc/server/task_service_server.go index 6d587215..400bf51f 100644 --- a/core/grpc/server/task_service_server.go +++ b/core/grpc/server/task_service_server.go @@ -7,7 +7,7 @@ import ( "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/notification" @@ -110,7 +110,7 @@ func (svr TaskServiceServer) FetchTask(ctx context.Context, request *grpc.TaskSe if nodeKey == "" { return nil, errors.New("invalid node key") } - n, err := service.NewModelService[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + n, err := service.NewModelService[models.Node]().GetOne(bson.M{"key": nodeKey}, nil) if err != nil { return nil, trace.TraceError(err) } @@ -124,7 +124,7 @@ func (svr TaskServiceServer) FetchTask(ctx context.Context, request *grpc.TaskSe } if err := mongo.RunTransactionWithContext(ctx, func(sc mongo2.SessionContext) (err error) { // fetch task for the given node - t, err := service.NewModelService[models2.TaskV2]().GetOne(bson.M{ + t, err := service.NewModelService[models.Task]().GetOne(bson.M{ "node_id": n.Id, "status": constants.TaskStatusPending, }, opts) @@ -138,7 +138,7 @@ func (svr TaskServiceServer) FetchTask(ctx context.Context, request *grpc.TaskSe } // fetch task for any node - t, err = service.NewModelService[models2.TaskV2]().GetOne(bson.M{ + t, err = service.NewModelService[models.Task]().GetOne(bson.M{ "node_id": primitive.NilObjectID, "status": constants.TaskStatusPending, }, opts) @@ -177,7 +177,7 @@ func (svr TaskServiceServer) SendNotification(_ context.Context, request *grpc.T var args []any // task - task, err := service.NewModelService[models2.TaskV2]().GetById(taskId) + task, err := service.NewModelService[models.Task]().GetById(taskId) if err != nil { log.Errorf("task not found: %s", request.TaskId) return nil, trace.TraceError(err) @@ -185,7 +185,7 @@ func (svr TaskServiceServer) SendNotification(_ context.Context, request *grpc.T args = append(args, task) // task stat - taskStat, err := service.NewModelService[models2.TaskStatV2]().GetById(task.Id) + taskStat, err := service.NewModelService[models.TaskStat]().GetById(task.Id) if err != nil { log.Errorf("task stat not found for task: %s", request.TaskId) return nil, trace.TraceError(err) @@ -193,7 +193,7 @@ func (svr TaskServiceServer) SendNotification(_ context.Context, request *grpc.T args = append(args, taskStat) // spider - spider, err := service.NewModelService[models2.SpiderV2]().GetById(task.SpiderId) + spider, err := service.NewModelService[models.Spider]().GetById(task.SpiderId) if err != nil { log.Errorf("spider not found for task: %s", request.TaskId) return nil, trace.TraceError(err) @@ -201,16 +201,16 @@ func (svr TaskServiceServer) SendNotification(_ context.Context, request *grpc.T args = append(args, spider) // node - node, err := service.NewModelService[models2.NodeV2]().GetById(task.NodeId) + node, err := service.NewModelService[models.Node]().GetById(task.NodeId) if err != nil { return nil, trace.TraceError(err) } args = append(args, node) // schedule - var schedule *models2.ScheduleV2 + var schedule *models.Schedule if !task.ScheduleId.IsZero() { - schedule, err = service.NewModelService[models2.ScheduleV2]().GetById(task.ScheduleId) + schedule, err = service.NewModelService[models.Schedule]().GetById(task.ScheduleId) if err != nil { log.Errorf("schedule not found for task: %s", request.TaskId) return nil, trace.TraceError(err) @@ -219,7 +219,7 @@ func (svr TaskServiceServer) SendNotification(_ context.Context, request *grpc.T } // settings - settings, err := service.NewModelService[models2.NotificationSettingV2]().GetMany(bson.M{ + settings, err := service.NewModelService[models.NotificationSetting]().GetMany(bson.M{ "enabled": true, "trigger": bson.M{ "$regex": constants.NotificationTriggerPatternTask, @@ -230,7 +230,7 @@ func (svr TaskServiceServer) SendNotification(_ context.Context, request *grpc.T } // notification service - svc := notification.GetNotificationServiceV2() + svc := notification.GetNotificationService() for _, s := range settings { // compatible with old settings @@ -286,23 +286,25 @@ func (svr TaskServiceServer) handleInsertLogs(taskId primitive.ObjectID, msg *gr return svr.statsSvc.InsertLogs(taskId, logs...) } -func (svr TaskServiceServer) saveTask(t *models2.TaskV2) (err error) { +func (svr TaskServiceServer) saveTask(t *models.Task) (err error) { t.SetUpdated(t.CreatedBy) - return service.NewModelService[models2.TaskV2]().ReplaceById(t.Id, *t) + return service.NewModelService[models.Task]().ReplaceById(t.Id, *t) } -func NewTaskServiceServer() (res *TaskServiceServer, err error) { - // task server - svr := &TaskServiceServer{ - subs: make(map[primitive.ObjectID]grpc.TaskService_SubscribeServer), +func newTaskServiceServer() *TaskServiceServer { + return &TaskServiceServer{ + cfgSvc: nodeconfig.GetNodeConfigService(), + subs: make(map[primitive.ObjectID]grpc.TaskService_SubscribeServer), + statsSvc: stats.GetTaskStatsService(), } - - svr.cfgSvc = nodeconfig.GetNodeConfigService() - - svr.statsSvc, err = stats.GetTaskStatsServiceV2() - if err != nil { - return nil, err - } - - return svr, nil +} + +var _taskServiceServer *TaskServiceServer +var _taskServiceServerOnce sync.Once + +func GetTaskServiceServer() *TaskServiceServer { + _taskServiceServerOnce.Do(func() { + _taskServiceServer = newTaskServiceServer() + }) + return _taskServiceServer } diff --git a/core/interfaces/module.go b/core/interfaces/module.go index 4fc69303..44693a5b 100644 --- a/core/interfaces/module.go +++ b/core/interfaces/module.go @@ -3,7 +3,6 @@ package interfaces type ModuleId int type Module interface { - Init() error Start() Wait() Stop() diff --git a/core/middlewares/auth_v2.go b/core/middlewares/auth.go similarity index 78% rename from core/middlewares/auth_v2.go rename to core/middlewares/auth.go index fb9ccea3..a4e873e8 100644 --- a/core/middlewares/auth_v2.go +++ b/core/middlewares/auth.go @@ -3,7 +3,7 @@ package middlewares import ( "errors" "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/user" "github.com/crawlab-team/crawlab/core/utils" @@ -12,12 +12,12 @@ import ( "go.mongodb.org/mongo-driver/bson" ) -func AuthorizationMiddlewareV2() gin.HandlerFunc { - userSvc, _ := user.GetUserServiceV2() +func AuthorizationMiddleware() gin.HandlerFunc { + userSvc, _ := user.GetUserService() return func(c *gin.Context) { // disable auth for test if viper.GetBool("auth.disabled") { - u, err := service.NewModelService[models.UserV2]().GetOne(bson.M{"username": constants.DefaultAdminUsername}, nil) + u, err := service.NewModelService[models.User]().GetOne(bson.M{"username": constants.DefaultAdminUsername}, nil) if err != nil { utils.HandleErrorInternalServerError(c, err) return diff --git a/core/models/client/model_service_test.go b/core/models/client/model_service_test.go index 8c545295..ba0085aa 100644 --- a/core/models/client/model_service_test.go +++ b/core/models/client/model_service_test.go @@ -2,11 +2,10 @@ package client_test import ( "context" + "github.com/crawlab-team/crawlab/core/models/models" "testing" "time" - "github.com/crawlab-team/crawlab/core/models/models/v2" - "github.com/apex/log" "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/models/client" @@ -51,10 +50,10 @@ func TestModelService_GetById(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := models.TestModelV2{ + m := models.TestModel{ Name: "Test Name", } - modelSvc := service.NewModelService[models.TestModelV2]() + modelSvc := service.NewModelService[models.TestModel]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -64,7 +63,7 @@ func TestModelService_GetById(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelService[models.TestModelV2]() + clientSvc := client.NewModelService[models.TestModel]() res, err := clientSvc.GetById(m.Id) require.Nil(t, err) assert.Equal(t, res.Id, m.Id) @@ -79,10 +78,10 @@ func TestModelService_GetOne(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := models.TestModelV2{ + m := models.TestModel{ Name: "Test Name", } - modelSvc := service.NewModelService[models.TestModelV2]() + modelSvc := service.NewModelService[models.TestModel]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -92,7 +91,7 @@ func TestModelService_GetOne(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelService[models.TestModelV2]() + clientSvc := client.NewModelService[models.TestModel]() res, err := clientSvc.GetOne(bson.M{"name": m.Name}, nil) require.Nil(t, err) assert.Equal(t, res.Id, m.Id) @@ -107,10 +106,10 @@ func TestModelService_GetMany(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := models.TestModelV2{ + m := models.TestModel{ Name: "Test Name", } - modelSvc := service.NewModelService[models.TestModelV2]() + modelSvc := service.NewModelService[models.TestModel]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -120,7 +119,7 @@ func TestModelService_GetMany(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelService[models.TestModelV2]() + clientSvc := client.NewModelService[models.TestModel]() res, err := clientSvc.GetMany(bson.M{"name": m.Name}, nil) require.Nil(t, err) assert.Equal(t, len(res), 1) @@ -136,10 +135,10 @@ func TestModelService_DeleteById(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := models.TestModelV2{ + m := models.TestModel{ Name: "Test Name", } - modelSvc := service.NewModelService[models.TestModelV2]() + modelSvc := service.NewModelService[models.TestModel]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -149,7 +148,7 @@ func TestModelService_DeleteById(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelService[models.TestModelV2]() + clientSvc := client.NewModelService[models.TestModel]() err = clientSvc.DeleteById(m.Id) require.Nil(t, err) @@ -166,10 +165,10 @@ func TestModelService_DeleteOne(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := models.TestModelV2{ + m := models.TestModel{ Name: "Test Name", } - modelSvc := service.NewModelService[models.TestModelV2]() + modelSvc := service.NewModelService[models.TestModel]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -179,7 +178,7 @@ func TestModelService_DeleteOne(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelService[models.TestModelV2]() + clientSvc := client.NewModelService[models.TestModel]() err = clientSvc.DeleteOne(bson.M{"name": m.Name}) require.Nil(t, err) @@ -196,10 +195,10 @@ func TestModelService_DeleteMany(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := models.TestModelV2{ + m := models.TestModel{ Name: "Test Name", } - modelSvc := service.NewModelService[models.TestModelV2]() + modelSvc := service.NewModelService[models.TestModel]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -209,7 +208,7 @@ func TestModelService_DeleteMany(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelService[models.TestModelV2]() + clientSvc := client.NewModelService[models.TestModel]() err = clientSvc.DeleteMany(bson.M{"name": m.Name}) require.Nil(t, err) @@ -226,10 +225,10 @@ func TestModelService_UpdateById(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := models.TestModelV2{ + m := models.TestModel{ Name: "Test Name", } - modelSvc := service.NewModelService[models.TestModelV2]() + modelSvc := service.NewModelService[models.TestModel]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -239,7 +238,7 @@ func TestModelService_UpdateById(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelService[models.TestModelV2]() + clientSvc := client.NewModelService[models.TestModel]() err = clientSvc.UpdateById(m.Id, bson.M{"$set": bson.M{"name": "New Name"}}) require.Nil(t, err) @@ -256,10 +255,10 @@ func TestModelService_UpdateOne(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := models.TestModelV2{ + m := models.TestModel{ Name: "Test Name", } - modelSvc := service.NewModelService[models.TestModelV2]() + modelSvc := service.NewModelService[models.TestModel]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -269,7 +268,7 @@ func TestModelService_UpdateOne(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelService[models.TestModelV2]() + clientSvc := client.NewModelService[models.TestModel]() err = clientSvc.UpdateOne(bson.M{"name": m.Name}, bson.M{"$set": bson.M{"name": "New Name"}}) require.Nil(t, err) @@ -286,13 +285,13 @@ func TestModelService_UpdateMany(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m1 := models.TestModelV2{ + m1 := models.TestModel{ Name: "Test Name", } - m2 := models.TestModelV2{ + m2 := models.TestModel{ Name: "Test Name", } - modelSvc := service.NewModelService[models.TestModelV2]() + modelSvc := service.NewModelService[models.TestModel]() _, err = modelSvc.InsertOne(m1) require.Nil(t, err) _, err = modelSvc.InsertOne(m2) @@ -303,7 +302,7 @@ func TestModelService_UpdateMany(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelService[models.TestModelV2]() + clientSvc := client.NewModelService[models.TestModel]() err = clientSvc.UpdateMany(bson.M{"name": "Test Name"}, bson.M{"$set": bson.M{"name": "New Name"}}) require.Nil(t, err) @@ -320,10 +319,10 @@ func TestModelService_ReplaceById(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := models.TestModelV2{ + m := models.TestModel{ Name: "Test Name", } - modelSvc := service.NewModelService[models.TestModelV2]() + modelSvc := service.NewModelService[models.TestModel]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -333,7 +332,7 @@ func TestModelService_ReplaceById(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelService[models.TestModelV2]() + clientSvc := client.NewModelService[models.TestModel]() m.Name = "New Name" err = clientSvc.ReplaceById(m.Id, m) require.Nil(t, err) @@ -351,10 +350,10 @@ func TestModelService_ReplaceOne(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - m := models.TestModelV2{ + m := models.TestModel{ Name: "Test Name", } - modelSvc := service.NewModelService[models.TestModelV2]() + modelSvc := service.NewModelService[models.TestModel]() id, err := modelSvc.InsertOne(m) require.Nil(t, err) m.SetId(id) @@ -364,7 +363,7 @@ func TestModelService_ReplaceOne(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelService[models.TestModelV2]() + clientSvc := client.NewModelService[models.TestModel]() m.Name = "New Name" err = clientSvc.ReplaceOne(bson.M{"name": "Test Name"}, m) require.Nil(t, err) @@ -386,8 +385,8 @@ func TestModelService_InsertOne(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelService[models.TestModelV2]() - m := models.TestModelV2{ + clientSvc := client.NewModelService[models.TestModel]() + m := models.TestModel{ Name: "Test Name", } id, err := clientSvc.InsertOne(m) @@ -410,8 +409,8 @@ func TestModelService_InsertMany(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelService[models.TestModelV2]() - testModels := []models.TestModelV2{ + clientSvc := client.NewModelService[models.TestModel]() + testModels := []models.TestModel{ {Name: "Test Name 1"}, {Name: "Test Name 2"}, } @@ -433,9 +432,9 @@ func TestModelService_Count(t *testing.T) { go startSvr(svr) defer stopSvr(svr) - modelSvc := service.NewModelService[models.TestModelV2]() + modelSvc := service.NewModelService[models.TestModel]() for i := 0; i < 5; i++ { - _, err = modelSvc.InsertOne(models.TestModelV2{ + _, err = modelSvc.InsertOne(models.TestModel{ Name: "Test Name", }) require.Nil(t, err) @@ -446,7 +445,7 @@ func TestModelService_Count(t *testing.T) { require.Nil(t, err) c.Connect() - clientSvc := client.NewModelService[models.TestModelV2]() + clientSvc := client.NewModelService[models.TestModel]() count, err := clientSvc.Count(bson.M{}) require.Nil(t, err) diff --git a/core/models/common/init_index.go b/core/models/common/init_index.go index c3872a42..46390f98 100644 --- a/core/models/common/init_index.go +++ b/core/models/common/init_index.go @@ -1,7 +1,7 @@ package common import ( - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "go.mongodb.org/mongo-driver/bson" @@ -11,7 +11,7 @@ import ( func InitIndexes() { // nodes - RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NodeV2{})), []mongo2.IndexModel{ + RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models.Node{})), []mongo2.IndexModel{ {Keys: bson.M{"key": 1}}, // key {Keys: bson.M{"name": 1}}, // name {Keys: bson.M{"is_master": 1}}, // is_master @@ -21,12 +21,12 @@ func InitIndexes() { }) // projects - RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ProjectV2{})), []mongo2.IndexModel{ + RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models.Project{})), []mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, }) // spiders - RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SpiderV2{})), []mongo2.IndexModel{ + RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models.Spider{})), []mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, {Keys: bson.M{"type": 1}}, {Keys: bson.M{"col_id": 1}}, @@ -34,7 +34,7 @@ func InitIndexes() { }) // tasks - RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskV2{})), []mongo2.IndexModel{ + RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models.Task{})), []mongo2.IndexModel{ {Keys: bson.M{"spider_id": 1}}, {Keys: bson.M{"status": 1}}, {Keys: bson.M{"node_id": 1}}, @@ -49,36 +49,36 @@ func InitIndexes() { }) // task stats - RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskStatV2{})), []mongo2.IndexModel{ + RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TaskStat{})), []mongo2.IndexModel{ {Keys: bson.M{"created_ts": -1}, Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24 * 30)}, }) // schedules - RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ScheduleV2{})), []mongo2.IndexModel{ + RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models.Schedule{})), []mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, {Keys: bson.M{"spider_id": 1}}, {Keys: bson.M{"enabled": 1}}, }) // users - RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserV2{})), []mongo2.IndexModel{ + RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models.User{})), []mongo2.IndexModel{ {Keys: bson.M{"username": 1}}, {Keys: bson.M{"role": 1}}, {Keys: bson.M{"email": 1}}, }) // settings - RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SettingV2{})), []mongo2.IndexModel{ + RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models.Setting{})), []mongo2.IndexModel{ {Keys: bson.M{"key": 1}, Options: options.Index().SetUnique(true)}, }) // tokens - RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TokenV2{})), []mongo2.IndexModel{ + RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models.Token{})), []mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, }) // data collections - RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataCollectionV2{})), []mongo2.IndexModel{ + RecreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DataCollection{})), []mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, }) } diff --git a/core/models/models/v2/base_v2.go b/core/models/models/base.go similarity index 52% rename from core/models/models/v2/base_v2.go rename to core/models/models/base.go index f98a1206..ccae347c 100644 --- a/core/models/models/v2/base_v2.go +++ b/core/models/models/base.go @@ -7,7 +7,7 @@ import ( "go.mongodb.org/mongo-driver/bson/primitive" ) -type BaseModelV2[T any] struct { +type BaseModel[T any] struct { Id primitive.ObjectID `json:"_id" bson:"_id"` CreatedAt time.Time `json:"created_ts,omitempty" bson:"created_ts,omitempty"` CreatedBy primitive.ObjectID `json:"created_by,omitempty" bson:"created_by,omitempty"` @@ -15,102 +15,91 @@ type BaseModelV2[T any] struct { UpdatedBy primitive.ObjectID `json:"updated_by,omitempty" bson:"updated_by,omitempty"` } -func (m *BaseModelV2[T]) GetId() primitive.ObjectID { +func (m *BaseModel[T]) GetId() primitive.ObjectID { return m.Id } -func (m *BaseModelV2[T]) SetId(id primitive.ObjectID) { +func (m *BaseModel[T]) SetId(id primitive.ObjectID) { m.Id = id } -func (m *BaseModelV2[T]) GetCreatedAt() time.Time { +func (m *BaseModel[T]) GetCreatedAt() time.Time { return m.CreatedAt } -func (m *BaseModelV2[T]) SetCreatedAt(t time.Time) { +func (m *BaseModel[T]) SetCreatedAt(t time.Time) { m.CreatedAt = t } -func (m *BaseModelV2[T]) GetCreatedBy() primitive.ObjectID { +func (m *BaseModel[T]) GetCreatedBy() primitive.ObjectID { return m.CreatedBy } -func (m *BaseModelV2[T]) SetCreatedBy(id primitive.ObjectID) { +func (m *BaseModel[T]) SetCreatedBy(id primitive.ObjectID) { m.CreatedBy = id } -func (m *BaseModelV2[T]) GetUpdatedAt() time.Time { +func (m *BaseModel[T]) GetUpdatedAt() time.Time { return m.UpdatedAt } -func (m *BaseModelV2[T]) SetUpdatedAt(t time.Time) { +func (m *BaseModel[T]) SetUpdatedAt(t time.Time) { m.UpdatedAt = t } -func (m *BaseModelV2[T]) GetUpdatedBy() primitive.ObjectID { +func (m *BaseModel[T]) GetUpdatedBy() primitive.ObjectID { return m.UpdatedBy } -func (m *BaseModelV2[T]) SetUpdatedBy(id primitive.ObjectID) { +func (m *BaseModel[T]) SetUpdatedBy(id primitive.ObjectID) { m.UpdatedBy = id } -func (m *BaseModelV2[T]) SetCreated(id primitive.ObjectID) { +func (m *BaseModel[T]) SetCreated(id primitive.ObjectID) { m.SetCreatedAt(time.Now()) m.SetCreatedBy(id) } -func (m *BaseModelV2[T]) SetUpdated(id primitive.ObjectID) { +func (m *BaseModel[T]) SetUpdated(id primitive.ObjectID) { m.SetUpdatedAt(time.Now()) m.SetUpdatedBy(id) } func GetModelInstances() []any { return []any{ - *new(TestModelV2), - *new(DataCollectionV2), - *new(DatabaseV2), - *new(DatabaseMetricV2), - *new(DependencyV2), - *new(DependencyLogV2), - *new(DependencySettingV2), - *new(DependencyTaskV2), - *new(DependencyRepoV2), - *new(EnvironmentV2), - *new(GitV2), - *new(MetricV2), - *new(NodeV2), - *new(NotificationAlertV2), - *new(NotificationChannelV2), - *new(NotificationRequestV2), - *new(NotificationSettingV2), - *new(PermissionV2), - *new(ProjectV2), - *new(RolePermissionV2), - *new(RoleV2), - *new(ScheduleV2), - *new(SettingV2), - *new(SpiderV2), - *new(SpiderStatV2), - *new(TaskStatV2), - *new(TaskV2), - *new(TokenV2), - *new(UserRoleV2), - *new(UserV2), + *new(TestModel), + *new(DataCollection), + *new(Database), + *new(DatabaseMetric), + *new(Dependency), + *new(DependencyLog), + *new(DependencySetting), + *new(DependencyTask), + *new(DependencyRepo), + *new(Environment), + *new(Git), + *new(Metric), + *new(Node), + *new(NotificationAlert), + *new(NotificationChannel), + *new(NotificationRequest), + *new(NotificationSetting), + *new(Permission), + *new(Project), + *new(RolePermission), + *new(Role), + *new(Schedule), + *new(Setting), + *new(Spider), + *new(SpiderStat), + *new(TaskStat), + *new(Task), + *new(Token), + *new(UserRole), + *new(User), } } -func GetSystemModelColNames() []string { - colNames := make([]string, 0) - for _, instance := range GetModelInstances() { - colName := GetCollectionNameByInstance(instance) - if colName != "" { - colNames = append(colNames, colName) - } - } - return colNames -} - func GetCollectionNameByInstance(v any) string { t := reflect.TypeOf(v) field := t.Field(0) @@ -127,3 +116,14 @@ func GetSystemModelColNamesMap() map[string]bool { } return colNamesMap } + +func GetSystemModelColNames() []string { + colNames := make([]string, 0) + for _, instance := range GetModelInstances() { + colName := GetCollectionNameByInstance(instance) + if colName != "" { + colNames = append(colNames, colName) + } + } + return colNames +} diff --git a/core/models/models/data_collection.go b/core/models/models/data_collection.go new file mode 100644 index 00000000..587b5db9 --- /dev/null +++ b/core/models/models/data_collection.go @@ -0,0 +1,17 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/entity" +) + +type DataCollection struct { + any `collection:"data_collections"` + BaseModel[DataCollection] `bson:",inline"` + Name string `json:"name" bson:"name"` + Fields []entity.DataField `json:"fields" bson:"fields"` + Dedup struct { + Enabled bool `json:"enabled" bson:"enabled"` + Keys []string `json:"keys" bson:"keys"` + Type string `json:"type" bson:"type"` + } `json:"dedup" bson:"dedup"` +} diff --git a/core/models/models/v2/database_v2.go b/core/models/models/database.go similarity index 52% rename from core/models/models/v2/database_v2.go rename to core/models/models/database.go index e09a6fb3..c510b0f7 100644 --- a/core/models/models/v2/database_v2.go +++ b/core/models/models/database.go @@ -4,24 +4,24 @@ import ( "time" ) -type DatabaseV2 struct { - any `collection:"databases"` - BaseModelV2[DatabaseV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - DataSource string `json:"data_source" bson:"data_source"` - Host string `json:"host" bson:"host"` - Port int `json:"port" bson:"port"` - URI string `json:"uri,omitempty" bson:"uri,omitempty"` - Database string `json:"database,omitempty" bson:"database,omitempty"` - Username string `json:"username,omitempty" bson:"username,omitempty"` - Password string `json:"password,omitempty" bson:"-"` - EncryptedPassword string `json:"-,omitempty" bson:"encrypted_password,omitempty"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Active bool `json:"active" bson:"active"` - ActiveAt time.Time `json:"active_ts" bson:"active_ts"` - IsDefault bool `json:"is_default" bson:"-"` +type Database struct { + any `collection:"databases"` + BaseModel[Database] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + DataSource string `json:"data_source" bson:"data_source"` + Host string `json:"host" bson:"host"` + Port int `json:"port" bson:"port"` + URI string `json:"uri,omitempty" bson:"uri,omitempty"` + Database string `json:"database,omitempty" bson:"database,omitempty"` + Username string `json:"username,omitempty" bson:"username,omitempty"` + Password string `json:"password,omitempty" bson:"-"` + EncryptedPassword string `json:"-,omitempty" bson:"encrypted_password,omitempty"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Active bool `json:"active" bson:"active"` + ActiveAt time.Time `json:"active_ts" bson:"active_ts"` + IsDefault bool `json:"is_default" bson:"-"` MongoParams *struct { AuthSource string `json:"auth_source,omitempty" bson:"auth_source,omitempty"` diff --git a/core/models/models/database_metric.go b/core/models/models/database_metric.go new file mode 100644 index 00000000..195615a0 --- /dev/null +++ b/core/models/models/database_metric.go @@ -0,0 +1,24 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type DatabaseMetric struct { + any `collection:"database_metrics"` + BaseModel[DatabaseMetric] `bson:",inline"` + DatabaseId primitive.ObjectID `json:"database_id" bson:"database_id"` + CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` + TotalMemory uint64 `json:"total_memory" bson:"total_memory"` + AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` + UsedMemory uint64 `json:"used_memory" bson:"used_memory"` + UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` + TotalDisk uint64 `json:"total_disk" bson:"total_disk"` + AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` + UsedDisk uint64 `json:"used_disk" bson:"used_disk"` + UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` + Connections int `json:"connections" bson:"connections"` + QueryPerSecond float64 `json:"query_per_second" bson:"query_per_second"` + TotalQuery uint64 `json:"total_query,omitempty" bson:"total_query,omitempty"` + CacheHitRatio float64 `json:"cache_hit_ratio" bson:"cache_hit_ratio"` + ReplicationLag float64 `json:"replication_lag" bson:"replication_lag"` + LockWaitTime float64 `json:"lock_wait_time" bson:"lock_wait_time"` +} diff --git a/core/models/models/dependency.go b/core/models/models/dependency.go new file mode 100644 index 00000000..a6e7d4f7 --- /dev/null +++ b/core/models/models/dependency.go @@ -0,0 +1,20 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Dependency struct { + any `collection:"dependencies"` + BaseModel[Dependency] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Type string `json:"type" bson:"type"` + Version string `json:"version" bson:"version"` + Status string `json:"status" bson:"status"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + Logs []string `json:"logs,omitempty" bson:"logs,omitempty"` + NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-"` + Versions []string `json:"versions,omitempty" bson:"-"` +} diff --git a/core/models/models/dependency_log.go b/core/models/models/dependency_log.go new file mode 100644 index 00000000..6964c80c --- /dev/null +++ b/core/models/models/dependency_log.go @@ -0,0 +1,10 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type DependencyLog struct { + any `collection:"dependency_logs"` + BaseModel[DependencyLog] `bson:",inline"` + TaskId primitive.ObjectID `json:"task_id" bson:"task_id"` + Content string `json:"content" bson:"content"` +} diff --git a/core/models/models/dependency_repo.go b/core/models/models/dependency_repo.go new file mode 100644 index 00000000..d205dce0 --- /dev/null +++ b/core/models/models/dependency_repo.go @@ -0,0 +1,10 @@ +package models + +type DependencyRepo struct { + any `collection:"dependency_repos"` + BaseModel[DependencyRepo] `bson:",inline"` + Name string `json:"name" bson:"name"` + Type string `json:"type" bson:"type"` + LatestVersion string `json:"latest_version" bson:"latest_version"` + AllVersions []string `json:"all_versions" bson:"all_versions"` +} diff --git a/core/models/models/dependency_setting.go b/core/models/models/dependency_setting.go new file mode 100644 index 00000000..6858f0f7 --- /dev/null +++ b/core/models/models/dependency_setting.go @@ -0,0 +1,17 @@ +package models + +import ( + "time" +) + +type DependencySetting struct { + any `collection:"dependency_settings"` + BaseModel[DependencySetting] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + Cmd string `json:"cmd" bson:"cmd"` + Proxy string `json:"proxy" bson:"proxy"` + LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"` +} diff --git a/core/models/models/dependency_task.go b/core/models/models/dependency_task.go new file mode 100644 index 00000000..15466d91 --- /dev/null +++ b/core/models/models/dependency_task.go @@ -0,0 +1,15 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type DependencyTask struct { + any `collection:"dependency_tasks"` + BaseModel[DependencyTask] `bson:",inline"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` + Type string `json:"type" bson:"type"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Action string `json:"action" bson:"action"` + DepNames []string `json:"dep_names" bson:"dep_names"` +} diff --git a/core/models/models/environment.go b/core/models/models/environment.go new file mode 100644 index 00000000..889d412c --- /dev/null +++ b/core/models/models/environment.go @@ -0,0 +1,8 @@ +package models + +type Environment struct { + any `collection:"environments"` + BaseModel[Environment] `bson:",inline"` + Key string `json:"key" bson:"key"` + Value string `json:"value" bson:"value"` +} diff --git a/core/models/models/git.go b/core/models/models/git.go new file mode 100644 index 00000000..45a3365f --- /dev/null +++ b/core/models/models/git.go @@ -0,0 +1,26 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/vcs" + "time" +) + +type Git struct { + any `collection:"gits"` + BaseModel[Git] `bson:",inline"` + Url string `json:"url" bson:"url"` + Name string `json:"name" bson:"name"` + AuthType string `json:"auth_type" bson:"auth_type"` + Username string `json:"username" bson:"username"` + Password string `json:"password" bson:"password"` + CurrentBranch string `json:"current_branch" bson:"current_branch"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Spiders []Spider `json:"spiders,omitempty" bson:"-"` + Refs []vcs.GitRef `json:"refs" bson:"refs"` + RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"` + CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"` + + // settings + AutoPull bool `json:"auto_pull" bson:"auto_pull"` +} diff --git a/core/models/models/metric.go b/core/models/models/metric.go new file mode 100644 index 00000000..8f72eb45 --- /dev/null +++ b/core/models/models/metric.go @@ -0,0 +1,25 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Metric struct { + any `collection:"metrics"` + BaseModel[Metric] `bson:",inline"` + Type string `json:"type" bson:"type"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` + TotalMemory uint64 `json:"total_memory" bson:"total_memory"` + AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` + UsedMemory uint64 `json:"used_memory" bson:"used_memory"` + UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` + TotalDisk uint64 `json:"total_disk" bson:"total_disk"` + AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` + UsedDisk uint64 `json:"used_disk" bson:"used_disk"` + UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` + DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"` + DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"` + NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"` + NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"` +} diff --git a/core/models/models/node.go b/core/models/models/node.go new file mode 100644 index 00000000..1b846025 --- /dev/null +++ b/core/models/models/node.go @@ -0,0 +1,23 @@ +package models + +import ( + "time" +) + +type Node struct { + any `collection:"nodes"` + BaseModel[Node] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Ip string `json:"ip" bson:"ip"` + Mac string `json:"mac" bson:"mac"` + Hostname string `json:"hostname" bson:"hostname"` + Description string `json:"description" bson:"description"` + IsMaster bool `json:"is_master" bson:"is_master"` + Status string `json:"status" bson:"status"` + Enabled bool `json:"enabled" bson:"enabled"` + Active bool `json:"active" bson:"active"` + ActiveAt time.Time `json:"active_at" bson:"active_ts"` + AvailableRunners int `json:"available_runners" bson:"available_runners"` + MaxRunners int `json:"max_runners" bson:"max_runners"` +} diff --git a/core/models/models/notification_alert.go b/core/models/models/notification_alert.go new file mode 100644 index 00000000..65004964 --- /dev/null +++ b/core/models/models/notification_alert.go @@ -0,0 +1,19 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type NotificationAlert struct { + any `collection:"notification_alerts"` + BaseModel[NotificationAlert] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + HasMetricTarget bool `json:"has_metric_target" bson:"has_metric_target"` + MetricTargetId primitive.ObjectID `json:"metric_target_id,omitempty" bson:"metric_target_id,omitempty"` + MetricName string `json:"metric_name" bson:"metric_name"` + Operator string `json:"operator" bson:"operator"` + LastingSeconds int `json:"lasting_seconds" bson:"lasting_seconds"` + TargetValue float32 `json:"target_value" bson:"target_value"` + Level string `json:"level" bson:"level"` + TemplateKey string `json:"template_key,omitempty" bson:"template_key,omitempty"` +} diff --git a/core/models/models/notification_channel.go b/core/models/models/notification_channel.go new file mode 100644 index 00000000..d8ad1cbc --- /dev/null +++ b/core/models/models/notification_channel.go @@ -0,0 +1,18 @@ +package models + +type NotificationChannel struct { + any `collection:"notification_channels"` + BaseModel[NotificationChannel] `bson:",inline"` + Type string `json:"type" bson:"type"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Provider string `json:"provider" bson:"provider"` + SMTPServer string `json:"smtp_server,omitempty" bson:"smtp_server,omitempty"` + SMTPPort int `json:"smtp_port,omitempty" bson:"smtp_port,omitempty"` + SMTPUsername string `json:"smtp_username,omitempty" bson:"smtp_username,omitempty"` + SMTPPassword string `json:"smtp_password,omitempty" bson:"smtp_password,omitempty"` + WebhookUrl string `json:"webhook_url,omitempty" bson:"webhook_url,omitempty"` + TelegramBotToken string `json:"telegram_bot_token,omitempty" bson:"telegram_bot_token,omitempty"` + TelegramChatId string `json:"telegram_chat_id,omitempty" bson:"telegram_chat_id,omitempty"` + GoogleOAuth2Json string `json:"google_oauth2_json,omitempty" bson:"google_oauth2_json,omitempty"` +} diff --git a/core/models/models/notification_request.go b/core/models/models/notification_request.go new file mode 100644 index 00000000..93adf0c0 --- /dev/null +++ b/core/models/models/notification_request.go @@ -0,0 +1,21 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type NotificationRequest struct { + any `collection:"notification_requests"` + BaseModel[NotificationRequest] `bson:",inline"` + Status string `json:"status" bson:"status"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + Title string `json:"title" bson:"title"` + Content string `json:"content" bson:"content"` + SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` + SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"` + MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty"` + MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` + MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` + SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` + ChannelId primitive.ObjectID `json:"channel_id" bson:"channel_id"` + Setting *NotificationSetting `json:"setting,omitempty" bson:"-"` + Channel *NotificationChannel `json:"channel,omitempty" bson:"-"` +} diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/notification_setting.go similarity index 72% rename from core/models/models/v2/notification_setting_v2.go rename to core/models/models/notification_setting.go index 7a932b3b..80826bec 100644 --- a/core/models/models/v2/notification_setting_v2.go +++ b/core/models/models/notification_setting.go @@ -2,12 +2,12 @@ package models import "go.mongodb.org/mongo-driver/bson/primitive" -type NotificationSettingV2 struct { - any `collection:"notification_settings"` - BaseModelV2[NotificationSettingV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` +type NotificationSetting struct { + any `collection:"notification_settings"` + BaseModel[NotificationSetting] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` Title string `json:"title,omitempty" bson:"title,omitempty"` Template string `json:"template" bson:"template"` @@ -27,8 +27,8 @@ type NotificationSettingV2 struct { MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` - ChannelIds []primitive.ObjectID `json:"channel_ids,omitempty" bson:"channel_ids,omitempty"` - Channels []NotificationChannelV2 `json:"channels,omitempty" bson:"-"` + ChannelIds []primitive.ObjectID `json:"channel_ids,omitempty" bson:"channel_ids,omitempty"` + Channels []NotificationChannel `json:"channels,omitempty" bson:"-"` AlertId primitive.ObjectID `json:"alert_id,omitempty" bson:"alert_id,omitempty"` } diff --git a/core/models/models/permission.go b/core/models/models/permission.go new file mode 100644 index 00000000..05438166 --- /dev/null +++ b/core/models/models/permission.go @@ -0,0 +1,13 @@ +package models + +type Permission struct { + any `collection:"permissions"` + BaseModel[Permission] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Type string `json:"type" bson:"type"` + Target []string `json:"target" bson:"target"` + Allow []string `json:"allow" bson:"allow"` + Deny []string `json:"deny" bson:"deny"` +} diff --git a/core/models/models/project.go b/core/models/models/project.go new file mode 100644 index 00000000..e487ef7b --- /dev/null +++ b/core/models/models/project.go @@ -0,0 +1,9 @@ +package models + +type Project struct { + any `collection:"projects"` + BaseModel[Project] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Spiders int `json:"spiders" bson:"-"` +} diff --git a/core/models/models/role.go b/core/models/models/role.go new file mode 100644 index 00000000..068c01e2 --- /dev/null +++ b/core/models/models/role.go @@ -0,0 +1,9 @@ +package models + +type Role struct { + any `collection:"roles"` + BaseModel[Role] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` +} diff --git a/core/models/models/role_permission.go b/core/models/models/role_permission.go new file mode 100644 index 00000000..bab01418 --- /dev/null +++ b/core/models/models/role_permission.go @@ -0,0 +1,12 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type RolePermission struct { + any `collection:"role_permissions"` + BaseModel[RolePermission] `bson:",inline"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"` +} diff --git a/core/models/models/schedule.go b/core/models/models/schedule.go new file mode 100644 index 00000000..22df35da --- /dev/null +++ b/core/models/models/schedule.go @@ -0,0 +1,22 @@ +package models + +import ( + "github.com/robfig/cron/v3" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Schedule struct { + any `collection:"schedules"` + BaseModel[Schedule] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` + Cron string `json:"cron" bson:"cron"` + EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` + Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` + Mode string `json:"mode" bson:"mode"` + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` + Priority int `json:"priority" bson:"priority"` + Enabled bool `json:"enabled" bson:"enabled"` +} diff --git a/core/models/models/setting.go b/core/models/models/setting.go new file mode 100644 index 00000000..6f61f1c3 --- /dev/null +++ b/core/models/models/setting.go @@ -0,0 +1,12 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson" +) + +type Setting struct { + any `collection:"settings"` + BaseModel[Setting] `bson:",inline"` + Key string `json:"key" bson:"key"` + Value bson.M `json:"value" bson:"value"` +} diff --git a/core/models/models/spider.go b/core/models/models/spider.go new file mode 100644 index 00000000..c1052466 --- /dev/null +++ b/core/models/models/spider.go @@ -0,0 +1,32 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Spider struct { + any `collection:"spiders"` + BaseModel[Spider] `bson:",inline"` + Name string `json:"name" bson:"name"` // spider name + ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id (deprecated) # TODO: remove this field in the future + ColName string `json:"col_name,omitempty" bson:"col_name"` // data collection name + DbName string `json:"db_name,omitempty" bson:"db_name"` // database name + DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id + DataSource *Database `json:"data_source,omitempty" bson:"-"` // data source + Description string `json:"description" bson:"description"` // description + ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id + Mode string `json:"mode" bson:"mode"` // default Task.Mode + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds + GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id + GitRootPath string `json:"git_root_path" bson:"git_root_path"` + Git *Git `json:"git,omitempty" bson:"-"` + + // stats + Stat *SpiderStat `json:"stat,omitempty" bson:"-"` + + // execution + Cmd string `json:"cmd" bson:"cmd"` // execute command + Param string `json:"param" bson:"param"` // default task param + Priority int `json:"priority" bson:"priority"` + AutoInstall bool `json:"auto_install" bson:"auto_install"` +} diff --git a/core/models/models/spider_stat.go b/core/models/models/spider_stat.go new file mode 100644 index 00000000..a3c9f631 --- /dev/null +++ b/core/models/models/spider_stat.go @@ -0,0 +1,20 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type SpiderStat struct { + any `collection:"spider_stats"` + BaseModel[SpiderStat] `bson:",inline"` + LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"` + LastTask *Task `json:"last_task,omitempty" bson:"-"` + Tasks int `json:"tasks" bson:"tasks"` + Results int `json:"results" bson:"results"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second + AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second + AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second + AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second +} diff --git a/core/models/models/task.go b/core/models/models/task.go new file mode 100644 index 00000000..9fa2083e --- /dev/null +++ b/core/models/models/task.go @@ -0,0 +1,27 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Task struct { + any `collection:"tasks"` + BaseModel[Task] `bson:",inline"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` + Status string `json:"status" bson:"status"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` + Error string `json:"error" bson:"error"` + Pid int `json:"pid" bson:"pid"` + ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` + Type string `json:"type" bson:"type"` + Mode string `json:"mode" bson:"mode"` + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` + Priority int `json:"priority" bson:"priority"` + Stat *TaskStat `json:"stat,omitempty" bson:"-"` + Spider *Spider `json:"spider,omitempty" bson:"-"` + Schedule *Schedule `json:"schedule,omitempty" bson:"-"` + Node *Node `json:"node,omitempty" bson:"-"` + UserId primitive.ObjectID `json:"-" bson:"-"` +} diff --git a/core/models/models/task_stat.go b/core/models/models/task_stat.go new file mode 100644 index 00000000..900e8c24 --- /dev/null +++ b/core/models/models/task_stat.go @@ -0,0 +1,16 @@ +package models + +import ( + "time" +) + +type TaskStat struct { + any `collection:"task_stats"` + BaseModel[TaskStat] `bson:",inline"` + StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` + EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond + ResultCount int64 `json:"result_count" bson:"result_count"` +} diff --git a/core/models/models/test.go b/core/models/models/test.go new file mode 100644 index 00000000..bd31b833 --- /dev/null +++ b/core/models/models/test.go @@ -0,0 +1,7 @@ +package models + +type TestModel struct { + any `collection:"testmodels"` + BaseModel[TestModel] `bson:",inline"` + Name string `json:"name" bson:"name"` +} diff --git a/core/models/models/token.go b/core/models/models/token.go new file mode 100644 index 00000000..5657777c --- /dev/null +++ b/core/models/models/token.go @@ -0,0 +1,8 @@ +package models + +type Token struct { + any `collection:"tokens"` + BaseModel[Token] `bson:",inline"` + Name string `json:"name" bson:"name"` + Token string `json:"token" bson:"token"` +} diff --git a/core/models/models/user.go b/core/models/models/user.go new file mode 100644 index 00000000..d84b4efa --- /dev/null +++ b/core/models/models/user.go @@ -0,0 +1,10 @@ +package models + +type User struct { + any `collection:"users"` + BaseModel[User] `bson:",inline"` + Username string `json:"username" bson:"username"` + Password string `json:"-,omitempty" bson:"password"` + Role string `json:"role" bson:"role"` + Email string `json:"email" bson:"email"` +} diff --git a/core/models/models/user_role.go b/core/models/models/user_role.go new file mode 100644 index 00000000..ca54f520 --- /dev/null +++ b/core/models/models/user_role.go @@ -0,0 +1,12 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type UserRole struct { + any `collection:"user_roles"` + BaseModel[UserRole] `bson:",inline"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + UserId primitive.ObjectID `json:"user_id" bson:"user_id"` +} diff --git a/core/models/models/v2/data_collection_v2.go b/core/models/models/v2/data_collection_v2.go deleted file mode 100644 index 8ec5e8cb..00000000 --- a/core/models/models/v2/data_collection_v2.go +++ /dev/null @@ -1,17 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/entity" -) - -type DataCollectionV2 struct { - any `collection:"data_collections"` - BaseModelV2[DataCollectionV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Fields []entity.DataField `json:"fields" bson:"fields"` - Dedup struct { - Enabled bool `json:"enabled" bson:"enabled"` - Keys []string `json:"keys" bson:"keys"` - Type string `json:"type" bson:"type"` - } `json:"dedup" bson:"dedup"` -} diff --git a/core/models/models/v2/database_metric_v2.go b/core/models/models/v2/database_metric_v2.go deleted file mode 100644 index 0d5b0081..00000000 --- a/core/models/models/v2/database_metric_v2.go +++ /dev/null @@ -1,24 +0,0 @@ -package models - -import "go.mongodb.org/mongo-driver/bson/primitive" - -type DatabaseMetricV2 struct { - any `collection:"database_metrics"` - BaseModelV2[DatabaseMetricV2] `bson:",inline"` - DatabaseId primitive.ObjectID `json:"database_id" bson:"database_id"` - CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` - TotalMemory uint64 `json:"total_memory" bson:"total_memory"` - AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` - UsedMemory uint64 `json:"used_memory" bson:"used_memory"` - UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` - TotalDisk uint64 `json:"total_disk" bson:"total_disk"` - AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` - UsedDisk uint64 `json:"used_disk" bson:"used_disk"` - UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` - Connections int `json:"connections" bson:"connections"` - QueryPerSecond float64 `json:"query_per_second" bson:"query_per_second"` - TotalQuery uint64 `json:"total_query,omitempty" bson:"total_query,omitempty"` - CacheHitRatio float64 `json:"cache_hit_ratio" bson:"cache_hit_ratio"` - ReplicationLag float64 `json:"replication_lag" bson:"replication_lag"` - LockWaitTime float64 `json:"lock_wait_time" bson:"lock_wait_time"` -} diff --git a/core/models/models/v2/dependency_log_v2.go b/core/models/models/v2/dependency_log_v2.go deleted file mode 100644 index f0178c2e..00000000 --- a/core/models/models/v2/dependency_log_v2.go +++ /dev/null @@ -1,10 +0,0 @@ -package models - -import "go.mongodb.org/mongo-driver/bson/primitive" - -type DependencyLogV2 struct { - any `collection:"dependency_logs"` - BaseModelV2[DependencyLogV2] `bson:",inline"` - TaskId primitive.ObjectID `json:"task_id" bson:"task_id"` - Content string `json:"content" bson:"content"` -} diff --git a/core/models/models/v2/dependency_repo.go b/core/models/models/v2/dependency_repo.go deleted file mode 100644 index 464464e6..00000000 --- a/core/models/models/v2/dependency_repo.go +++ /dev/null @@ -1,10 +0,0 @@ -package models - -type DependencyRepoV2 struct { - any `collection:"dependency_repos"` - BaseModelV2[DependencyRepoV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Type string `json:"type" bson:"type"` - LatestVersion string `json:"latest_version" bson:"latest_version"` - AllVersions []string `json:"all_versions" bson:"all_versions"` -} diff --git a/core/models/models/v2/dependency_setting_v2.go b/core/models/models/v2/dependency_setting_v2.go deleted file mode 100644 index 528c44d0..00000000 --- a/core/models/models/v2/dependency_setting_v2.go +++ /dev/null @@ -1,17 +0,0 @@ -package models - -import ( - "time" -) - -type DependencySettingV2 struct { - any `collection:"dependency_settings"` - BaseModelV2[DependencySettingV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - Cmd string `json:"cmd" bson:"cmd"` - Proxy string `json:"proxy" bson:"proxy"` - LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"` -} diff --git a/core/models/models/v2/dependency_task_v2.go b/core/models/models/v2/dependency_task_v2.go deleted file mode 100644 index c9fd40d6..00000000 --- a/core/models/models/v2/dependency_task_v2.go +++ /dev/null @@ -1,15 +0,0 @@ -package models - -import "go.mongodb.org/mongo-driver/bson/primitive" - -type DependencyTaskV2 struct { - any `collection:"dependency_tasks"` - BaseModelV2[DependencyTaskV2] `bson:",inline"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` - Type string `json:"type" bson:"type"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - Action string `json:"action" bson:"action"` - DepNames []string `json:"dep_names" bson:"dep_names"` -} diff --git a/core/models/models/v2/dependency_v2.go b/core/models/models/v2/dependency_v2.go deleted file mode 100644 index a7fd95a0..00000000 --- a/core/models/models/v2/dependency_v2.go +++ /dev/null @@ -1,20 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type DependencyV2 struct { - any `collection:"dependencies"` - BaseModelV2[DependencyV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - Type string `json:"type" bson:"type"` - Version string `json:"version" bson:"version"` - Status string `json:"status" bson:"status"` - Error string `json:"error,omitempty" bson:"error,omitempty"` - Logs []string `json:"logs,omitempty" bson:"logs,omitempty"` - NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-"` - Versions []string `json:"versions,omitempty" bson:"-"` -} diff --git a/core/models/models/v2/environment_v2.go b/core/models/models/v2/environment_v2.go deleted file mode 100644 index 776a1142..00000000 --- a/core/models/models/v2/environment_v2.go +++ /dev/null @@ -1,8 +0,0 @@ -package models - -type EnvironmentV2 struct { - any `collection:"environments"` - BaseModelV2[EnvironmentV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Value string `json:"value" bson:"value"` -} diff --git a/core/models/models/v2/git_v2.go b/core/models/models/v2/git_v2.go deleted file mode 100644 index 730c2c89..00000000 --- a/core/models/models/v2/git_v2.go +++ /dev/null @@ -1,26 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/vcs" - "time" -) - -type GitV2 struct { - any `collection:"gits"` - BaseModelV2[GitV2] `bson:",inline"` - Url string `json:"url" bson:"url"` - Name string `json:"name" bson:"name"` - AuthType string `json:"auth_type" bson:"auth_type"` - Username string `json:"username" bson:"username"` - Password string `json:"password" bson:"password"` - CurrentBranch string `json:"current_branch" bson:"current_branch"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"` - Refs []vcs.GitRef `json:"refs" bson:"refs"` - RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"` - CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"` - - // settings - AutoPull bool `json:"auto_pull" bson:"auto_pull"` -} diff --git a/core/models/models/v2/metric_v2.go b/core/models/models/v2/metric_v2.go deleted file mode 100644 index 8d23dd91..00000000 --- a/core/models/models/v2/metric_v2.go +++ /dev/null @@ -1,25 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type MetricV2 struct { - any `collection:"metrics"` - BaseModelV2[MetricV2] `bson:",inline"` - Type string `json:"type" bson:"type"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` - TotalMemory uint64 `json:"total_memory" bson:"total_memory"` - AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` - UsedMemory uint64 `json:"used_memory" bson:"used_memory"` - UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` - TotalDisk uint64 `json:"total_disk" bson:"total_disk"` - AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` - UsedDisk uint64 `json:"used_disk" bson:"used_disk"` - UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` - DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"` - DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"` - NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"` - NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"` -} diff --git a/core/models/models/v2/node_v2.go b/core/models/models/v2/node_v2.go deleted file mode 100644 index 968b9bc9..00000000 --- a/core/models/models/v2/node_v2.go +++ /dev/null @@ -1,23 +0,0 @@ -package models - -import ( - "time" -) - -type NodeV2 struct { - any `collection:"nodes"` - BaseModelV2[NodeV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Ip string `json:"ip" bson:"ip"` - Mac string `json:"mac" bson:"mac"` - Hostname string `json:"hostname" bson:"hostname"` - Description string `json:"description" bson:"description"` - IsMaster bool `json:"is_master" bson:"is_master"` - Status string `json:"status" bson:"status"` - Enabled bool `json:"enabled" bson:"enabled"` - Active bool `json:"active" bson:"active"` - ActiveAt time.Time `json:"active_at" bson:"active_ts"` - AvailableRunners int `json:"available_runners" bson:"available_runners"` - MaxRunners int `json:"max_runners" bson:"max_runners"` -} diff --git a/core/models/models/v2/notification_alert_v2.go b/core/models/models/v2/notification_alert_v2.go deleted file mode 100644 index d91e4436..00000000 --- a/core/models/models/v2/notification_alert_v2.go +++ /dev/null @@ -1,19 +0,0 @@ -package models - -import "go.mongodb.org/mongo-driver/bson/primitive" - -type NotificationAlertV2 struct { - any `collection:"notification_alerts"` - BaseModelV2[NotificationAlertV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - HasMetricTarget bool `json:"has_metric_target" bson:"has_metric_target"` - MetricTargetId primitive.ObjectID `json:"metric_target_id,omitempty" bson:"metric_target_id,omitempty"` - MetricName string `json:"metric_name" bson:"metric_name"` - Operator string `json:"operator" bson:"operator"` - LastingSeconds int `json:"lasting_seconds" bson:"lasting_seconds"` - TargetValue float32 `json:"target_value" bson:"target_value"` - Level string `json:"level" bson:"level"` - TemplateKey string `json:"template_key,omitempty" bson:"template_key,omitempty"` -} diff --git a/core/models/models/v2/notification_channel_v2.go b/core/models/models/v2/notification_channel_v2.go deleted file mode 100644 index 2e9ebed3..00000000 --- a/core/models/models/v2/notification_channel_v2.go +++ /dev/null @@ -1,18 +0,0 @@ -package models - -type NotificationChannelV2 struct { - any `collection:"notification_channels"` - BaseModelV2[NotificationChannelV2] `bson:",inline"` - Type string `json:"type" bson:"type"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Provider string `json:"provider" bson:"provider"` - SMTPServer string `json:"smtp_server,omitempty" bson:"smtp_server,omitempty"` - SMTPPort int `json:"smtp_port,omitempty" bson:"smtp_port,omitempty"` - SMTPUsername string `json:"smtp_username,omitempty" bson:"smtp_username,omitempty"` - SMTPPassword string `json:"smtp_password,omitempty" bson:"smtp_password,omitempty"` - WebhookUrl string `json:"webhook_url,omitempty" bson:"webhook_url,omitempty"` - TelegramBotToken string `json:"telegram_bot_token,omitempty" bson:"telegram_bot_token,omitempty"` - TelegramChatId string `json:"telegram_chat_id,omitempty" bson:"telegram_chat_id,omitempty"` - GoogleOAuth2Json string `json:"google_oauth2_json,omitempty" bson:"google_oauth2_json,omitempty"` -} diff --git a/core/models/models/v2/notification_request_v2.go b/core/models/models/v2/notification_request_v2.go deleted file mode 100644 index bbfe6023..00000000 --- a/core/models/models/v2/notification_request_v2.go +++ /dev/null @@ -1,21 +0,0 @@ -package models - -import "go.mongodb.org/mongo-driver/bson/primitive" - -type NotificationRequestV2 struct { - any `collection:"notification_requests"` - BaseModelV2[NotificationRequestV2] `bson:",inline"` - Status string `json:"status" bson:"status"` - Error string `json:"error,omitempty" bson:"error,omitempty"` - Title string `json:"title" bson:"title"` - Content string `json:"content" bson:"content"` - SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` - SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"` - MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty"` - MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` - MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` - SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` - ChannelId primitive.ObjectID `json:"channel_id" bson:"channel_id"` - Setting *NotificationSettingV2 `json:"setting,omitempty" bson:"-"` - Channel *NotificationChannelV2 `json:"channel,omitempty" bson:"-"` -} diff --git a/core/models/models/v2/permission_v2.go b/core/models/models/v2/permission_v2.go deleted file mode 100644 index 77c830a3..00000000 --- a/core/models/models/v2/permission_v2.go +++ /dev/null @@ -1,13 +0,0 @@ -package models - -type PermissionV2 struct { - any `collection:"permissions"` - BaseModelV2[PermissionV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Type string `json:"type" bson:"type"` - Target []string `json:"target" bson:"target"` - Allow []string `json:"allow" bson:"allow"` - Deny []string `json:"deny" bson:"deny"` -} diff --git a/core/models/models/v2/project_v2.go b/core/models/models/v2/project_v2.go deleted file mode 100644 index ee07e705..00000000 --- a/core/models/models/v2/project_v2.go +++ /dev/null @@ -1,9 +0,0 @@ -package models - -type ProjectV2 struct { - any `collection:"projects"` - BaseModelV2[ProjectV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Spiders int `json:"spiders" bson:"-"` -} diff --git a/core/models/models/v2/role_permission_v2.go b/core/models/models/v2/role_permission_v2.go deleted file mode 100644 index 386fa052..00000000 --- a/core/models/models/v2/role_permission_v2.go +++ /dev/null @@ -1,12 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type RolePermissionV2 struct { - any `collection:"role_permissions"` - BaseModelV2[RolePermissionV2] `bson:",inline"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"` -} diff --git a/core/models/models/v2/role_v2.go b/core/models/models/v2/role_v2.go deleted file mode 100644 index de287a61..00000000 --- a/core/models/models/v2/role_v2.go +++ /dev/null @@ -1,9 +0,0 @@ -package models - -type RoleV2 struct { - any `collection:"roles"` - BaseModelV2[RoleV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` -} diff --git a/core/models/models/v2/schedule_v2.go b/core/models/models/v2/schedule_v2.go deleted file mode 100644 index 4d8cf42c..00000000 --- a/core/models/models/v2/schedule_v2.go +++ /dev/null @@ -1,22 +0,0 @@ -package models - -import ( - "github.com/robfig/cron/v3" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type ScheduleV2 struct { - any `collection:"schedules"` - BaseModelV2[ScheduleV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` - Cron string `json:"cron" bson:"cron"` - EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` - Cmd string `json:"cmd" bson:"cmd"` - Param string `json:"param" bson:"param"` - Mode string `json:"mode" bson:"mode"` - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` - Priority int `json:"priority" bson:"priority"` - Enabled bool `json:"enabled" bson:"enabled"` -} diff --git a/core/models/models/v2/setting_v2.go b/core/models/models/v2/setting_v2.go deleted file mode 100644 index 86481607..00000000 --- a/core/models/models/v2/setting_v2.go +++ /dev/null @@ -1,12 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson" -) - -type SettingV2 struct { - any `collection:"settings"` - BaseModelV2[SettingV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Value bson.M `json:"value" bson:"value"` -} diff --git a/core/models/models/v2/spider_stat_v2.go b/core/models/models/v2/spider_stat_v2.go deleted file mode 100644 index ea01c0cc..00000000 --- a/core/models/models/v2/spider_stat_v2.go +++ /dev/null @@ -1,20 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type SpiderStatV2 struct { - any `collection:"spider_stats"` - BaseModelV2[SpiderStatV2] `bson:",inline"` - LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"` - LastTask *TaskV2 `json:"last_task,omitempty" bson:"-"` - Tasks int `json:"tasks" bson:"tasks"` - Results int `json:"results" bson:"results"` - WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second - RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second - TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second - AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second - AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second - AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second -} diff --git a/core/models/models/v2/spider_v2.go b/core/models/models/v2/spider_v2.go deleted file mode 100644 index 3c99f1fb..00000000 --- a/core/models/models/v2/spider_v2.go +++ /dev/null @@ -1,32 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type SpiderV2 struct { - any `collection:"spiders"` - BaseModelV2[SpiderV2] `bson:",inline"` - Name string `json:"name" bson:"name"` // spider name - ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id (deprecated) # TODO: remove this field in the future - ColName string `json:"col_name,omitempty" bson:"col_name"` // data collection name - DbName string `json:"db_name,omitempty" bson:"db_name"` // database name - DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id - DataSource *DatabaseV2 `json:"data_source,omitempty" bson:"-"` // data source - Description string `json:"description" bson:"description"` // description - ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id - Mode string `json:"mode" bson:"mode"` // default Task.Mode - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds - GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id - GitRootPath string `json:"git_root_path" bson:"git_root_path"` - Git *GitV2 `json:"git,omitempty" bson:"-"` - - // stats - Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"` - - // execution - Cmd string `json:"cmd" bson:"cmd"` // execute command - Param string `json:"param" bson:"param"` // default task param - Priority int `json:"priority" bson:"priority"` - AutoInstall bool `json:"auto_install" bson:"auto_install"` -} diff --git a/core/models/models/v2/task_stat_v2.go b/core/models/models/v2/task_stat_v2.go deleted file mode 100644 index 65459244..00000000 --- a/core/models/models/v2/task_stat_v2.go +++ /dev/null @@ -1,16 +0,0 @@ -package models - -import ( - "time" -) - -type TaskStatV2 struct { - any `collection:"task_stats"` - BaseModelV2[TaskStatV2] `bson:",inline"` - StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` - EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` - WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond - RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond - TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond - ResultCount int64 `json:"result_count" bson:"result_count"` -} diff --git a/core/models/models/v2/task_v2.go b/core/models/models/v2/task_v2.go deleted file mode 100644 index ad7fb05b..00000000 --- a/core/models/models/v2/task_v2.go +++ /dev/null @@ -1,27 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type TaskV2 struct { - any `collection:"tasks"` - BaseModelV2[TaskV2] `bson:",inline"` - SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` - Status string `json:"status" bson:"status"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - Cmd string `json:"cmd" bson:"cmd"` - Param string `json:"param" bson:"param"` - Error string `json:"error" bson:"error"` - Pid int `json:"pid" bson:"pid"` - ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` - Type string `json:"type" bson:"type"` - Mode string `json:"mode" bson:"mode"` - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` - Priority int `json:"priority" bson:"priority"` - Stat *TaskStatV2 `json:"stat,omitempty" bson:"-"` - Spider *SpiderV2 `json:"spider,omitempty" bson:"-"` - Schedule *ScheduleV2 `json:"schedule,omitempty" bson:"-"` - Node *NodeV2 `json:"node,omitempty" bson:"-"` - UserId primitive.ObjectID `json:"-" bson:"-"` -} diff --git a/core/models/models/v2/test_v2.go b/core/models/models/v2/test_v2.go deleted file mode 100644 index 1468abc1..00000000 --- a/core/models/models/v2/test_v2.go +++ /dev/null @@ -1,7 +0,0 @@ -package models - -type TestModelV2 struct { - any `collection:"testmodels"` - BaseModelV2[TestModelV2] `bson:",inline"` - Name string `json:"name" bson:"name"` -} diff --git a/core/models/models/v2/token_v2.go b/core/models/models/v2/token_v2.go deleted file mode 100644 index c792c9ee..00000000 --- a/core/models/models/v2/token_v2.go +++ /dev/null @@ -1,8 +0,0 @@ -package models - -type TokenV2 struct { - any `collection:"tokens"` - BaseModelV2[TokenV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Token string `json:"token" bson:"token"` -} diff --git a/core/models/models/v2/user_role_v2.go b/core/models/models/v2/user_role_v2.go deleted file mode 100644 index aa0ac34a..00000000 --- a/core/models/models/v2/user_role_v2.go +++ /dev/null @@ -1,12 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type UserRoleV2 struct { - any `collection:"user_roles"` - BaseModelV2[UserRoleV2] `bson:",inline"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - UserId primitive.ObjectID `json:"user_id" bson:"user_id"` -} diff --git a/core/models/models/v2/user_v2.go b/core/models/models/v2/user_v2.go deleted file mode 100644 index acfbe2b6..00000000 --- a/core/models/models/v2/user_v2.go +++ /dev/null @@ -1,10 +0,0 @@ -package models - -type UserV2 struct { - any `collection:"users"` - BaseModelV2[UserV2] `bson:",inline"` - Username string `json:"username" bson:"username"` - Password string `json:"-,omitempty" bson:"password"` - Role string `json:"role" bson:"role"` - Email string `json:"email" bson:"email"` -} diff --git a/core/models/service/base_service.go b/core/models/service/base_service.go index 4cc2002d..1add6c18 100644 --- a/core/models/service/base_service.go +++ b/core/models/service/base_service.go @@ -279,7 +279,7 @@ func NewModelService[T any]() *ModelService[T] { defer mu.Unlock() if _, exists := onceMap[typeName]; !exists { - onceMap[typeName] = new(sync.Once) + onceMap[typeName] = &sync.Once{} } var instance *ModelService[T] diff --git a/core/models/service/base_service_test.go b/core/models/service/base_service_test.go index d26559b4..d354f6ab 100644 --- a/core/models/service/base_service_test.go +++ b/core/models/service/base_service_test.go @@ -3,10 +3,10 @@ package service_test import ( "context" "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/models/models" "testing" "time" - "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "github.com/spf13/viper" @@ -17,9 +17,9 @@ import ( ) type TestModel struct { - Id primitive.ObjectID `bson:"_id,omitempty" collection:"testmodels"` - models.BaseModelV2[TestModel] `bson:",inline"` - Name string `bson:"name"` + Id primitive.ObjectID `bson:"_id,omitempty" collection:"testmodels"` + models.BaseModel[TestModel] `bson:",inline"` + Name string `bson:"name"` } func setupTestDB() { @@ -36,7 +36,7 @@ func teardownTestDB() { log.Infof("dropped test db") } -func TestModelServiceV2(t *testing.T) { +func TestModelService(t *testing.T) { setupTestDB() defer teardownTestDB() diff --git a/core/node/config/config_service.go b/core/node/config/config_service.go index 9a4cee4f..19322947 100644 --- a/core/node/config/config_service.go +++ b/core/node/config/config_service.go @@ -116,7 +116,7 @@ func newNodeConfigService() (svc2 interfaces.NodeConfigService, err error) { } var _service interfaces.NodeConfigService -var _serviceOnce = new(sync.Once) +var _serviceOnce sync.Once func GetNodeConfigService() interfaces.NodeConfigService { _serviceOnce.Do(func() { diff --git a/core/node/service/master_service.go b/core/node/service/master_service.go index 5c42c117..b70bb4e1 100644 --- a/core/node/service/master_service.go +++ b/core/node/service/master_service.go @@ -9,7 +9,7 @@ import ( "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/common" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/notification" @@ -29,12 +29,12 @@ import ( type MasterService struct { // dependencies - cfgSvc interfaces.NodeConfigService - server *server.GrpcServer - schedulerSvc *scheduler.Service - handlerSvc *handler.Service - scheduleSvc *schedule.ServiceV2 - systemSvc *system.ServiceV2 + cfgSvc interfaces.NodeConfigService + server *server.GrpcServer + taskSchedulerSvc *scheduler.Service + taskHandlerSvc *handler.Service + scheduleSvc *schedule.Service + systemSvc *system.Service // settings cfgPath string @@ -43,11 +43,6 @@ type MasterService struct { stopOnError bool } -func (svc *MasterService) Init() (err error) { - // do nothing - return nil -} - func (svc *MasterService) Start() { // create indexes common.InitIndexes() @@ -66,10 +61,10 @@ func (svc *MasterService) Start() { go svc.Monitor() // start task handler - go svc.handlerSvc.Start() + go svc.taskHandlerSvc.Start() // start task scheduler - go svc.schedulerSvc.Start() + go svc.taskSchedulerSvc.Start() // start schedule service go svc.scheduleSvc.Start() @@ -87,7 +82,7 @@ func (svc *MasterService) Wait() { func (svc *MasterService) Stop() { _ = svc.server.Stop() - svc.handlerSvc.Stop() + svc.taskHandlerSvc.Stop() log.Infof("master[%s] service has stopped", svc.GetConfigService().GetNodeKey()) } @@ -133,11 +128,11 @@ func (svc *MasterService) SetMonitorInterval(duration time.Duration) { func (svc *MasterService) Register() (err error) { nodeKey := svc.GetConfigService().GetNodeKey() nodeName := svc.GetConfigService().GetNodeName() - node, err := service.NewModelService[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + node, err := service.NewModelService[models.Node]().GetOne(bson.M{"key": nodeKey}, nil) if err != nil && err.Error() == mongo2.ErrNoDocuments.Error() { // not exists log.Infof("master[%s] does not exist in db", nodeKey) - node := models2.NodeV2{ + node := models.Node{ Key: nodeKey, Name: nodeName, MaxRunners: config.DefaultConfigOptions.MaxRunners, @@ -149,7 +144,7 @@ func (svc *MasterService) Register() (err error) { } node.SetCreated(primitive.NilObjectID) node.SetUpdated(primitive.NilObjectID) - id, err := service.NewModelService[models2.NodeV2]().InsertOne(node) + id, err := service.NewModelService[models.Node]().InsertOne(node) if err != nil { return err } @@ -161,7 +156,7 @@ func (svc *MasterService) Register() (err error) { node.Status = constants.NodeStatusOnline node.Active = true node.ActiveAt = time.Now() - err = service.NewModelService[models2.NodeV2]().ReplaceById(node.Id, *node) + err = service.NewModelService[models.Node]().ReplaceById(node.Id, *node) if err != nil { return err } @@ -192,7 +187,7 @@ func (svc *MasterService) monitor() (err error) { wg := sync.WaitGroup{} wg.Add(len(workerNodes)) for _, n := range workerNodes { - go func(n *models2.NodeV2) { + go func(n *models.Node) { defer wg.Done() // subscribe @@ -222,12 +217,12 @@ func (svc *MasterService) monitor() (err error) { return nil } -func (svc *MasterService) getAllWorkerNodes() (nodes []models2.NodeV2, err error) { +func (svc *MasterService) getAllWorkerNodes() (nodes []models.Node, err error) { query := bson.M{ "key": bson.M{"$ne": svc.cfgSvc.GetNodeKey()}, // not self "active": true, // active } - nodes, err = service.NewModelService[models2.NodeV2]().GetMany(query, nil) + nodes, err = service.NewModelService[models.Node]().GetMany(query, nil) if err != nil { if errors.Is(err, mongo2.ErrNoDocuments) { return nil, nil @@ -239,7 +234,7 @@ func (svc *MasterService) getAllWorkerNodes() (nodes []models2.NodeV2, err error func (svc *MasterService) updateMasterNodeStatus() (err error) { nodeKey := svc.GetConfigService().GetNodeKey() - node, err := service.NewModelService[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + node, err := service.NewModelService[models.Node]().GetOne(bson.M{"key": nodeKey}, nil) if err != nil { return err } @@ -250,7 +245,7 @@ func (svc *MasterService) updateMasterNodeStatus() (err error) { node.ActiveAt = time.Now() newStatus := node.Status - err = service.NewModelService[models2.NodeV2]().ReplaceById(node.Id, *node) + err = service.NewModelService[models.Node]().ReplaceById(node.Id, *node) if err != nil { return err } @@ -264,11 +259,11 @@ func (svc *MasterService) updateMasterNodeStatus() (err error) { return nil } -func (svc *MasterService) setWorkerNodeOffline(node *models2.NodeV2) { +func (svc *MasterService) setWorkerNodeOffline(node *models.Node) { node.Status = constants.NodeStatusOffline node.Active = false err := backoff.Retry(func() error { - return service.NewModelService[models2.NodeV2]().ReplaceById(node.Id, *node) + return service.NewModelService[models.Node]().ReplaceById(node.Id, *node) }, backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 3)) if err != nil { trace.PrintError(err) @@ -276,12 +271,12 @@ func (svc *MasterService) setWorkerNodeOffline(node *models2.NodeV2) { svc.sendNotification(node) } -func (svc *MasterService) subscribeNode(n *models2.NodeV2) (ok bool) { +func (svc *MasterService) subscribeNode(n *models.Node) (ok bool) { _, ok = svc.server.NodeSvr.GetSubscribeStream(n.Id) return ok } -func (svc *MasterService) pingNodeClient(n *models2.NodeV2) (ok bool) { +func (svc *MasterService) pingNodeClient(n *models.Node) (ok bool) { stream, ok := svc.server.NodeSvr.GetSubscribeStream(n.Id) if !ok { log.Errorf("cannot get worker node client[%s]", n.Key) @@ -297,85 +292,50 @@ func (svc *MasterService) pingNodeClient(n *models2.NodeV2) (ok bool) { return true } -func (svc *MasterService) updateNodeAvailableRunners(node *models2.NodeV2) (err error) { +func (svc *MasterService) updateNodeAvailableRunners(node *models.Node) (err error) { query := bson.M{ "node_id": node.Id, "status": constants.TaskStatusRunning, } - runningTasksCount, err := service.NewModelService[models2.TaskV2]().Count(query) + runningTasksCount, err := service.NewModelService[models.Task]().Count(query) if err != nil { return trace.TraceError(err) } node.AvailableRunners = node.MaxRunners - runningTasksCount - err = service.NewModelService[models2.NodeV2]().ReplaceById(node.Id, *node) + err = service.NewModelService[models.Node]().ReplaceById(node.Id, *node) if err != nil { return err } return nil } -func (svc *MasterService) sendNotification(node *models2.NodeV2) { +func (svc *MasterService) sendNotification(node *models.Node) { if !utils.IsPro() { return } - go notification.GetNotificationServiceV2().SendNodeNotification(node) + go notification.GetNotificationService().SendNodeNotification(node) } -func newMasterServiceV2() (res *MasterService, err error) { - // master service - svc := &MasterService{ - cfgPath: config2.GetConfigPath(), - monitorInterval: 15 * time.Second, - stopOnError: false, +func newMasterService() *MasterService { + return &MasterService{ + cfgPath: config2.GetConfigPath(), + cfgSvc: config.GetNodeConfigService(), + monitorInterval: 15 * time.Second, + stopOnError: false, + server: server.GetGrpcServer(), + taskSchedulerSvc: scheduler.GetTaskSchedulerService(), + taskHandlerSvc: handler.GetTaskHandlerService(), + scheduleSvc: schedule.GetScheduleService(), + systemSvc: system.GetSystemService(), } - - // node config service - svc.cfgSvc = config.GetNodeConfigService() - - // grpc server - svc.server, err = server.GetGrpcServer() - if err != nil { - return nil, err - } - - // scheduler service - svc.schedulerSvc, err = scheduler.GetTaskSchedulerService() - if err != nil { - return nil, err - } - - // handler service - svc.handlerSvc, err = handler.GetTaskHandlerService() - if err != nil { - return nil, err - } - - // schedule service - svc.scheduleSvc, err = schedule.GetScheduleServiceV2() - if err != nil { - return nil, err - } - - // system service - svc.systemSvc = system.GetSystemServiceV2() - - // init - if err := svc.Init(); err != nil { - return nil, err - } - - return svc, nil } var masterService *MasterService -var masterServiceOnce = new(sync.Once) +var masterServiceOnce sync.Once -func GetMasterService() (res *MasterService, err error) { +func GetMasterService() *MasterService { masterServiceOnce.Do(func() { - masterService, err = newMasterServiceV2() - if err != nil { - log.Errorf("failed to get master service: %v", err) - } + masterService = newMasterService() }) - return masterService, err + return masterService } diff --git a/core/node/service/worker_service.go b/core/node/service/worker_service.go index 4a7ae399..987b9585 100644 --- a/core/node/service/worker_service.go +++ b/core/node/service/worker_service.go @@ -5,13 +5,14 @@ import ( "sync" "time" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/apex/log" "github.com/cenkalti/backoff/v4" "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/grpc/client" "github.com/crawlab-team/crawlab/core/interfaces" client2 "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/models/v2" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/handler" "github.com/crawlab-team/crawlab/core/utils" @@ -33,15 +34,10 @@ type WorkerService struct { // internals stopped bool - n *models.NodeV2 + n *models.Node s grpc.NodeService_SubscribeClient } -func (svc *WorkerService) Init() (err error) { - // do nothing - return nil -} - func (svc *WorkerService) Start() { // start grpc client if err := svc.client.Start(); err != nil { @@ -90,7 +86,7 @@ func (svc *WorkerService) register() { log.Fatalf("failed to register worker[%s] to master: %v", svc.cfgSvc.GetNodeKey(), err) panic(err) } - svc.n, err = client2.NewModelService[models.NodeV2]().GetOne(bson.M{"key": svc.GetConfigService().GetNodeKey()}, nil) + svc.n, err = client2.NewModelService[models.Node]().GetOne(bson.M{"key": svc.GetConfigService().GetNodeKey()}, nil) if err != nil { log.Fatalf("failed to get node: %v", err) panic(err) @@ -197,42 +193,22 @@ func (svc *WorkerService) sendHeartbeat() { } } -var workerServiceV2 *WorkerService -var workerServiceV2Once = new(sync.Once) - -func newWorkerService() (res *WorkerService, err error) { - svc := &WorkerService{ +func newWorkerService() *WorkerService { + return &WorkerService{ cfgPath: config.GetConfigPath(), heartbeatInterval: 15 * time.Second, + cfgSvc: nodeconfig.GetNodeConfigService(), + client: client.GetGrpcClient(), + handlerSvc: handler.GetTaskHandlerService(), } - - // node config service - svc.cfgSvc = nodeconfig.GetNodeConfigService() - - // grpc client - svc.client = client.GetGrpcClient() - - // handler service - svc.handlerSvc, err = handler.GetTaskHandlerService() - if err != nil { - return nil, err - } - - // init - err = svc.Init() - if err != nil { - return nil, err - } - - return svc, nil } -func GetWorkerService() (res *WorkerService, err error) { - workerServiceV2Once.Do(func() { - workerServiceV2, err = newWorkerService() - if err != nil { - log.Errorf("failed to get worker service: %v", err) - } +var workerService *WorkerService +var workerServiceOnce sync.Once + +func GetWorkerService() *WorkerService { + workerServiceOnce.Do(func() { + workerService = newWorkerService() }) - return workerServiceV2, err + return workerService } diff --git a/core/notification/entity.go b/core/notification/entity.go index cf7933ee..6ae0fed0 100644 --- a/core/notification/entity.go +++ b/core/notification/entity.go @@ -1,13 +1,15 @@ package notification -import "github.com/crawlab-team/crawlab/core/models/models/v2" +import ( + "github.com/crawlab-team/crawlab/core/models/models" +) type VariableData struct { - Task *models.TaskV2 `json:"task"` - TaskStat *models.TaskStatV2 `json:"task_stat"` - Spider *models.SpiderV2 `json:"spider"` - Node *models.NodeV2 `json:"node"` - Schedule *models.ScheduleV2 `json:"schedule"` - Alert *models.NotificationAlertV2 `json:"alert"` - Metric *models.MetricV2 `json:"metric"` + Task *models.Task `json:"task"` + TaskStat *models.TaskStat `json:"task_stat"` + Spider *models.Spider `json:"spider"` + Node *models.Node `json:"node"` + Schedule *models.Schedule `json:"schedule"` + Alert *models.NotificationAlert `json:"alert"` + Metric *models.Metric `json:"metric"` } diff --git a/core/notification/im.go b/core/notification/im.go index 181b8b89..ea2da58f 100644 --- a/core/notification/im.go +++ b/core/notification/im.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/trace" "github.com/imroc/req" "regexp" @@ -16,7 +16,7 @@ type ResBody struct { ErrMsg string `json:"errmsg"` } -func SendIMNotification(ch *models.NotificationChannelV2, title, content string) error { +func SendIMNotification(ch *models.NotificationChannel, title, content string) error { switch ch.Provider { case ChannelIMProviderLark: return sendIMLark(ch, title, content) @@ -192,7 +192,7 @@ func convertMarkdownToTelegram(markdownText string) string { return markdownText } -func sendIMLark(ch *models.NotificationChannelV2, title, content string) error { +func sendIMLark(ch *models.NotificationChannel, title, content string) error { data := req.Param{ "msg_type": "interactive", "card": req.Param{ @@ -220,7 +220,7 @@ func sendIMLark(ch *models.NotificationChannelV2, title, content string) error { return nil } -func sendIMDingTalk(ch *models.NotificationChannelV2, title string, content string) error { +func sendIMDingTalk(ch *models.NotificationChannel, title string, content string) error { data := req.Param{ "msgtype": "markdown", "markdown": req.Param{ @@ -238,7 +238,7 @@ func sendIMDingTalk(ch *models.NotificationChannelV2, title string, content stri return nil } -func sendIMWechatWork(ch *models.NotificationChannelV2, title string, content string) error { +func sendIMWechatWork(ch *models.NotificationChannel, title string, content string) error { data := req.Param{ "msgtype": "markdown", "markdown": req.Param{ @@ -255,7 +255,7 @@ func sendIMWechatWork(ch *models.NotificationChannelV2, title string, content st return nil } -func sendIMSlack(ch *models.NotificationChannelV2, title, content string) error { +func sendIMSlack(ch *models.NotificationChannel, title, content string) error { data := req.Param{ "blocks": []req.Param{ {"type": "header", "text": req.Param{"type": "plain_text", "text": title}}, @@ -269,7 +269,7 @@ func sendIMSlack(ch *models.NotificationChannelV2, title, content string) error return nil } -func sendIMTelegram(ch *models.NotificationChannelV2, title string, content string) error { +func sendIMTelegram(ch *models.NotificationChannel, title string, content string) error { type ResBody struct { Ok bool `json:"ok"` Description string `json:"description"` @@ -305,7 +305,7 @@ func sendIMTelegram(ch *models.NotificationChannelV2, title string, content stri return nil } -func sendIMDiscord(ch *models.NotificationChannelV2, title string, content string) error { +func sendIMDiscord(ch *models.NotificationChannel, title string, content string) error { data := req.Param{ "embeds": []req.Param{ { @@ -321,7 +321,7 @@ func sendIMDiscord(ch *models.NotificationChannelV2, title string, content strin return nil } -func sendIMMSTeams(ch *models.NotificationChannelV2, title string, content string) error { +func sendIMMSTeams(ch *models.NotificationChannel, title string, content string) error { data := req.Param{ "type": "message", "attachments": []req.Param{{ diff --git a/core/notification/mail.go b/core/notification/mail.go index 49cd5bd0..8546d47b 100644 --- a/core/notification/mail.go +++ b/core/notification/mail.go @@ -4,7 +4,7 @@ import ( "errors" "github.com/PuerkitoBio/goquery" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/trace" "gopkg.in/gomail.v2" "net/mail" @@ -12,7 +12,7 @@ import ( "strings" ) -func SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, to, cc, bcc []string, title, content string) error { +func SendMail(s *models.NotificationSetting, ch *models.NotificationChannel, to, cc, bcc []string, title, content string) error { // sender email senderEmail := ch.SMTPUsername if s.UseCustomSenderEmail { diff --git a/core/notification/mail_gmail.go b/core/notification/mail_gmail.go index aebf4e5f..1d952032 100644 --- a/core/notification/mail_gmail.go +++ b/core/notification/mail_gmail.go @@ -4,14 +4,14 @@ import ( "context" "encoding/base64" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/trace" "golang.org/x/oauth2/google" "google.golang.org/api/gmail/v1" "strings" ) -func sendMailGmail(ch *models.NotificationChannelV2, smtpConfig smtpAuthentication, options sendOptions, htmlBody, txtBody string) error { +func sendMailGmail(ch *models.NotificationChannel, smtpConfig smtpAuthentication, options sendOptions, htmlBody, txtBody string) error { // 读取服务账户 JSON 密钥 b := []byte(ch.GoogleOAuth2Json) diff --git a/core/notification/service_v2.go b/core/notification/service.go similarity index 82% rename from core/notification/service_v2.go rename to core/notification/service.go index 1563fd0e..973c7011 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service.go @@ -2,25 +2,26 @@ package notification import ( "fmt" + "regexp" + "strings" + "sync" + "time" + "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/trace" "github.com/gomarkdown/markdown" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - "regexp" - "strings" - "sync" - "time" ) -type ServiceV2 struct { +type Service struct { } -func (svc *ServiceV2) Send(s *models.NotificationSettingV2, args ...any) { +func (svc *Service) Send(s *models.NotificationSetting, args ...any) { title := s.Title wg := sync.WaitGroup{} @@ -28,9 +29,9 @@ func (svc *ServiceV2) Send(s *models.NotificationSettingV2, args ...any) { for _, chId := range s.ChannelIds { go func(chId primitive.ObjectID) { defer wg.Done() - ch, err := service.NewModelService[models.NotificationChannelV2]().GetById(chId) + ch, err := service.NewModelService[models.NotificationChannel]().GetById(chId) if err != nil { - log.Errorf("[NotificationServiceV2] get channel error: %v", err) + log.Errorf("[NotificationService] get channel error: %v", err) return } content := svc.getContent(s, ch, args...) @@ -45,7 +46,7 @@ func (svc *ServiceV2) Send(s *models.NotificationSettingV2, args ...any) { wg.Wait() } -func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) { +func (svc *Service) SendMail(s *models.NotificationSetting, ch *models.NotificationChannel, title, content string) { mailTo := s.MailTo mailCc := s.MailCc mailBcc := s.MailBcc @@ -56,28 +57,28 @@ func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, ch *models.Notif // send mail err := SendMail(s, ch, mailTo, mailCc, mailBcc, title, content) if err != nil { - log.Errorf("[NotificationServiceV2] send mail error: %v", err) + log.Errorf("[NotificationService] send mail error: %v", err) } // save request go svc.saveRequest(r, err) } -func (svc *ServiceV2) SendIM(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) { +func (svc *Service) SendIM(s *models.NotificationSetting, ch *models.NotificationChannel, title, content string) { // request r, _ := svc.createRequest(s, ch, title, content) // send mobile notification err := SendIMNotification(ch, title, content) if err != nil { - log.Errorf("[NotificationServiceV2] send mobile notification error: %v", err) + log.Errorf("[NotificationService] send mobile notification error: %v", err) } // save request go svc.saveRequest(r, err) } -func (svc *ServiceV2) getContent(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, args ...any) (content string) { +func (svc *Service) getContent(s *models.NotificationSetting, ch *models.NotificationChannel, args ...any) (content string) { vd := svc.getVariableData(args...) switch s.TemplateMode { case constants.NotificationTemplateModeMarkdown: @@ -99,7 +100,7 @@ func (svc *ServiceV2) getContent(s *models.NotificationSettingV2, ch *models.Not return content } -func (svc *ServiceV2) geContentWithVariables(template string, variables []entity.NotificationVariable, vd VariableData) (content string) { +func (svc *Service) geContentWithVariables(template string, variables []entity.NotificationVariable, vd VariableData) (content string) { content = template for _, v := range variables { switch v.Category { @@ -306,29 +307,29 @@ func (svc *ServiceV2) geContentWithVariables(template string, variables []entity return content } -func (svc *ServiceV2) getVariableData(args ...any) (vd VariableData) { +func (svc *Service) getVariableData(args ...any) (vd VariableData) { for _, arg := range args { switch arg.(type) { - case *models.TaskV2: - vd.Task = arg.(*models.TaskV2) - case *models.TaskStatV2: - vd.TaskStat = arg.(*models.TaskStatV2) - case *models.SpiderV2: - vd.Spider = arg.(*models.SpiderV2) - case *models.NodeV2: - vd.Node = arg.(*models.NodeV2) - case *models.ScheduleV2: - vd.Schedule = arg.(*models.ScheduleV2) - case *models.NotificationAlertV2: - vd.Alert = arg.(*models.NotificationAlertV2) - case *models.MetricV2: - vd.Metric = arg.(*models.MetricV2) + case *models.Task: + vd.Task = arg.(*models.Task) + case *models.TaskStat: + vd.TaskStat = arg.(*models.TaskStat) + case *models.Spider: + vd.Spider = arg.(*models.Spider) + case *models.Node: + vd.Node = arg.(*models.Node) + case *models.Schedule: + vd.Schedule = arg.(*models.Schedule) + case *models.NotificationAlert: + vd.Alert = arg.(*models.NotificationAlert) + case *models.Metric: + vd.Metric = arg.(*models.Metric) } } return vd } -func (svc *ServiceV2) parseTemplateVariables(template string) (variables []entity.NotificationVariable) { +func (svc *Service) parseTemplateVariables(template string) (variables []entity.NotificationVariable) { // regex pattern regex := regexp.MustCompile("\\$\\{(\\w+):(\\w+)}") @@ -358,26 +359,26 @@ func (svc *ServiceV2) parseTemplateVariables(template string) (variables []entit return variables } -func (svc *ServiceV2) getUsernameById(id primitive.ObjectID) (username string) { +func (svc *Service) getUsernameById(id primitive.ObjectID) (username string) { if id.IsZero() { return "" } - u, err := service.NewModelService[models.UserV2]().GetById(id) + u, err := service.NewModelService[models.User]().GetById(id) if err != nil { - log.Errorf("[NotificationServiceV2] get user error: %v", err) + log.Errorf("[NotificationService] get user error: %v", err) return "" } return u.Username } -func (svc *ServiceV2) getFormattedTime(t time.Time) (res string) { +func (svc *Service) getFormattedTime(t time.Time) (res string) { if t.IsZero() { return "N/A" } return t.Local().Format(time.DateTime) } -func (svc *ServiceV2) getFormattedTargetValue(a *models.NotificationAlertV2) (res string) { +func (svc *Service) getFormattedTargetValue(a *models.NotificationAlert) (res string) { if strings.HasSuffix(a.MetricName, "_percent") { return fmt.Sprintf("%.2f%%", a.TargetValue) } else if strings.HasSuffix(a.MetricName, "_memory") { @@ -391,7 +392,7 @@ func (svc *ServiceV2) getFormattedTargetValue(a *models.NotificationAlertV2) (re } } -func (svc *ServiceV2) getFormattedMetricValue(metricName string, m *models.MetricV2) (res string) { +func (svc *Service) getFormattedMetricValue(metricName string, m *models.Metric) (res string) { switch metricName { case "cpu_usage_percent": return fmt.Sprintf("%.2f%%", m.CpuUsagePercent) @@ -424,17 +425,17 @@ func (svc *ServiceV2) getFormattedMetricValue(metricName string, m *models.Metri } } -func (svc *ServiceV2) convertMarkdownToHtml(content string) (html string) { +func (svc *Service) convertMarkdownToHtml(content string) (html string) { return string(markdown.ToHTML([]byte(content), nil, nil)) } -func (svc *ServiceV2) SendNodeNotification(node *models.NodeV2) { +func (svc *Service) SendNodeNotification(node *models.Node) { // arguments var args []any args = append(args, node) // settings - settings, err := service.NewModelService[models.NotificationSettingV2]().GetMany(bson.M{ + settings, err := service.NewModelService[models.NotificationSetting]().GetMany(bson.M{ "enabled": true, "trigger": bson.M{ "$regex": constants.NotificationTriggerPatternNode, @@ -463,12 +464,12 @@ func (svc *ServiceV2) SendNodeNotification(node *models.NodeV2) { } } -func (svc *ServiceV2) createRequest(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) (res *models.NotificationRequestV2, err error) { +func (svc *Service) createRequest(s *models.NotificationSetting, ch *models.NotificationChannel, title, content string) (res *models.NotificationRequest, err error) { senderEmail := ch.SMTPUsername if s.UseCustomSenderEmail { senderEmail = s.SenderEmail } - r := models.NotificationRequestV2{ + r := models.NotificationRequest{ Status: StatusSending, SettingId: s.Id, ChannelId: ch.Id, @@ -482,15 +483,15 @@ func (svc *ServiceV2) createRequest(s *models.NotificationSettingV2, ch *models. } r.SetCreatedAt(time.Now()) r.SetUpdatedAt(time.Now()) - r.Id, err = service.NewModelService[models.NotificationRequestV2]().InsertOne(r) + r.Id, err = service.NewModelService[models.NotificationRequest]().InsertOne(r) if err != nil { - log.Errorf("[NotificationServiceV2] save request error: %v", err) + log.Errorf("[NotificationService] save request error: %v", err) return nil, err } return &r, nil } -func (svc *ServiceV2) saveRequest(r *models.NotificationRequestV2, err error) { +func (svc *Service) saveRequest(r *models.NotificationRequest, err error) { if r == nil { return } @@ -502,22 +503,22 @@ func (svc *ServiceV2) saveRequest(r *models.NotificationRequestV2, err error) { r.Status = StatusSuccess } r.SetUpdatedAt(time.Now()) - err = service.NewModelService[models.NotificationRequestV2]().ReplaceById(r.Id, *r) + err = service.NewModelService[models.NotificationRequest]().ReplaceById(r.Id, *r) if err != nil { - log.Errorf("[NotificationServiceV2] save request error: %v", err) + log.Errorf("[NotificationService] save request error: %v", err) } } -func newNotificationServiceV2() *ServiceV2 { - return &ServiceV2{} +func newNotificationService() *Service { + return &Service{} } -var _serviceV2 *ServiceV2 -var _serviceV2Once = new(sync.Once) +var _service *Service +var _serviceOnce sync.Once -func GetNotificationServiceV2() *ServiceV2 { - _serviceV2Once.Do(func() { - _serviceV2 = newNotificationServiceV2() +func GetNotificationService() *Service { + _serviceOnce.Do(func() { + _service = newNotificationService() }) - return _serviceV2 + return _service } diff --git a/core/notification/service_v2_test.go b/core/notification/service_test.go similarity index 96% rename from core/notification/service_v2_test.go rename to core/notification/service_test.go index a2ca0b28..139c2991 100644 --- a/core/notification/service_v2_test.go +++ b/core/notification/service_test.go @@ -8,7 +8,7 @@ import ( ) func TestParseTemplateVariables_WithValidTemplate_ReturnsVariables(t *testing.T) { - svc := ServiceV2{} + svc := Service{} template := "Dear ${user:name}, your task ${task:id} is ${task:status}." expected := []entity.NotificationVariable{ {Category: "user", Name: "name"}, @@ -23,7 +23,7 @@ func TestParseTemplateVariables_WithValidTemplate_ReturnsVariables(t *testing.T) } func TestParseTemplateVariables_WithRepeatedVariables_ReturnsUniqueVariables(t *testing.T) { - svc := ServiceV2{} + svc := Service{} template := "Dear ${user:name}, your task ${task:id} is ${task:status}. Again, ${user:name} and ${task:id}." expected := []entity.NotificationVariable{ {Category: "user", Name: "name"}, diff --git a/core/schedule/service_v2.go b/core/schedule/service.go similarity index 64% rename from core/schedule/service_v2.go rename to core/schedule/service.go index 34ea78e3..bea589e3 100644 --- a/core/schedule/service_v2.go +++ b/core/schedule/service.go @@ -2,9 +2,8 @@ package schedule import ( "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/utils" @@ -16,10 +15,9 @@ import ( "time" ) -type ServiceV2 struct { +type Service struct { // dependencies - interfaces.WithConfigPath - modelSvc *service.ModelService[models2.ScheduleV2] + modelSvc *service.ModelService[models.Schedule] adminSvc *admin.Service // settings variables @@ -31,63 +29,63 @@ type ServiceV2 struct { // internals cron *cron.Cron logger cron.Logger - schedules []models2.ScheduleV2 + schedules []models.Schedule stopped bool mu sync.Mutex } -func (svc *ServiceV2) GetLocation() (loc *time.Location) { +func (svc *Service) GetLocation() (loc *time.Location) { return svc.loc } -func (svc *ServiceV2) SetLocation(loc *time.Location) { +func (svc *Service) SetLocation(loc *time.Location) { svc.loc = loc } -func (svc *ServiceV2) GetDelay() (delay bool) { +func (svc *Service) GetDelay() (delay bool) { return svc.delay } -func (svc *ServiceV2) SetDelay(delay bool) { +func (svc *Service) SetDelay(delay bool) { svc.delay = delay } -func (svc *ServiceV2) GetSkip() (skip bool) { +func (svc *Service) GetSkip() (skip bool) { return svc.skip } -func (svc *ServiceV2) SetSkip(skip bool) { +func (svc *Service) SetSkip(skip bool) { svc.skip = skip } -func (svc *ServiceV2) GetUpdateInterval() (interval time.Duration) { +func (svc *Service) GetUpdateInterval() (interval time.Duration) { return svc.updateInterval } -func (svc *ServiceV2) SetUpdateInterval(interval time.Duration) { +func (svc *Service) SetUpdateInterval(interval time.Duration) { svc.updateInterval = interval } -func (svc *ServiceV2) Init() (err error) { +func (svc *Service) Init() (err error) { return svc.fetch() } -func (svc *ServiceV2) Start() { +func (svc *Service) Start() { svc.cron.Start() go svc.Update() } -func (svc *ServiceV2) Wait() { +func (svc *Service) Wait() { utils.DefaultWait() svc.Stop() } -func (svc *ServiceV2) Stop() { +func (svc *Service) Stop() { svc.stopped = true svc.cron.Stop() } -func (svc *ServiceV2) Enable(s models2.ScheduleV2, by primitive.ObjectID) (err error) { +func (svc *Service) Enable(s models.Schedule, by primitive.ObjectID) (err error) { svc.mu.Lock() defer svc.mu.Unlock() @@ -101,7 +99,7 @@ func (svc *ServiceV2) Enable(s models2.ScheduleV2, by primitive.ObjectID) (err e return svc.modelSvc.ReplaceById(s.Id, s) } -func (svc *ServiceV2) Disable(s models2.ScheduleV2, by primitive.ObjectID) (err error) { +func (svc *Service) Disable(s models.Schedule, by primitive.ObjectID) (err error) { svc.mu.Lock() defer svc.mu.Unlock() @@ -112,7 +110,7 @@ func (svc *ServiceV2) Disable(s models2.ScheduleV2, by primitive.ObjectID) (err return svc.modelSvc.ReplaceById(s.Id, s) } -func (svc *ServiceV2) Update() { +func (svc *Service) Update() { for { if svc.stopped { return @@ -124,11 +122,11 @@ func (svc *ServiceV2) Update() { } } -func (svc *ServiceV2) GetCron() (c *cron.Cron) { +func (svc *Service) GetCron() (c *cron.Cron) { return svc.cron } -func (svc *ServiceV2) update() { +func (svc *Service) update() { // fetch enabled schedules if err := svc.fetch(); err != nil { trace.PrintError(err) @@ -162,7 +160,7 @@ func (svc *ServiceV2) update() { } } -func (svc *ServiceV2) getEntryIdsMap() (res map[cron.EntryID]bool) { +func (svc *Service) getEntryIdsMap() (res map[cron.EntryID]bool) { res = map[cron.EntryID]bool{} for _, e := range svc.cron.Entries() { res[e.ID] = false @@ -170,7 +168,7 @@ func (svc *ServiceV2) getEntryIdsMap() (res map[cron.EntryID]bool) { return res } -func (svc *ServiceV2) fetch() (err error) { +func (svc *Service) fetch() (err error) { query := bson.M{ "enabled": true, } @@ -181,7 +179,7 @@ func (svc *ServiceV2) fetch() (err error) { return nil } -func (svc *ServiceV2) schedule(id primitive.ObjectID) (fn func()) { +func (svc *Service) schedule(id primitive.ObjectID) (fn func()) { return func() { // schedule s, err := svc.modelSvc.GetById(id) @@ -191,7 +189,7 @@ func (svc *ServiceV2) schedule(id primitive.ObjectID) (fn func()) { } // spider - spider, err := service.NewModelService[models2.SpiderV2]().GetById(s.SpiderId) + spider, err := service.NewModelService[models.Spider]().GetById(s.SpiderId) if err != nil { trace.PrintError(err) return @@ -236,21 +234,17 @@ func (svc *ServiceV2) schedule(id primitive.ObjectID) (fn func()) { } } -func NewScheduleServiceV2() (svc2 *ServiceV2, err error) { +func newScheduleService() *Service { // service - svc := &ServiceV2{ - WithConfigPath: config.NewConfigPathService(), - loc: time.Local, + svc := &Service{ + loc: time.Local, // TODO: implement delay and skip delay: false, skip: false, updateInterval: 1 * time.Minute, + adminSvc: admin.GetSpiderAdminService(), + modelSvc: service.NewModelService[models.Schedule](), } - svc.adminSvc, err = admin.GetSpiderAdminService() - if err != nil { - return nil, err - } - svc.modelSvc = service.NewModelService[models2.ScheduleV2]() // logger svc.logger = NewLogger() @@ -264,27 +258,19 @@ func NewScheduleServiceV2() (svc2 *ServiceV2, err error) { // initialize if err := svc.Init(); err != nil { - return nil, err + log.Fatalf("failed to initialize schedule service: %v", err) + panic(err) } - return svc, nil + return svc } -var svcV2 *ServiceV2 -var svcV2Once = new(sync.Once) +var _service *Service +var _serviceOnce sync.Once -func GetScheduleServiceV2() (res *ServiceV2, err error) { - if svcV2 != nil { - return svcV2, nil - } - svcV2Once.Do(func() { - svcV2, err = NewScheduleServiceV2() - if err != nil { - log.Errorf("failed to get schedule service: %v", err) - } +func GetScheduleService() *Service { + _serviceOnce.Do(func() { + _service = newScheduleService() }) - if err != nil { - return nil, err - } - return svcV2, nil + return _service } diff --git a/core/spider/admin/service.go b/core/spider/admin/service.go index 94d92d54..34c2a7c3 100644 --- a/core/spider/admin/service.go +++ b/core/spider/admin/service.go @@ -2,11 +2,9 @@ package admin import ( "errors" - log2 "github.com/apex/log" - config2 "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/scheduler" @@ -18,17 +16,12 @@ import ( type Service struct { // dependencies - nodeCfgSvc interfaces.NodeConfigService schedulerSvc *scheduler.Service - syncLock bool - - // settings - cfgPath string } func (svc *Service) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { // spider - s, err := service.NewModelService[models2.SpiderV2]().GetById(id) + s, err := service.NewModelService[models.Spider]().GetById(id) if err != nil { return nil, err } @@ -37,7 +30,7 @@ func (svc *Service) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRunOp return svc.scheduleTasks(s, opts) } -func (svc *Service) scheduleTasks(s *models2.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { +func (svc *Service) scheduleTasks(s *models.Spider, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { // get node ids nodeIds, err := svc.getNodeIds(opts) if err != nil { @@ -47,7 +40,7 @@ func (svc *Service) scheduleTasks(s *models2.SpiderV2, opts *interfaces.SpiderRu // iterate node ids for _, nodeId := range nodeIds { // task - t := &models2.TaskV2{ + t := &models.Task{ SpiderId: s.Id, NodeId: nodeId, NodeIds: opts.NodeIds, @@ -95,7 +88,7 @@ func (svc *Service) getNodeIds(opts *interfaces.SpiderRunOptions) (nodeIds []pri "enabled": true, "status": constants.NodeStatusOnline, } - nodes, err := service.NewModelService[models2.NodeV2]().GetMany(query, nil) + nodes, err := service.NewModelService[models.Node]().GetMany(query, nil) if err != nil { return nil, err } @@ -119,7 +112,7 @@ func (svc *Service) isMultiTask(opts *interfaces.SpiderRunOptions) (res bool) { "enabled": true, "status": constants.NodeStatusOnline, } - nodes, err := service.NewModelService[models2.NodeV2]().GetMany(query, nil) + nodes, err := service.NewModelService[models.Node]().GetMany(query, nil) if err != nil { trace.PrintError(err) return false @@ -134,39 +127,23 @@ func (svc *Service) isMultiTask(opts *interfaces.SpiderRunOptions) (res bool) { } } -func newSpiderAdminService() (svc2 *Service, err error) { - svc := &Service{ - nodeCfgSvc: config.GetNodeConfigService(), - cfgPath: config2.GetConfigPath(), - } - svc.schedulerSvc, err = scheduler.GetTaskSchedulerService() - if err != nil { - return nil, err - } - +func newSpiderAdminService() *Service { + nodeCfgSvc := config.GetNodeConfigService() // validate node type - if !svc.nodeCfgSvc.IsMaster() { - return nil, errors.New("only master node can run spider admin service") + if !nodeCfgSvc.IsMaster() { + panic("only master node can run spider admin service") + } + return &Service{ + schedulerSvc: scheduler.GetTaskSchedulerService(), } - - return svc, nil } -var svc *Service -var svcOnce = new(sync.Once) +var _service *Service +var _serviceOnce sync.Once -func GetSpiderAdminService() (svc2 *Service, err error) { - if svc != nil { - return svc, nil - } - svcOnce.Do(func() { - svc, err = newSpiderAdminService() - if err != nil { - log2.Errorf("[GetSpiderAdminService] error: %v", err) - } +func GetSpiderAdminService() *Service { + _serviceOnce.Do(func() { + _service = newSpiderAdminService() }) - if err != nil { - return nil, err - } - return svc, nil + return _service } diff --git a/core/stats/service.go b/core/stats/service.go index ae34ca69..92a967cb 100644 --- a/core/stats/service.go +++ b/core/stats/service.go @@ -4,7 +4,7 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/db/mongo" "go.mongodb.org/mongo-driver/bson" mongo2 "go.mongodb.org/mongo-driver/mongo" @@ -17,7 +17,7 @@ func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) stats := bson.M{} // nodes - stats["nodes"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.NodeV2{})).Count(bson.M{"active": true}) + stats["nodes"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Node{})).Count(bson.M{"active": true}) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -26,7 +26,7 @@ func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) } // projects - stats["projects"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.ProjectV2{})).Count(nil) + stats["projects"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Project{})).Count(nil) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -35,7 +35,7 @@ func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) } // spiders - stats["spiders"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.SpiderV2{})).Count(nil) + stats["spiders"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Spider{})).Count(nil) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -44,7 +44,7 @@ func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) } // schedules - stats["schedules"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.ScheduleV2{})).Count(nil) + stats["schedules"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Schedule{})).Count(nil) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -53,7 +53,7 @@ func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) } // tasks - stats["tasks"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskV2{})).Count(nil) + stats["tasks"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Task{})).Count(nil) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -62,7 +62,7 @@ func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) } // error tasks - stats["error_tasks"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskV2{})).Count(bson.M{"status": constants.TaskStatusError}) + stats["error_tasks"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Task{})).Count(bson.M{"status": constants.TaskStatusError}) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -80,7 +80,7 @@ func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) } // users - stats["users"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.UserV2{})).Count(nil) + stats["users"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.User{})).Count(nil) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -154,7 +154,7 @@ func (svc *Service) getDailyTasksStats(query bson.M) (data interface{}, err erro }}, } var results []entity.StatsDailyItem - if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskStatV2{})).Aggregate(pipeline, nil).All(&results); err != nil { + if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskStat{})).Aggregate(pipeline, nil).All(&results); err != nil { return nil, err } return results, nil @@ -172,7 +172,7 @@ func (svc *Service) getOverviewResults(query bson.M) (data interface{}, err erro }}, } var res bson.M - if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskStatV2{})).Aggregate(pipeline, nil).One(&res); err != nil { + if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskStat{})).Aggregate(pipeline, nil).One(&res); err != nil { return nil, err } return res["results"], nil @@ -197,7 +197,7 @@ func (svc *Service) getTaskStatsByStatus(query bson.M) (data interface{}, err er }}, } var results []bson.M - if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskV2{})).Aggregate(pipeline, nil).All(&results); err != nil { + if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Task{})).Aggregate(pipeline, nil).All(&results); err != nil { return nil, err } return results, nil @@ -216,7 +216,7 @@ func (svc *Service) getTaskStatsByNode(query bson.M) (data interface{}, err erro {{ "$lookup", bson.M{ - "from": models.GetCollectionNameByInstance(models.NodeV2{}), + "from": models.GetCollectionNameByInstance(models.Node{}), "localField": "_id", "foreignField": "_id", "as": "_n", @@ -233,7 +233,7 @@ func (svc *Service) getTaskStatsByNode(query bson.M) (data interface{}, err erro }}, } var results []bson.M - if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskV2{})).Aggregate(pipeline, nil).All(&results); err != nil { + if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Task{})).Aggregate(pipeline, nil).All(&results); err != nil { return nil, err } return results, nil @@ -252,7 +252,7 @@ func (svc *Service) getTaskStatsBySpider(query bson.M) (data interface{}, err er {{ "$lookup", bson.M{ - "from": models.GetCollectionNameByInstance(models.SpiderV2{}), + "from": models.GetCollectionNameByInstance(models.Spider{}), "localField": "_id", "foreignField": "_id", "as": "_s", @@ -270,7 +270,7 @@ func (svc *Service) getTaskStatsBySpider(query bson.M) (data interface{}, err er {{"$limit", 10}}, } var results []bson.M - if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskV2{})).Aggregate(pipeline, nil).All(&results); err != nil { + if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Task{})).Aggregate(pipeline, nil).All(&results); err != nil { return nil, err } return results, nil @@ -282,7 +282,7 @@ func (svc *Service) getTaskStatsHistogram(query bson.M) (data interface{}, err e {{ "$lookup", bson.M{ - "from": models.GetCollectionNameByInstance(models.TaskStatV2{}), + "from": models.GetCollectionNameByInstance(models.TaskStat{}), "localField": "_id", "foreignField": "_id", "as": "_ts", @@ -304,7 +304,7 @@ func (svc *Service) getTaskStatsHistogram(query bson.M) (data interface{}, err e }}, } var res bson.M - if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskV2{})).Aggregate(pipeline, nil).One(&res); err != nil { + if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Task{})).Aggregate(pipeline, nil).One(&res); err != nil { return nil, err } return res, nil diff --git a/core/system/service.go b/core/system/service.go new file mode 100644 index 00000000..24a49612 --- /dev/null +++ b/core/system/service.go @@ -0,0 +1,72 @@ +package system + +import ( + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "go.mongodb.org/mongo-driver/bson" + "sync" +) + +type Service struct { +} + +func (svc *Service) Init() (err error) { + // initialize data + if err := svc.initData(); err != nil { + return err + } + + return nil +} + +func (svc *Service) initData() (err error) { + total, err := service.NewModelService[models.Setting]().Count(bson.M{ + "key": "site_title", + }) + if err != nil { + return err + } + if total > 0 { + return nil + } + + // data to initialize + settings := []models.Setting{ + { + Key: "site_title", + Value: bson.M{ + "customize_site_title": false, + "site_title": "", + }, + }, + } + _, err = service.NewModelService[models.Setting]().InsertMany(settings) + if err != nil { + return err + } + return nil +} + +func newSystemService() *Service { + // service + svc := &Service{} + + if err := svc.Init(); err != nil { + panic(err) + } + + return svc +} + +var _service *Service +var _serviceOnce sync.Once + +func GetSystemService() *Service { + if _service == nil { + _service = newSystemService() + } + _serviceOnce.Do(func() { + _service = newSystemService() + }) + return _service +} diff --git a/core/system/service_v2.go b/core/system/service_v2.go deleted file mode 100644 index 59cfe01f..00000000 --- a/core/system/service_v2.go +++ /dev/null @@ -1,72 +0,0 @@ -package system - -import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" - "github.com/crawlab-team/crawlab/core/models/service" - "go.mongodb.org/mongo-driver/bson" - "sync" -) - -type ServiceV2 struct { -} - -func (svc *ServiceV2) Init() (err error) { - // initialize data - if err := svc.initData(); err != nil { - return err - } - - return nil -} - -func (svc *ServiceV2) initData() (err error) { - total, err := service.NewModelService[models.SettingV2]().Count(bson.M{ - "key": "site_title", - }) - if err != nil { - return err - } - if total > 0 { - return nil - } - - // data to initialize - settings := []models.SettingV2{ - { - Key: "site_title", - Value: bson.M{ - "customize_site_title": false, - "site_title": "", - }, - }, - } - _, err = service.NewModelService[models.SettingV2]().InsertMany(settings) - if err != nil { - return err - } - return nil -} - -func newSystemServiceV2() *ServiceV2 { - // service - svc := &ServiceV2{} - - if err := svc.Init(); err != nil { - panic(err) - } - - return svc -} - -var _serviceV2 *ServiceV2 -var _serviceV2Once = new(sync.Once) - -func GetSystemServiceV2() *ServiceV2 { - if _serviceV2 == nil { - _serviceV2 = newSystemServiceV2() - } - _serviceV2Once.Do(func() { - _serviceV2 = newSystemServiceV2() - }) - return _serviceV2 -} diff --git a/core/task/handler/runner.go b/core/task/handler/runner.go index 507566e6..545ae014 100644 --- a/core/task/handler/runner.go +++ b/core/task/handler/runner.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/crawlab-team/crawlab/core/models/models" "io" "net/http" "os" @@ -22,7 +23,6 @@ import ( client2 "github.com/crawlab-team/crawlab/core/grpc/client" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/sys_exec" "github.com/crawlab-team/crawlab/core/utils" @@ -47,8 +47,8 @@ type Runner struct { cmd *exec.Cmd // process command instance pid int // process id tid primitive.ObjectID // task id - t *models.TaskV2 // task model.Task - s *models.SpiderV2 // spider model.Spider + t *models.Task // task model.Task + s *models.Spider // spider model.Spider ch chan constants.TaskSignal // channel to communicate between Service and Runner err error // standard process error cwd string // working directory @@ -307,7 +307,7 @@ func (r *Runner) configureEnv() { } // Global environment variables - envs, err := client.NewModelService[models.EnvironmentV2]().GetMany(nil, nil) + envs, err := client.NewModelService[models.Environment]().GetMany(nil, nil) if err != nil { trace.PrintError(err) return @@ -502,12 +502,12 @@ func (r *Runner) updateTask(status string, e error) (err error) { r.t.Error = e.Error() } if r.svc.GetNodeConfigService().IsMaster() { - err = service.NewModelService[models.TaskV2]().ReplaceById(r.t.Id, *r.t) + err = service.NewModelService[models.Task]().ReplaceById(r.t.Id, *r.t) if err != nil { return err } } else { - err = client.NewModelService[models.TaskV2]().ReplaceById(r.t.Id, *r.t) + err = client.NewModelService[models.Task]().ReplaceById(r.t.Id, *r.t) if err != nil { return err } @@ -556,7 +556,7 @@ func (r *Runner) writeLogLines(lines []string) { } func (r *Runner) _updateTaskStat(status string) { - ts, err := client.NewModelService[models.TaskStatV2]().GetById(r.tid) + ts, err := client.NewModelService[models.TaskStat]().GetById(r.tid) if err != nil { trace.PrintError(err) return @@ -566,24 +566,24 @@ func (r *Runner) _updateTaskStat(status string) { // do nothing case constants.TaskStatusRunning: ts.StartTs = time.Now() - ts.WaitDuration = ts.StartTs.Sub(ts.BaseModelV2.CreatedAt).Milliseconds() + ts.WaitDuration = ts.StartTs.Sub(ts.CreatedAt).Milliseconds() case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: if ts.StartTs.IsZero() { ts.StartTs = time.Now() - ts.WaitDuration = ts.StartTs.Sub(ts.BaseModelV2.CreatedAt).Milliseconds() + ts.WaitDuration = ts.StartTs.Sub(ts.CreatedAt).Milliseconds() } ts.EndTs = time.Now() ts.RuntimeDuration = ts.EndTs.Sub(ts.StartTs).Milliseconds() - ts.TotalDuration = ts.EndTs.Sub(ts.BaseModelV2.CreatedAt).Milliseconds() + ts.TotalDuration = ts.EndTs.Sub(ts.CreatedAt).Milliseconds() } if r.svc.GetNodeConfigService().IsMaster() { - err = service.NewModelService[models.TaskStatV2]().ReplaceById(ts.Id, *ts) + err = service.NewModelService[models.TaskStat]().ReplaceById(ts.Id, *ts) if err != nil { trace.PrintError(err) return } } else { - err = client.NewModelService[models.TaskStatV2]().ReplaceById(ts.Id, *ts) + err = client.NewModelService[models.TaskStat]().ReplaceById(ts.Id, *ts) if err != nil { trace.PrintError(err) return @@ -606,7 +606,7 @@ func (r *Runner) sendNotification() { func (r *Runner) _updateSpiderStat(status string) { // task stat - ts, err := client.NewModelService[models.TaskStatV2]().GetById(r.tid) + ts, err := client.NewModelService[models.TaskStat]().GetById(r.tid) if err != nil { trace.PrintError(err) return @@ -644,13 +644,13 @@ func (r *Runner) _updateSpiderStat(status string) { // perform update if r.svc.GetNodeConfigService().IsMaster() { - err = service.NewModelService[models.SpiderStatV2]().UpdateById(r.s.Id, update) + err = service.NewModelService[models.SpiderStat]().UpdateById(r.s.Id, update) if err != nil { trace.PrintError(err) return } } else { - err = client.NewModelService[models.SpiderStatV2]().UpdateById(r.s.Id, update) + err = client.NewModelService[models.SpiderStat]().UpdateById(r.s.Id, update) if err != nil { trace.PrintError(err) return @@ -669,7 +669,7 @@ func (r *Runner) configureCwd() { } } -func NewTaskRunnerV2(id primitive.ObjectID, svc *Service) (r2 *Runner, err error) { +func newTaskRunner(id primitive.ObjectID, svc *Service) (r2 *Runner, err error) { // validate options if id.IsZero() { return nil, constants.ErrInvalidOptions @@ -698,7 +698,7 @@ func NewTaskRunnerV2(id primitive.ObjectID, svc *Service) (r2 *Runner, err error } // task fs service - r.fsSvc = fs.NewFsServiceV2(filepath.Join(viper.GetString("workspace"), r.s.Id.Hex())) + r.fsSvc = fs.NewFsService(filepath.Join(viper.GetString("workspace"), r.s.Id.Hex())) // grpc client r.c = client2.GetGrpcClient() diff --git a/core/task/handler/service.go b/core/task/handler/service.go index 788e0ad6..3438794c 100644 --- a/core/task/handler/service.go +++ b/core/task/handler/service.go @@ -10,7 +10,7 @@ import ( grpcclient "github.com/crawlab-team/crawlab/core/grpc/client" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/client" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/grpc" @@ -110,7 +110,7 @@ func (svc *Service) fetchAndRunTasks() { t.Error = err.Error() t.Status = constants.TaskStatusError t.SetUpdated(t.CreatedBy) - _ = client.NewModelService[models2.TaskV2]().ReplaceById(t.Id, *t) + _ = client.NewModelService[models.Task]().ReplaceById(t.Id, *t) continue } continue @@ -148,15 +148,15 @@ func (svc *Service) GetNodeConfigService() (cfgSvc interfaces.NodeConfigService) return svc.cfgSvc } -func (svc *Service) GetCurrentNode() (n *models2.NodeV2, err error) { +func (svc *Service) GetCurrentNode() (n *models.Node, err error) { // node key nodeKey := svc.cfgSvc.GetNodeKey() // current node if svc.cfgSvc.IsMaster() { - n, err = service.NewModelService[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + n, err = service.NewModelService[models.Node]().GetOne(bson.M{"key": nodeKey}, nil) } else { - n, err = client.NewModelService[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + n, err = client.NewModelService[models.Node]().GetOne(bson.M{"key": nodeKey}, nil) } if err != nil { return nil, err @@ -165,11 +165,11 @@ func (svc *Service) GetCurrentNode() (n *models2.NodeV2, err error) { return n, nil } -func (svc *Service) GetTaskById(id primitive.ObjectID) (t *models2.TaskV2, err error) { +func (svc *Service) GetTaskById(id primitive.ObjectID) (t *models.Task, err error) { if svc.cfgSvc.IsMaster() { - t, err = service.NewModelService[models2.TaskV2]().GetById(id) + t, err = service.NewModelService[models.Task]().GetById(id) } else { - t, err = client.NewModelService[models2.TaskV2]().GetById(id) + t, err = client.NewModelService[models.Task]().GetById(id) } if err != nil { return nil, err @@ -178,12 +178,12 @@ func (svc *Service) GetTaskById(id primitive.ObjectID) (t *models2.TaskV2, err e return t, nil } -func (svc *Service) UpdateTask(t *models2.TaskV2) (err error) { +func (svc *Service) UpdateTask(t *models.Task) (err error) { t.SetUpdated(t.CreatedBy) if svc.cfgSvc.IsMaster() { - err = service.NewModelService[models2.TaskV2]().ReplaceById(t.Id, *t) + err = service.NewModelService[models.Task]().ReplaceById(t.Id, *t) } else { - err = client.NewModelService[models2.TaskV2]().ReplaceById(t.Id, *t) + err = client.NewModelService[models.Task]().ReplaceById(t.Id, *t) } if err != nil { return err @@ -191,11 +191,11 @@ func (svc *Service) UpdateTask(t *models2.TaskV2) (err error) { return nil } -func (svc *Service) GetSpiderById(id primitive.ObjectID) (s *models2.SpiderV2, err error) { +func (svc *Service) GetSpiderById(id primitive.ObjectID) (s *models.Spider, err error) { if svc.cfgSvc.IsMaster() { - s, err = service.NewModelService[models2.SpiderV2]().GetById(id) + s, err = service.NewModelService[models.Spider]().GetById(id) } else { - s, err = client.NewModelService[models2.SpiderV2]().GetById(id) + s, err = client.NewModelService[models.Spider]().GetById(id) } if err != nil { return nil, err @@ -217,13 +217,13 @@ func (svc *Service) getRunnerCount() (count int) { }, } if svc.cfgSvc.IsMaster() { - count, err = service.NewModelService[models2.TaskV2]().Count(query) + count, err = service.NewModelService[models.Task]().Count(query) if err != nil { trace.PrintError(err) return } } else { - count, err = client.NewModelService[models2.TaskV2]().Count(query) + count, err = client.NewModelService[models.Task]().Count(query) if err != nil { trace.PrintError(err) return @@ -273,9 +273,9 @@ func (svc *Service) updateNodeStatus() (err error) { // save node n.SetUpdated(n.CreatedBy) if svc.cfgSvc.IsMaster() { - err = service.NewModelService[models2.NodeV2]().ReplaceById(n.Id, *n) + err = service.NewModelService[models.Node]().ReplaceById(n.Id, *n) } else { - err = client.NewModelService[models2.NodeV2]().ReplaceById(n.Id, *n) + err = client.NewModelService[models.Node]().ReplaceById(n.Id, *n) } if err != nil { return err @@ -311,7 +311,7 @@ func (svc *Service) runTask(taskId primitive.ObjectID) (err error) { } // create a new task runner - r, err := NewTaskRunnerV2(taskId, svc) + r, err := newTaskRunner(taskId, svc) if err != nil { err = fmt.Errorf("failed to create task runner: %v", err) log.Errorf("run task error: %v", err) @@ -439,7 +439,7 @@ func (svc *Service) cancelTask(taskId primitive.ObjectID, force bool) (err error return nil } -func newTaskHandlerService() (svc2 *Service, err error) { +func newTaskHandlerService() *Service { // service svc := &Service{ exitWatchDuration: 60 * time.Second, @@ -459,24 +459,15 @@ func newTaskHandlerService() (svc2 *Service, err error) { log.Debugf("[NewTaskHandlerService] svc[cfgPath: %s]", svc.cfgSvc.GetConfigPath()) - return svc, nil + return svc } -var _serviceV2 *Service -var _serviceV2Once = new(sync.Once) +var _service *Service +var _serviceOnce sync.Once -func GetTaskHandlerService() (svr *Service, err error) { - if _serviceV2 != nil { - return _serviceV2, nil - } - _serviceV2Once.Do(func() { - _serviceV2, err = newTaskHandlerService() - if err != nil { - log.Errorf("failed to create task handler service: %v", err) - } +func GetTaskHandlerService() *Service { + _serviceOnce.Do(func() { + _service = newTaskHandlerService() }) - if err != nil { - return nil, err - } - return _serviceV2, nil + return _service } diff --git a/core/task/log/driver.go b/core/task/log/driver.go index 139ec44d..1a81ced8 100644 --- a/core/task/log/driver.go +++ b/core/task/log/driver.go @@ -1,18 +1,16 @@ package log -func GetLogDriver(logDriverType string) (driver Driver, err error) { +import "fmt" + +func GetLogDriver(logDriverType string) Driver { switch logDriverType { case DriverTypeFile: - driver, err = GetFileLogDriver() - if err != nil { - return driver, err - } + return GetFileLogDriver() case DriverTypeMongo: - return driver, ErrNotImplemented + panic("mongo driver not implemented") case DriverTypeEs: - return driver, ErrNotImplemented + panic("es driver not implemented") default: - return driver, ErrInvalidType + panic(fmt.Sprintf("invalid log driver type: %s", logDriverType)) } - return driver, nil } diff --git a/core/task/log/file_driver.go b/core/task/log/file_driver.go index 852c0b21..7c41c214 100644 --- a/core/task/log/file_driver.go +++ b/core/task/log/file_driver.go @@ -26,10 +26,8 @@ type FileLogDriver struct { mu sync.Mutex } -func (d *FileLogDriver) Init() (err error) { +func (d *FileLogDriver) Init() { go d.cleanup() - - return nil } func (d *FileLogDriver) Close() (err error) { @@ -255,30 +253,25 @@ func (d *FileLogDriver) cleanup() { } } -var logDriver Driver - -func newFileLogDriver() (driver Driver, err error) { +func newFileLogDriver() Driver { // driver - driver = &FileLogDriver{ + driver := &FileLogDriver{ logFileName: "log.txt", mu: sync.Mutex{}, } // init - if err := driver.Init(); err != nil { - return nil, err - } + driver.Init() - return driver, nil + return driver } -func GetFileLogDriver() (driver Driver, err error) { - if logDriver != nil { - return logDriver, nil - } - logDriver, err = newFileLogDriver() - if err != nil { - return nil, err - } - return logDriver, nil +var logDriver Driver +var logDriverOnce sync.Once + +func GetFileLogDriver() Driver { + logDriverOnce.Do(func() { + logDriver = newFileLogDriver() + }) + return logDriver } diff --git a/core/task/log/interface.go b/core/task/log/interface.go index fd3745cb..aaf0f664 100644 --- a/core/task/log/interface.go +++ b/core/task/log/interface.go @@ -1,7 +1,7 @@ package log type Driver interface { - Init() (err error) + Init() Close() (err error) WriteLine(id string, line string) (err error) WriteLines(id string, lines []string) (err error) diff --git a/core/task/scheduler/service.go b/core/task/scheduler/service.go index f7cc5974..f1c12c5e 100644 --- a/core/task/scheduler/service.go +++ b/core/task/scheduler/service.go @@ -7,10 +7,8 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/grpc/server" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" - nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/handler" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/grpc" @@ -18,12 +16,12 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" + "sync" "time" ) type Service struct { // dependencies - nodeCfgSvc interfaces.NodeConfigService svr *server.GrpcServer handlerSvc *handler.Service @@ -37,27 +35,27 @@ func (svc *Service) Start() { utils.DefaultWait() } -func (svc *Service) Enqueue(t *models2.TaskV2, by primitive.ObjectID) (t2 *models2.TaskV2, err error) { +func (svc *Service) Enqueue(t *models.Task, by primitive.ObjectID) (t2 *models.Task, err error) { // set task status t.Status = constants.TaskStatusPending t.SetCreated(by) t.SetUpdated(by) // add task - taskModelSvc := service.NewModelService[models2.TaskV2]() + taskModelSvc := service.NewModelService[models.Task]() t.Id, err = taskModelSvc.InsertOne(*t) if err != nil { return nil, err } // task stat - ts := models2.TaskStatV2{} + ts := models.TaskStat{} ts.SetId(t.Id) ts.SetCreated(by) ts.SetUpdated(by) // add task stat - _, err = service.NewModelService[models2.TaskStatV2]().InsertOne(ts) + _, err = service.NewModelService[models.TaskStat]().InsertOne(ts) if err != nil { return nil, trace.TraceError(err) } @@ -68,7 +66,7 @@ func (svc *Service) Enqueue(t *models2.TaskV2, by primitive.ObjectID) (t2 *model func (svc *Service) Cancel(id, by primitive.ObjectID, force bool) (err error) { // task - t, err := service.NewModelService[models2.TaskV2]().GetById(id) + t, err := service.NewModelService[models.Task]().GetById(id) if err != nil { log.Errorf("task not found: %s", id.Hex()) return err @@ -101,7 +99,7 @@ func (svc *Service) Cancel(id, by primitive.ObjectID, force bool) (err error) { } } -func (svc *Service) cancelOnMaster(t *models2.TaskV2, by primitive.ObjectID, force bool) (err error) { +func (svc *Service) cancelOnMaster(t *models.Task, by primitive.ObjectID, force bool) (err error) { if err := svc.handlerSvc.Cancel(t.Id, force); err != nil { log.Errorf("failed to cancel task on master: %s", t.Id.Hex()) return err @@ -112,7 +110,7 @@ func (svc *Service) cancelOnMaster(t *models2.TaskV2, by primitive.ObjectID, for return svc.SaveTask(t, by) } -func (svc *Service) cancelOnWorker(t *models2.TaskV2, by primitive.ObjectID, force bool) (err error) { +func (svc *Service) cancelOnWorker(t *models.Task, by primitive.ObjectID, force bool) (err error) { // get subscribe stream stream, ok := svc.svr.TaskSvr.GetSubscribeStream(t.Id) if !ok { @@ -141,22 +139,22 @@ func (svc *Service) SetInterval(interval time.Duration) { svc.interval = interval } -func (svc *Service) SaveTask(t *models2.TaskV2, by primitive.ObjectID) (err error) { +func (svc *Service) SaveTask(t *models.Task, by primitive.ObjectID) (err error) { if t.Id.IsZero() { t.SetCreated(by) t.SetUpdated(by) - _, err = service.NewModelService[models2.TaskV2]().InsertOne(*t) + _, err = service.NewModelService[models.Task]().InsertOne(*t) return err } else { t.SetUpdated(by) - return service.NewModelService[models2.TaskV2]().ReplaceById(t.Id, *t) + return service.NewModelService[models.Task]().ReplaceById(t.Id, *t) } } // initTaskStatus initialize task status of existing tasks func (svc *Service) initTaskStatus() { // set status of running tasks as TaskStatusAbnormal - runningTasks, err := service.NewModelService[models2.TaskV2]().GetMany(bson.M{ + runningTasks, err := service.NewModelService[models.Task]().GetMany(bson.M{ "status": bson.M{ "$in": []string{ constants.TaskStatusPending, @@ -173,7 +171,7 @@ func (svc *Service) initTaskStatus() { return } for _, t := range runningTasks { - go func(t *models2.TaskV2) { + go func(t *models.Task) { t.Status = constants.TaskStatusAbnormal if err := svc.SaveTask(t, primitive.NilObjectID); err != nil { trace.PrintError(err) @@ -182,11 +180,11 @@ func (svc *Service) initTaskStatus() { } } -func (svc *Service) isMasterNode(t *models2.TaskV2) (ok bool, err error) { +func (svc *Service) isMasterNode(t *models.Task) (ok bool, err error) { if t.NodeId.IsZero() { return false, trace.TraceError(errors.ErrorTaskNoNodeId) } - n, err := service.NewModelService[models2.NodeV2]().GetById(t.NodeId) + n, err := service.NewModelService[models.Node]().GetById(t.NodeId) if err != nil { if errors2.Is(err, mongo2.ErrNoDocuments) { return false, trace.TraceError(errors.ErrorTaskNodeNotFound) @@ -199,7 +197,7 @@ func (svc *Service) isMasterNode(t *models2.TaskV2) (ok bool, err error) { func (svc *Service) cleanupTasks() { for { // task stats over 30 days ago - taskStats, err := service.NewModelService[models2.TaskStatV2]().GetMany(bson.M{ + taskStats, err := service.NewModelService[models.TaskStat]().GetMany(bson.M{ "created_ts": bson.M{ "$lt": time.Now().Add(-30 * 24 * time.Hour), }, @@ -217,14 +215,14 @@ func (svc *Service) cleanupTasks() { if len(ids) > 0 { // remove tasks - if err := service.NewModelService[models2.TaskV2]().DeleteMany(bson.M{ + if err := service.NewModelService[models.Task]().DeleteMany(bson.M{ "_id": bson.M{"$in": ids}, }); err != nil { trace.PrintError(err) } // remove task stats - if err := service.NewModelService[models2.TaskStatV2]().DeleteMany(bson.M{ + if err := service.NewModelService[models.TaskStat]().DeleteMany(bson.M{ "_id": bson.M{"$in": ids}, }); err != nil { trace.PrintError(err) @@ -235,35 +233,20 @@ func (svc *Service) cleanupTasks() { } } -func NewTaskSchedulerService() (svc2 *Service, err error) { - // service - svc := &Service{ - interval: 5 * time.Second, +func newTaskSchedulerService() *Service { + return &Service{ + interval: 5 * time.Second, + svr: server.GetGrpcServer(), + handlerSvc: handler.GetTaskHandlerService(), } - svc.nodeCfgSvc = nodeconfig.GetNodeConfigService() - svc.svr, err = server.GetGrpcServer() - if err != nil { - log.Errorf("failed to get grpc server: %v", err) - return nil, err - } - svc.handlerSvc, err = handler.GetTaskHandlerService() - if err != nil { - log.Errorf("failed to get task handler service: %v", err) - return nil, err - } - - return svc, nil } -var svc *Service +var _service *Service +var _serviceOnce sync.Once -func GetTaskSchedulerService() (svr *Service, err error) { - if svc != nil { - return svc, nil - } - svc, err = NewTaskSchedulerService() - if err != nil { - return nil, err - } - return svc, nil +func GetTaskSchedulerService() *Service { + _serviceOnce.Do(func() { + _service = newTaskSchedulerService() + }) + return _service } diff --git a/core/task/stats/service.go b/core/task/stats/service.go index 805ed81c..6130201f 100644 --- a/core/task/stats/service.go +++ b/core/task/stats/service.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/database" interfaces2 "github.com/crawlab-team/crawlab/core/database/interfaces" "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/log" @@ -98,13 +98,13 @@ func (svc *Service) getDatabaseServiceItem(taskId primitive.ObjectID) (item *dat } // task - t, err := service.NewModelService[models2.TaskV2]().GetById(taskId) + t, err := service.NewModelService[models.Task]().GetById(taskId) if err != nil { return nil, err } // spider - s, err := service.NewModelService[models2.SpiderV2]().GetById(t.SpiderId) + s, err := service.NewModelService[models.Spider]().GetById(t.SpiderId) if err != nil { return nil, err } @@ -137,7 +137,7 @@ func (svc *Service) getDatabaseServiceItem(taskId primitive.ObjectID) (item *dat } func (svc *Service) updateTaskStats(id primitive.ObjectID, resultCount int) { - err := service.NewModelService[models2.TaskStatV2]().UpdateById(id, bson.M{ + err := service.NewModelService[models.TaskStat]().UpdateById(id, bson.M{ "$inc": bson.M{ "result_count": resultCount, }, @@ -176,7 +176,7 @@ func (svc *Service) normalizeRecord(item *databaseServiceItem, record map[string return res } -func NewTaskStatsServiceV2() (svc2 *Service, err error) { +func NewTaskStatsService() *Service { // service svc := &Service{ mu: sync.Mutex{}, @@ -187,23 +187,17 @@ func NewTaskStatsServiceV2() (svc2 *Service, err error) { svc.nodeCfgSvc = nodeconfig.GetNodeConfigService() // log driver - svc.logDriver, err = log.GetLogDriver(log.DriverTypeFile) - if err != nil { - return nil, err - } + svc.logDriver = log.GetLogDriver(log.DriverTypeFile) - return svc, nil + return svc } -var _serviceV2 *Service +var _service *Service +var _serviceOnce sync.Once -func GetTaskStatsServiceV2() (svr *Service, err error) { - if _serviceV2 != nil { - return _serviceV2, nil - } - _serviceV2, err = NewTaskStatsServiceV2() - if err != nil { - return nil, err - } - return _serviceV2, nil +func GetTaskStatsService() *Service { + _serviceOnce.Do(func() { + _service = NewTaskStatsService() + }) + return _service } diff --git a/core/user/service_v2.go b/core/user/service.go similarity index 71% rename from core/user/service_v2.go rename to core/user/service.go index eef97051..97133b1b 100644 --- a/core/user/service_v2.go +++ b/core/user/service.go @@ -4,7 +4,7 @@ import ( "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" mongo2 "github.com/crawlab-team/crawlab/db/mongo" @@ -17,13 +17,13 @@ import ( "time" ) -type ServiceV2 struct { +type Service struct { jwtSecret string jwtSigningMethod jwt.SigningMethod - modelSvc *service.ModelService[models.UserV2] + modelSvc *service.ModelService[models.User] } -func (svc *ServiceV2) Init() (err error) { +func (svc *Service) Init() (err error) { _, err = svc.modelSvc.GetOne(bson.M{"username": constants.DefaultAdminUsername}, nil) if err == nil { return nil @@ -40,15 +40,15 @@ func (svc *ServiceV2) Init() (err error) { ) } -func (svc *ServiceV2) SetJwtSecret(secret string) { +func (svc *Service) SetJwtSecret(secret string) { svc.jwtSecret = secret } -func (svc *ServiceV2) SetJwtSigningMethod(method jwt.SigningMethod) { +func (svc *Service) SetJwtSigningMethod(method jwt.SigningMethod) { svc.jwtSigningMethod = method } -func (svc *ServiceV2) Create(username, password, role, email string, by primitive.ObjectID) (err error) { +func (svc *Service) Create(username, password, role, email string, by primitive.ObjectID) (err error) { // validate options if username == "" || password == "" { return trace.TraceError(errors.ErrorUserMissingRequiredFields) @@ -70,7 +70,7 @@ func (svc *ServiceV2) Create(username, password, role, email string, by primitiv // transaction return mongo2.RunTransaction(func(ctx mongo.SessionContext) error { // add user - u := models.UserV2{ + u := models.User{ Username: username, Role: role, Password: utils.EncryptMd5(password), @@ -84,7 +84,7 @@ func (svc *ServiceV2) Create(username, password, role, email string, by primitiv }) } -func (svc *ServiceV2) Login(username, password string) (token string, u *models.UserV2, err error) { +func (svc *Service) Login(username, password string) (token string, u *models.User, err error) { u, err = svc.modelSvc.GetOne(bson.M{"username": username}, nil) if err != nil { return "", nil, err @@ -99,11 +99,11 @@ func (svc *ServiceV2) Login(username, password string) (token string, u *models. return token, u, nil } -func (svc *ServiceV2) CheckToken(tokenStr string) (u *models.UserV2, err error) { +func (svc *Service) CheckToken(tokenStr string) (u *models.User, err error) { return svc.checkToken(tokenStr) } -func (svc *ServiceV2) ChangePassword(id primitive.ObjectID, password string, by primitive.ObjectID) (err error) { +func (svc *Service) ChangePassword(id primitive.ObjectID, password string, by primitive.ObjectID) (err error) { u, err := svc.modelSvc.GetById(id) if err != nil { return err @@ -113,11 +113,11 @@ func (svc *ServiceV2) ChangePassword(id primitive.ObjectID, password string, by return svc.modelSvc.ReplaceById(id, *u) } -func (svc *ServiceV2) MakeToken(user *models.UserV2) (tokenStr string, err error) { +func (svc *Service) MakeToken(user *models.User) (tokenStr string, err error) { return svc.makeToken(user) } -func (svc *ServiceV2) makeToken(user *models.UserV2) (tokenStr string, err error) { +func (svc *Service) makeToken(user *models.User) (tokenStr string, err error) { token := jwt.NewWithClaims(svc.jwtSigningMethod, jwt.MapClaims{ "id": user.Id, "username": user.Username, @@ -126,7 +126,7 @@ func (svc *ServiceV2) makeToken(user *models.UserV2) (tokenStr string, err error return token.SignedString([]byte(svc.jwtSecret)) } -func (svc *ServiceV2) checkToken(tokenStr string) (user *models.UserV2, err error) { +func (svc *Service) checkToken(tokenStr string) (user *models.User, err error) { token, err := jwt.Parse(tokenStr, svc.getSecretFunc()) if err != nil { return @@ -162,16 +162,16 @@ func (svc *ServiceV2) checkToken(tokenStr string) (user *models.UserV2, err erro return } -func (svc *ServiceV2) getSecretFunc() jwt.Keyfunc { +func (svc *Service) getSecretFunc() jwt.Keyfunc { return func(token *jwt.Token) (interface{}, error) { return []byte(svc.jwtSecret), nil } } -func newUserServiceV2() (svc *ServiceV2, err error) { +func newUserService() (svc *Service, err error) { // service - svc = &ServiceV2{ - modelSvc: service.NewModelService[models.UserV2](), + svc = &Service{ + modelSvc: service.NewModelService[models.User](), jwtSecret: "crawlab", jwtSigningMethod: jwt.SigningMethodHS256, } @@ -185,15 +185,15 @@ func newUserServiceV2() (svc *ServiceV2, err error) { return svc, nil } -var userSvcV2 *ServiceV2 -var userSvcV2Once sync.Once +var userSvc *Service +var userSvcOnce sync.Once -func GetUserServiceV2() (svc *ServiceV2, err error) { - userSvcV2Once.Do(func() { - userSvcV2, err = newUserServiceV2() +func GetUserService() (svc *Service, err error) { + userSvcOnce.Do(func() { + userSvc, err = newUserService() if err != nil { return } }) - return userSvcV2, nil + return userSvc, nil }