From d6badb533dfc46393225e5ddc349b767e39bdce9 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 11 Mar 2025 23:45:06 +0800 Subject: [PATCH 01/13] feat: enhance API routing and OpenAPI documentation support - Introduced a new OpenAPI wrapper using Fizz for improved API documentation - Refactored base controller to support more flexible route handling - Added dynamic route registration with OpenAPI metadata - Implemented generic response types for consistent API responses - Updated router initialization to support OpenAPI documentation endpoint - Improved route and resource naming utilities - Migrated existing controllers to use new routing and response mechanisms --- core/apps/api.go | 6 +- core/apps/server.go | 7 +- core/controllers/base.go | 197 +++++++++++++++++++++------------ core/controllers/http.go | 16 --- core/controllers/openapi.go | 16 +++ core/controllers/router.go | 183 +++++++++++++++++++++++++++--- core/controllers/spider.go | 4 +- core/controllers/task.go | 4 +- core/controllers/utils_http.go | 43 ++++++- core/go.mod | 9 +- core/go.sum | 56 ++++++++++ core/notification/im.go | 3 +- core/openapi/wrapper.go | 116 +++++++++++++++++++ go.work | 2 +- go.work.sum | 13 ++- 15 files changed, 556 insertions(+), 119 deletions(-) delete mode 100644 core/controllers/http.go create mode 100644 core/controllers/openapi.go create mode 100644 core/openapi/wrapper.go diff --git a/core/apps/api.go b/core/apps/api.go index 4f25e6b6..6016a528 100644 --- a/core/apps/api.go +++ b/core/apps/api.go @@ -50,7 +50,7 @@ func (app *Api) Start() { // http server app.srv = &http.Server{ - Handler: app.app, + Handler: app.GetHttpServerHandler(), Addr: address, } @@ -85,6 +85,10 @@ func (app *Api) Stop() { } } +func (app *Api) GetHttpServerHandler() http.Handler { + return controllers.GetGlobalFizzWrapper().GetFizz() +} + func (app *Api) GetGinEngine() *gin.Engine { return app.app } diff --git a/core/apps/server.go b/core/apps/server.go index 3724ebec..bac4b25a 100644 --- a/core/apps/server.go +++ b/core/apps/server.go @@ -2,13 +2,14 @@ package apps import ( "fmt" + "net/http" + _ "net/http/pprof" + "sync" + "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/node/service" "github.com/crawlab-team/crawlab/core/utils" "github.com/spf13/viper" - "net/http" - _ "net/http/pprof" - "sync" ) type Server struct { diff --git a/core/controllers/base.go b/core/controllers/base.go index d8c4dd05..3b06c103 100644 --- a/core/controllers/base.go +++ b/core/controllers/base.go @@ -1,11 +1,12 @@ package controllers import ( - "errors" + "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/mongo" "github.com/gin-gonic/gin" + "github.com/juju/errors" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" @@ -14,7 +15,7 @@ import ( type Action struct { Method string Path string - HandlerFunc gin.HandlerFunc + HandlerFunc interface{} } type BaseController[T any] struct { @@ -22,39 +23,68 @@ type BaseController[T any] struct { actions []Action } -func (ctr *BaseController[T]) GetById(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +type GetByIdParams struct { + Id string `path:"id" description:"The ID of the item to get"` +} + +// GetAllParams represents parameters for GetAll method +type GetAllParams struct { + Query bson.M `json:"query"` + Sort bson.D `json:"sort"` +} + +// GetListParams represents parameters for GetList with pagination +type GetListParams struct { + Query bson.M `json:"query"` + Sort bson.D `json:"sort"` + Pagination *entity.Pagination `json:"pagination"` + All bool `query:"all" description:"Whether to get all items"` +} + +func (ctr *BaseController[T]) GetById(_ *gin.Context, params *GetByIdParams) (response *Response[T], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return nil, errors.BadRequestf("invalid id: %s", params.Id) } model, err := ctr.modelSvc.GetById(id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return nil, err } - HandleSuccessWithData(c, model) + return GetSuccessDataResponse(*model) } -func (ctr *BaseController[T]) GetList(c *gin.Context) { +func (ctr *BaseController[T]) GetList(c *gin.Context, params *GetListParams) (response *ListResponse[T], err error) { // get all if query field "all" is set true - all := MustGetFilterAll(c) + all := params.All || MustGetFilterAll(c) + + // Prepare parameters + query := MustGetFilterQuery(c) + sort := MustGetSortOption(c) + if all { - ctr.getAll(c) - return + allParams := &GetAllParams{ + Query: query, + Sort: sort, + } + return ctr.GetAll(c, allParams) } - // get list - ctr.getList(c) + // get list with pagination + pagination := MustGetPagination(c) + listParams := &GetListParams{ + Query: query, + Sort: sort, + Pagination: pagination, + } + return ctr.GetWithPagination(c, listParams) } -func (ctr *BaseController[T]) Post(c *gin.Context) { +func (ctr *BaseController[T]) Post(c *gin.Context) (response *Response[T], err error) { var model T if err := c.ShouldBindJSON(&model); err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorDataResponse[T](err) } u := GetUserFromContext(c) m := any(&model).(interfaces.Model) @@ -64,30 +94,26 @@ func (ctr *BaseController[T]) Post(c *gin.Context) { col := ctr.modelSvc.GetCol() res, err := col.GetCollection().InsertOne(col.GetContext(), m) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorDataResponse[T](err) } result, err := ctr.modelSvc.GetById(res.InsertedID.(primitive.ObjectID)) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorDataResponse[T](err) } - HandleSuccessWithData(c, result) + return GetSuccessDataResponse(*result) } -func (ctr *BaseController[T]) PutById(c *gin.Context) { +func (ctr *BaseController[T]) PutById(c *gin.Context) (response *Response[T], err error) { id, err := primitive.ObjectIDFromHex(c.Param("id")) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorDataResponse[T](err) } var model T if err := c.ShouldBindJSON(&model); err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorDataResponse[T](err) } u := GetUserFromContext(c) @@ -95,20 +121,18 @@ func (ctr *BaseController[T]) PutById(c *gin.Context) { m.SetUpdated(u.Id) if err := ctr.modelSvc.ReplaceById(id, model); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorDataResponse[T](err) } result, err := ctr.modelSvc.GetById(id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorDataResponse[T](err) } - HandleSuccessWithData(c, result) + return GetSuccessDataResponse(*result) } -func (ctr *BaseController[T]) PatchList(c *gin.Context) { +func (ctr *BaseController[T]) PatchList(c *gin.Context) (res *Response[T], err error) { type Payload struct { Ids []primitive.ObjectID `json:"ids"` Update bson.M `json:"update"` @@ -116,8 +140,7 @@ func (ctr *BaseController[T]) PatchList(c *gin.Context) { var payload Payload if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorDataResponse[T](err) } // query @@ -129,45 +152,43 @@ func (ctr *BaseController[T]) PatchList(c *gin.Context) { // update if err := ctr.modelSvc.UpdateMany(query, bson.M{"$set": payload.Update}); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorDataResponse[T](err) } - HandleSuccess(c) + // Return an empty response with success status + var emptyModel T + return GetSuccessDataResponse(emptyModel) } -func (ctr *BaseController[T]) DeleteById(c *gin.Context) { +func (ctr *BaseController[T]) DeleteById(c *gin.Context) (res *Response[T], err error) { id, err := primitive.ObjectIDFromHex(c.Param("id")) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorDataResponse[T](err) } if err := ctr.modelSvc.DeleteById(id); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorDataResponse[T](err) } - HandleSuccess(c) + var emptyModel T + return GetSuccessDataResponse(emptyModel) } -func (ctr *BaseController[T]) DeleteList(c *gin.Context) { +func (ctr *BaseController[T]) DeleteList(c *gin.Context) (res *Response[T], err error) { type Payload struct { Ids []string `json:"ids"` } var payload Payload if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorDataResponse[T](err) } var ids []primitive.ObjectID for _, id := range payload.Ids { objectId, err := primitive.ObjectIDFromHex(id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorDataResponse[T](err) } ids = append(ids, objectId) } @@ -177,39 +198,46 @@ func (ctr *BaseController[T]) DeleteList(c *gin.Context) { "$in": ids, }, }); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorDataResponse[T](err) } - HandleSuccess(c) + var emptyModel T + return GetSuccessDataResponse(emptyModel) } -func (ctr *BaseController[T]) getAll(c *gin.Context) { - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) +// GetAll retrieves all items based on filter and sort +func (ctr *BaseController[T]) GetAll(_ *gin.Context, params *GetAllParams) (response *ListResponse[T], err error) { + query := params.Query + sort := params.Sort if sort == nil { sort = bson.D{{"_id", -1}} } + models, err := ctr.modelSvc.GetMany(query, &mongo.FindOptions{ Sort: sort, }) if err != nil { - HandleErrorInternalServerError(c, err) - return + return nil, err } + total, err := ctr.modelSvc.Count(query) if err != nil { - HandleErrorInternalServerError(c, err) - return + return nil, err } - HandleSuccessWithListData(c, models, total) + + return GetSuccessListResponse(models, total) } -func (ctr *BaseController[T]) getList(c *gin.Context) { +// GetWithPagination retrieves items with pagination +func (ctr *BaseController[T]) GetWithPagination(_ *gin.Context, params *GetListParams) (response *ListResponse[T], err error) { // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) + pagination := params.Pagination + query := params.Query + sort := params.Sort + + if pagination == nil { + pagination = GetDefaultPagination() + } // get list models, err := ctr.modelSvc.GetMany(query, &mongo.FindOptions{ @@ -219,22 +247,49 @@ func (ctr *BaseController[T]) getList(c *gin.Context) { }) if err != nil { if errors.Is(err, mongo2.ErrNoDocuments) { - HandleSuccessWithListData(c, nil, 0) + return GetSuccessListResponse[T](nil, 0) } else { - HandleErrorInternalServerError(c, err) + return nil, err } - return } // total count total, err := ctr.modelSvc.Count(query) if err != nil { - HandleErrorInternalServerError(c, err) - return + return nil, err } // response - HandleSuccessWithListData(c, models, total) + return GetSuccessListResponse(models, total) +} + +// getAll is kept for backward compatibility +func (ctr *BaseController[T]) getAll(c *gin.Context) (response *ListResponse[T], err error) { + query := MustGetFilterQuery(c) + sort := MustGetSortOption(c) + + params := &GetAllParams{ + Query: query, + Sort: sort, + } + + return ctr.GetAll(c, params) +} + +// getList is kept for backward compatibility +func (ctr *BaseController[T]) getList(c *gin.Context) (response *ListResponse[T], err error) { + // params + pagination := MustGetPagination(c) + query := MustGetFilterQuery(c) + sort := MustGetSortOption(c) + + params := &GetListParams{ + Query: query, + Sort: sort, + Pagination: pagination, + } + + return ctr.GetWithPagination(c, params) } func NewController[T any](actions ...Action) *BaseController[T] { diff --git a/core/controllers/http.go b/core/controllers/http.go deleted file mode 100644 index 8aa5169b..00000000 --- a/core/controllers/http.go +++ /dev/null @@ -1,16 +0,0 @@ -package controllers - -type Response[T any] struct { - Status string `json:"status"` - Message string `json:"message"` - Data T `json:"data"` - Error string `json:"error"` -} - -type ListResponse[T any] struct { - Status string `json:"status"` - Message string `json:"message"` - Total int `json:"total"` - Data []T `json:"data"` - Error string `json:"error"` -} diff --git a/core/controllers/openapi.go b/core/controllers/openapi.go new file mode 100644 index 00000000..d6996b7c --- /dev/null +++ b/core/controllers/openapi.go @@ -0,0 +1,16 @@ +package controllers + +import ( + "github.com/gin-gonic/gin" + "github.com/wI2L/fizz/openapi" +) + +func GetOpenAPI(c *gin.Context) { + info := &openapi.Info{ + Title: "Crawlab API", + Description: "REST API for Crawlab", + Version: "0.7.0", + } + handleFunc := globalWrapper.GetFizz().OpenAPI(info, "json") + handleFunc(c) +} diff --git a/core/controllers/router.go b/core/controllers/router.go index 42510dc1..9fa346fd 100644 --- a/core/controllers/router.go +++ b/core/controllers/router.go @@ -2,25 +2,40 @@ package controllers import ( "net/http" + "strings" "github.com/crawlab-team/crawlab/core/middlewares" "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/openapi" "github.com/gin-gonic/gin" ) // RouterGroups defines the different authentication levels for API routes type RouterGroups struct { - AuthGroup *gin.RouterGroup // Routes requiring full authentication - SyncAuthGroup *gin.RouterGroup // Routes for sync operations with special auth - AnonymousGroup *gin.RouterGroup // Public routes that don't require auth + AuthGroup *gin.RouterGroup // Routes requiring full authentication + SyncAuthGroup *gin.RouterGroup // Routes for sync operations with special auth + AnonymousGroup *gin.RouterGroup // Public routes that don't require auth + Wrapper *openapi.FizzWrapper // OpenAPI wrapper for documentation +} + +// Global variable to store the OpenAPI wrapper +// This is a workaround since we can't easily pass it through the Gin context +var globalWrapper *openapi.FizzWrapper + +func GetGlobalFizzWrapper() *openapi.FizzWrapper { + return globalWrapper } // NewRouterGroups initializes the router groups with their respective middleware func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) { + // Create OpenAPI wrapper + wrapper := openapi.NewFizzWrapper(app) + return &RouterGroups{ AuthGroup: app.Group("/", middlewares.AuthorizationMiddleware()), SyncAuthGroup: app.Group("/", middlewares.SyncAuthorizationMiddleware()), AnonymousGroup: app.Group("/"), + Wrapper: wrapper, } } @@ -30,36 +45,170 @@ func RegisterController[T any](group *gin.RouterGroup, basePath string, ctr *Bas // Track registered paths to avoid duplicates actionPaths := make(map[string]bool) for _, action := range ctr.actions { - group.Handle(action.Method, basePath+action.Path, action.HandlerFunc) path := basePath + action.Path key := action.Method + " - " + path actionPaths[key] = true + + // Create appropriate model response based on the action + responses := globalWrapper.BuildModelResponse() + + id := getIDForAction(action.Method, path) + summary := getSummaryForAction(action.Method, basePath, action.Path) + description := getDescriptionForAction(action.Method, basePath, action.Path) + + globalWrapper.RegisterRoute(action.Method, path, action.HandlerFunc, id, summary, description, responses) } - registerBuiltinHandler(group, http.MethodGet, basePath+"", ctr.GetList, actionPaths) - registerBuiltinHandler(group, http.MethodGet, basePath+"/:id", ctr.GetById, actionPaths) - registerBuiltinHandler(group, http.MethodPost, basePath+"", ctr.Post, actionPaths) - registerBuiltinHandler(group, http.MethodPut, basePath+"/:id", ctr.PutById, actionPaths) - registerBuiltinHandler(group, http.MethodPatch, basePath+"", ctr.PatchList, actionPaths) - registerBuiltinHandler(group, http.MethodDelete, basePath+"/:id", ctr.DeleteById, actionPaths) - registerBuiltinHandler(group, http.MethodDelete, basePath+"", ctr.DeleteList, actionPaths) + + // Register built-in handlers if they haven't been overridden + // Create a zero value of T to use as the model + var model T + registerBuiltinHandler(group, globalWrapper, basePath, http.MethodGet, "", ctr.GetList, actionPaths, "Get list", "Get a list of items", model) + registerBuiltinHandler(group, globalWrapper, basePath, http.MethodGet, "/:id", ctr.GetById, actionPaths, "Get by ID", "Get a single item by ID", model) + registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPost, "", ctr.Post, actionPaths, "Create", "Create a new item", model) + registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPut, "/:id", ctr.PutById, actionPaths, "Update by ID", "Update an item by ID", model) + registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPatch, "", ctr.PatchList, actionPaths, "Patch list", "Patch multiple items", model) + registerBuiltinHandler(group, globalWrapper, basePath, http.MethodDelete, "/:id", ctr.DeleteById, actionPaths, "Delete by ID", "Delete an item by ID", model) + registerBuiltinHandler(group, globalWrapper, basePath, http.MethodDelete, "", ctr.DeleteList, actionPaths, "Delete list", "Delete multiple items", model) } // RegisterActions registers a list of custom action handlers to a route group func RegisterActions(group *gin.RouterGroup, basePath string, actions []Action) { for _, action := range actions { - group.Handle(action.Method, basePath+action.Path, action.HandlerFunc) + path := basePath + action.Path + + // Create generic response + responses := globalWrapper.BuildModelResponse() + + id := getIDForAction(action.Method, path) + summary := getSummaryForAction(action.Method, basePath, action.Path) + description := getDescriptionForAction(action.Method, basePath, action.Path) + + globalWrapper.RegisterRoute(action.Method, path, action.HandlerFunc, id, summary, description, responses) } } // registerBuiltinHandler registers a standard handler if it hasn't been overridden // by a custom action -func registerBuiltinHandler(group *gin.RouterGroup, method, path string, handlerFunc gin.HandlerFunc, existingActionPaths map[string]bool) { +func registerBuiltinHandler[T any](group *gin.RouterGroup, wrapper *openapi.FizzWrapper, basePath, method, pathSuffix string, handlerFunc interface{}, existingActionPaths map[string]bool, summary, description string, model T) { + path := basePath + pathSuffix key := method + " - " + path _, ok := existingActionPaths[key] if ok { return } - group.Handle(method, path, handlerFunc) + + id := getIDForAction(method, path) + + // Create appropriate response based on the method + responses := wrapper.BuildModelResponse() + + wrapper.RegisterRoute(method, path, handlerFunc, id, summary, description, responses) +} + +// Helper functions to generate OpenAPI documentation +func getIDForAction(method, path string) string { + // Remove leading slash and convert remaining slashes to underscores + cleanPath := strings.TrimPrefix(path, "/") + cleanPath = strings.ReplaceAll(cleanPath, "/", "_") + cleanPath = strings.ReplaceAll(cleanPath, ":", "_") + cleanPath = strings.ReplaceAll(cleanPath, "__", "_") + + return method + "_" + cleanPath +} + +func getSummaryForAction(method, basePath, path string) string { + resource := getResourceName(basePath) + + switch method { + case http.MethodGet: + if path == "" { + return "List " + resource + } else if path == "/:id" { + return "Get " + resource + " by ID" + } + case http.MethodPost: + if path == "" { + return "Create " + resource + } + case http.MethodPut: + if path == "/:id" { + return "Update " + resource + " by ID" + } + case http.MethodPatch: + if path == "" { + return "Patch " + resource + " list" + } + case http.MethodDelete: + if path == "/:id" { + return "Delete " + resource + " by ID" + } else if path == "" { + return "Delete " + resource + " list" + } + } + + // For custom actions, use a more descriptive summary + if path != "" && path != "/:id" { + return method + " " + resource + path + } + + return method + " " + resource +} + +func getDescriptionForAction(method, basePath, path string) string { + resource := getResourceName(basePath) + + switch method { + case http.MethodGet: + if path == "" { + return "Get a list of " + resource + " items" + } else if path == "/:id" { + return "Get a single " + resource + " by ID" + } + case http.MethodPost: + if path == "" { + return "Create a new " + resource + } + case http.MethodPut: + if path == "/:id" { + return "Update a " + resource + " by ID" + } + case http.MethodPatch: + if path == "" { + return "Patch multiple " + resource + " items" + } + case http.MethodDelete: + if path == "/:id" { + return "Delete a " + resource + " by ID" + } else if path == "" { + return "Delete multiple " + resource + " items" + } + } + + // For custom actions, use a more descriptive description + if path != "" && path != "/:id" { + return "Perform " + method + " operation on " + resource + " with path " + path + } + + return "Perform " + method + " operation on " + resource +} + +func getResourceName(basePath string) string { + // Remove leading slash and get the last part of the path + if len(basePath) > 0 && basePath[0] == '/' { + basePath = basePath[1:] + } + + // Remove trailing slash if present + if len(basePath) > 0 && basePath[len(basePath)-1] == '/' { + basePath = basePath[:len(basePath)-1] + } + + // If path is empty, return "resource" + if basePath == "" { + return "resource" + } + + return basePath } // InitRoutes configures all API routes for the application @@ -67,6 +216,9 @@ func InitRoutes(app *gin.Engine) (err error) { // Initialize route groups with different auth levels groups := NewRouterGroups(app) + // Store the wrapper in the global variable for later use + globalWrapper = groups.Wrapper + // Register resource controllers with their respective endpoints // Each RegisterController call sets up standard CRUD operations // Additional custom actions can be specified in the controller initialization @@ -418,5 +570,8 @@ func InitRoutes(app *gin.Engine) (err error) { }, }) + // Register OpenAPI documentation route + groups.AnonymousGroup.GET("/openapi.json", GetOpenAPI) + return nil } diff --git a/core/controllers/spider.go b/core/controllers/spider.go index 7392a47e..07aeb47c 100644 --- a/core/controllers/spider.go +++ b/core/controllers/spider.go @@ -74,7 +74,7 @@ func GetSpiderById(c *gin.Context) { HandleSuccessWithData(c, s) } -func GetSpiderList(c *gin.Context) { +func GetSpiderList(c *gin.Context, params *GetListParams) { // get all list all := MustGetFilterAll(c) if all { @@ -85,7 +85,7 @@ func GetSpiderList(c *gin.Context) { // get list withStats := c.Query("stats") if withStats == "" { - NewController[models.Spider]().GetList(c) + NewController[models.Spider]().GetList(c, params) return } diff --git a/core/controllers/task.go b/core/controllers/task.go index af110dbe..e341841f 100644 --- a/core/controllers/task.go +++ b/core/controllers/task.go @@ -67,10 +67,10 @@ func GetTaskById(c *gin.Context) { HandleSuccessWithData(c, t) } -func GetTaskList(c *gin.Context) { +func GetTaskList(c *gin.Context, params *GetListParams) { withStats := c.Query("stats") if withStats == "" { - NewController[models.Task]().GetList(c) + NewController[models.Task]().GetList(c, params) return } diff --git a/core/controllers/utils_http.go b/core/controllers/utils_http.go index 0d50231d..54ac2987 100644 --- a/core/controllers/utils_http.go +++ b/core/controllers/utils_http.go @@ -1,14 +1,55 @@ package controllers import ( + "net/http" + "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/trace" "github.com/gin-gonic/gin" - "net/http" ) +type Response[T any] struct { + Status string `json:"status"` + Message string `json:"message"` + Data T `json:"data"` + Error string `json:"error"` +} + +type ListResponse[T any] struct { + Status string `json:"status"` + Message string `json:"message"` + Total int `json:"total"` + Data []T `json:"data"` + Error string `json:"error"` +} + +func GetSuccessDataResponse[T any](model T) (res *Response[T], err error) { + return &Response[T]{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageSuccess, + Data: model, + }, nil +} + +func GetSuccessListResponse[T any](models []T, total int) (res *ListResponse[T], err error) { + return &ListResponse[T]{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageSuccess, + Data: models, + Total: total, + }, nil +} + +func GetErrorDataResponse[T any](err error) (res *Response[T], err2 error) { + return &Response[T]{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageError, + Error: err.Error(), + }, err +} + func handleError(statusCode int, c *gin.Context, err error) { if utils.IsDev() { trace.PrintError(err) diff --git a/core/go.mod b/core/go.mod index ddffb649..c44dee01 100644 --- a/core/go.mod +++ b/core/go.mod @@ -1,6 +1,6 @@ module github.com/crawlab-team/crawlab/core -go 1.22.9 +go 1.23 replace ( github.com/crawlab-team/crawlab/grpc => ../grpc @@ -26,6 +26,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-uuid v1.0.3 + github.com/juju/errors v1.0.0 github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.0 @@ -33,8 +34,10 @@ require ( github.com/spf13/cobra v1.3.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 + github.com/wI2L/fizz v0.22.0 go.mongodb.org/mongo-driver v1.15.1 golang.org/x/oauth2 v0.23.0 + golang.org/x/text v0.21.0 google.golang.org/api v0.189.0 google.golang.org/grpc v1.69.2 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df @@ -72,6 +75,7 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/gofrs/uuid v3.2.0+incompatible // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/s2a-go v0.1.8 // indirect @@ -86,6 +90,7 @@ require ( github.com/klauspost/compress v1.17.7 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect + github.com/loopfz/gadgeto v0.9.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -128,10 +133,10 @@ require ( golang.org/x/net v0.33.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 // indirect google.golang.org/protobuf v1.36.1 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect + gopkg.in/go-playground/validator.v9 v9.30.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/core/go.sum b/core/go.sum index bbea437e..0c7649e5 100644 --- a/core/go.sum +++ b/core/go.sum @@ -61,6 +61,8 @@ github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= +github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= @@ -134,6 +136,7 @@ github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -169,8 +172,14 @@ github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyT github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/cors v1.3.0/go.mod h1:artPvLlhkF7oG06nK8v3U8TNz6IeX+w1uzCSEId5/Vc= +github.com/gin-contrib/sse v0.0.0-20190125020943-a7658810eb74/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= @@ -197,18 +206,29 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= @@ -290,6 +310,7 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -361,6 +382,9 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= @@ -377,14 +401,21 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/loopfz/gadgeto v0.9.0 h1:yrQVBgdGhAWOB+JjH98sJzYfSqpcpKzOBIEtJFcRl2s= +github.com/loopfz/gadgeto v0.9.0/go.mod h1:S3tK5SXmKY3l39rUpPZw1B/iiy1CftV13QABFhj32Ss= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -397,7 +428,9 @@ github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= @@ -440,6 +473,7 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -467,6 +501,8 @@ github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -542,8 +578,17 @@ github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9f github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0= +github.com/ugorji/go/codec v0.0.0-20190128213124-ee1426cffec0/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/wI2L/fizz v0.22.0 h1:mgRA+uUdESvgsIeBFkMSS/MEIQ4EZ4I2xyRxnCqkhJY= +github.com/wI2L/fizz v0.22.0/go.mod h1:CMxMR1amz8id9wr2YUpONf+F/F9hW1cqRXxVNNuWVxE= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -612,8 +657,10 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= @@ -811,6 +858,7 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1074,11 +1122,18 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/go-playground/validator.v9 v9.26.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/go-playground/validator.v9 v9.30.0 h1:Wk0Z37oBmKj9/n+tPyBHZmeL19LaCoK3Qq48VwYENss= +gopkg.in/go-playground/validator.v9 v9.30.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= @@ -1088,6 +1143,7 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/core/notification/im.go b/core/notification/im.go index 48020a9a..ea9003bf 100644 --- a/core/notification/im.go +++ b/core/notification/im.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/utils" "io" "net/http" @@ -23,7 +22,7 @@ type ResBody struct { } // RequestParam represents parameters for HTTP requests -type RequestParam entity.RequestParam +type RequestParam map[string]interface{} // performRequest performs an HTTP request with JSON body func performRequest(method, url string, data interface{}) (*http.Response, []byte, error) { diff --git a/core/openapi/wrapper.go b/core/openapi/wrapper.go new file mode 100644 index 00000000..e8259ba9 --- /dev/null +++ b/core/openapi/wrapper.go @@ -0,0 +1,116 @@ +package openapi + +import ( + "fmt" + + "github.com/loopfz/gadgeto/tonic" + + "github.com/gin-gonic/gin" + "github.com/wI2L/fizz" +) + +// FizzWrapper wraps an existing Gin Engine to add OpenAPI functionality +type FizzWrapper struct { + fizz *fizz.Fizz + gin *gin.Engine +} + +// NewFizzWrapper creates a new wrapper around an existing Gin Engine +// This approach ensures we don't break existing functionality +func NewFizzWrapper(engine *gin.Engine) *FizzWrapper { + // Create a new Fizz instance using the existing Gin engine + f := fizz.NewFromEngine(engine) + + return &FizzWrapper{ + fizz: f, + gin: engine, + } +} + +// GetFizz returns the underlying Fizz instance +func (w *FizzWrapper) GetFizz() *fizz.Fizz { + return w.fizz +} + +// GetGin returns the underlying Gin engine +func (w *FizzWrapper) GetGin() *gin.Engine { + return w.gin +} + +// Response represents an OpenAPI response +type Response struct { + Description string + Model interface{} +} + +// RegisterRoute registers a route with OpenAPI documentation +func (w *FizzWrapper) RegisterRoute(method, path string, handler interface{}, id, summary, description string, responses map[int]Response) { + // Build operation options for OpenAPI documentation + opts := w.buildOperationOptions(id, summary, description, responses) + + // Register the route with OpenAPI documentation + switch method { + case "GET": + w.fizz.GET(path, opts, tonic.Handler(handler, 200)) + case "POST": + w.fizz.POST(path, opts, tonic.Handler(handler, 200)) + case "PUT": + w.fizz.PUT(path, opts, tonic.Handler(handler, 200)) + case "DELETE": + w.fizz.DELETE(path, opts, tonic.Handler(handler, 200)) + case "PATCH": + w.fizz.PATCH(path, opts, tonic.Handler(handler, 200)) + case "HEAD": + w.fizz.HEAD(path, opts, tonic.Handler(handler, 200)) + case "OPTIONS": + w.fizz.OPTIONS(path, opts, tonic.Handler(handler, 200)) + } +} + +// BuildModelResponse builds a standard response model with a specific data type +func (w *FizzWrapper) BuildModelResponse() map[int]Response { + return map[int]Response{ + 400: { + Description: "Bad Request", + }, + 401: { + Description: "Unauthorized", + }, + 500: { + Description: "Internal Server Error", + }, + } +} + +// buildOperationOptions builds the options for a Fizz operation +func (w *FizzWrapper) buildOperationOptions(id, summary, description string, responses map[int]Response) []fizz.OperationOption { + var opts []fizz.OperationOption + + // Add ID + if id != "" { + opts = append(opts, fizz.ID(id)) + } + + // Add summary + if summary != "" { + opts = append(opts, fizz.Summary(summary)) + } + + // Add description + if description != "" { + opts = append(opts, fizz.Description(description)) + } + + // Add responses + if responses != nil { + for status, response := range responses { + if response.Model != nil { + opts = append(opts, fizz.Response(fmt.Sprintf("%d", status), response.Description, response.Model, nil, nil)) + } else { + opts = append(opts, fizz.Response(fmt.Sprintf("%d", status), response.Description, nil, nil, nil)) + } + } + } + + return opts +} diff --git a/go.work b/go.work index 91441d84..ea0e5963 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.22.9 +go 1.23 use ( backend diff --git a/go.work.sum b/go.work.sum index 12ce343b..001314c3 100644 --- a/go.work.sum +++ b/go.work.sum @@ -821,7 +821,6 @@ github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614095218-7b4ee8399a github.com/crawlab-team/crawlab/template-parser v0.0.0-20241016121324-eb56598d93dc/go.mod h1:lKaowLGJrCwvFJnW/KYbdz0Pj69My34yus6IsLKDMro= github.com/creack/pty v1.1.7 h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A= github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -844,6 +843,7 @@ github.com/fasthttp/websocket v1.4.3-rc.6/go.mod h1:43W9OM2T8FeXpCWMsBd9Cb7nE2CA github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/gin-contrib/cors v1.3.0 h1:PolezCc89peu+NgkIWt9OB01Kbzt6IP0J/JvkG6xxlg= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df h1:Bao6dhmbTA1KFVxmJ6nBoMuOJit2yjEgLJpIMYpop0E= @@ -972,6 +972,11 @@ github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhB github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9 h1:hJix6idebFclqlfZCHE7EUX7uqLCyb70nHNHH1XKGBg= +github.com/juju/errors v1.0.0 h1:yiq7kjCLll1BiaRuNY53MGI0+EQ3rF6GB+wvboZDefM= +github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5Qe8= +github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 h1:UUHMLvzt/31azWTN/ifGWef4WUqvXk0iRqdhdy/2uzI= +github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2 h1:Pp8RxiF4rSoXP9SED26WCfNB28/dwTDpPXS8XMJR8rc= github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= @@ -982,7 +987,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGi github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI= github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M= @@ -1027,7 +1031,6 @@ github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0Mw github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= @@ -1090,6 +1093,7 @@ github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b h1:m74UWYy+HBs+jMFR9 github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8= +github.com/ugorji/go v1.2.6 h1:tGiWC9HENWE2tqYycIqFTNorMmFRVhNwCpDOpWqnk8E= github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= @@ -1374,10 +1378,11 @@ google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec h1:RlWgLqCMMIYYEVcAR5MDsuHlVkaIPDAF+5Dehzg8L5A= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= modernc.org/b v1.0.2 h1:iPC2u39ebzq12GOC2yXT4mve0HrWcH85cz+midWjzeo= modernc.org/db v1.0.3 h1:apxOlWU69je04bY22OT6J0RL23mzvUy22EgTAVyw+Yg= From ddff881954fd2a5a895cd0e1e49c47d4493d82be Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 12 Mar 2025 23:20:06 +0800 Subject: [PATCH 02/13] chore: update Go version and dependencies - Updated Go version in go.work and backend/go.mod to 1.23.7 - Updated various dependencies in go.sum and backend/go.sum - Refactored models to remove generic type parameters from BaseModel - Introduced new utility functions for consistent API responses - Removed unused utility files from controllers --- backend/go.mod | 9 +- backend/go.sum | 55 +++ core/controllers/base.go | 242 +++++------- core/controllers/base_file.go | 34 +- core/controllers/base_test.go | 369 +++++++++++++++++- core/controllers/project.go | 27 +- core/controllers/router.go | 60 +-- core/controllers/router_test.go | 2 +- core/controllers/spider.go | 237 +++++------ core/controllers/spider_test.go | 29 +- core/controllers/user.go | 274 ++++++------- core/controllers/user_test.go | 63 +-- core/controllers/utils.go | 331 ++++++++++++++++ core/controllers/utils_context.go | 19 - core/controllers/utils_filter.go | 141 ------- core/controllers/utils_http.go | 110 ------ core/controllers/utils_logger.go | 5 - core/controllers/utils_pagination.go | 36 -- core/controllers/utils_sort.go | 58 --- core/controllers/ws_writer.go | 54 --- core/go.mod | 40 +- core/go.sum | 42 +- core/models/models/base.go | 26 +- core/models/models/data_collection.go | 10 +- core/models/models/database.go | 34 +- core/models/models/database_metric.go | 36 +- core/models/models/dependency.go | 22 +- core/models/models/dependency_config.go | 20 +- core/models/models/dependency_config_setup.go | 18 +- core/models/models/dependency_log.go | 8 +- core/models/models/dependency_pypi_project.go | 10 +- core/models/models/dependency_repo.go | 12 +- core/models/models/environment.go | 8 +- core/models/models/git.go | 28 +- core/models/models/llm_provider.go | 22 +- core/models/models/metric.go | 2 +- core/models/models/node.go | 30 +- core/models/models/notification_alert.go | 26 +- core/models/models/notification_channel.go | 28 +- core/models/models/notification_request.go | 32 +- core/models/models/notification_setting.go | 10 +- core/models/models/permission.go | 16 +- core/models/models/project.go | 10 +- core/models/models/role.go | 16 +- core/models/models/schedule.go | 26 +- core/models/models/setting.go | 8 +- core/models/models/spider.go | 34 +- core/models/models/spider_stat.go | 2 +- core/models/models/task.go | 38 +- core/models/models/task_stat.go | 16 +- core/models/models/test.go | 6 +- core/models/models/token.go | 8 +- core/models/models/user.go | 24 +- core/models/models/user_role.go | 8 +- core/models/service/base_service_test.go | 6 +- core/openapi/wrapper.go | 14 +- go.work | 2 +- go.work.sum | 8 +- 58 files changed, 1555 insertions(+), 1306 deletions(-) create mode 100644 core/controllers/utils.go delete mode 100644 core/controllers/utils_context.go delete mode 100644 core/controllers/utils_filter.go delete mode 100644 core/controllers/utils_http.go delete mode 100644 core/controllers/utils_logger.go delete mode 100644 core/controllers/utils_pagination.go delete mode 100644 core/controllers/utils_sort.go delete mode 100644 core/controllers/ws_writer.go diff --git a/backend/go.mod b/backend/go.mod index f36f94d7..cacc2600 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,6 +1,8 @@ module crawlab -go 1.22.9 +go 1.23 + +toolchain go1.23.0 replace ( github.com/crawlab-team/crawlab/core => ../core @@ -49,6 +51,7 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/gofrs/uuid v3.2.0+incompatible // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect @@ -66,10 +69,12 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/juju/errors v1.0.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.17.7 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect + github.com/loopfz/gadgeto v0.9.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -97,6 +102,7 @@ require ( github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect + github.com/wI2L/fizz v0.22.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect @@ -124,6 +130,7 @@ require ( google.golang.org/grpc v1.69.2 // indirect google.golang.org/protobuf v1.36.1 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect + gopkg.in/go-playground/validator.v9 v9.30.0 // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/backend/go.sum b/backend/go.sum index bbea437e..ebd397ca 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -61,6 +61,7 @@ github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= @@ -134,6 +135,7 @@ github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -169,8 +171,14 @@ github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyT github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/cors v1.3.0/go.mod h1:artPvLlhkF7oG06nK8v3U8TNz6IeX+w1uzCSEId5/Vc= +github.com/gin-contrib/sse v0.0.0-20190125020943-a7658810eb74/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= @@ -197,18 +205,29 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= @@ -290,6 +309,7 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -361,6 +381,11 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/errors v1.0.0 h1:yiq7kjCLll1BiaRuNY53MGI0+EQ3rF6GB+wvboZDefM= +github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5Qe8= +github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= @@ -377,14 +402,21 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/loopfz/gadgeto v0.9.0 h1:yrQVBgdGhAWOB+JjH98sJzYfSqpcpKzOBIEtJFcRl2s= +github.com/loopfz/gadgeto v0.9.0/go.mod h1:S3tK5SXmKY3l39rUpPZw1B/iiy1CftV13QABFhj32Ss= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -397,7 +429,9 @@ github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= @@ -440,6 +474,7 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -467,6 +502,8 @@ github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -542,8 +579,17 @@ github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9f github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0= +github.com/ugorji/go/codec v0.0.0-20190128213124-ee1426cffec0/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/wI2L/fizz v0.22.0 h1:mgRA+uUdESvgsIeBFkMSS/MEIQ4EZ4I2xyRxnCqkhJY= +github.com/wI2L/fizz v0.22.0/go.mod h1:CMxMR1amz8id9wr2YUpONf+F/F9hW1cqRXxVNNuWVxE= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -612,8 +658,10 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= @@ -811,6 +859,7 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1074,11 +1123,17 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/go-playground/validator.v9 v9.26.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/go-playground/validator.v9 v9.30.0 h1:Wk0Z37oBmKj9/n+tPyBHZmeL19LaCoK3Qq48VwYENss= +gopkg.in/go-playground/validator.v9 v9.30.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= diff --git a/core/controllers/base.go b/core/controllers/base.go index 3b06c103..52cc3bcf 100644 --- a/core/controllers/base.go +++ b/core/controllers/base.go @@ -1,7 +1,8 @@ package controllers import ( - "github.com/crawlab-team/crawlab/core/entity" + "time" + "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/mongo" @@ -23,28 +24,32 @@ type BaseController[T any] struct { actions []Action } -type GetByIdParams struct { - Id string `path:"id" description:"The ID of the item to get"` -} - -// GetAllParams represents parameters for GetAll method -type GetAllParams struct { - Query bson.M `json:"query"` - Sort bson.D `json:"sort"` -} - // GetListParams represents parameters for GetList with pagination type GetListParams struct { - Query bson.M `json:"query"` - Sort bson.D `json:"sort"` - Pagination *entity.Pagination `json:"pagination"` - All bool `query:"all" description:"Whether to get all items"` + Conditions string `query:"conditions" description:"Filter conditions. Format: [{\"key\":\"name\",\"op\":\"eq\",\"value\":\"test\"}]"` + Sort bson.D `query:"sort" description:"Sort options"` + Page int `query:"page" default:"1" description:"Page number"` + Size int `query:"size" default:"10" description:"Page size"` + All bool `query:"all" default:"false" description:"Whether to get all items"` +} + +func (ctr *BaseController[T]) GetList(_ *gin.Context, params *GetListParams) (response *ListResponse[T], err error) { + // get all if query field "all" is set true + if params.All { + return ctr.GetAll(params) + } + + return ctr.GetWithPagination(params) +} + +type GetByIdParams struct { + Id string `path:"id" description:"The ID of the item to get"` } func (ctr *BaseController[T]) GetById(_ *gin.Context, params *GetByIdParams) (response *Response[T], err error) { id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - return nil, errors.BadRequestf("invalid id: %s", params.Id) + return GetErrorResponse[T](errors.BadRequestf("invalid id format")) } model, err := ctr.modelSvc.GetById(id) @@ -52,143 +57,132 @@ func (ctr *BaseController[T]) GetById(_ *gin.Context, params *GetByIdParams) (re return nil, err } - return GetSuccessDataResponse(*model) + return GetDataResponse(*model) } -func (ctr *BaseController[T]) GetList(c *gin.Context, params *GetListParams) (response *ListResponse[T], err error) { - // get all if query field "all" is set true - all := params.All || MustGetFilterAll(c) - - // Prepare parameters - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - - if all { - allParams := &GetAllParams{ - Query: query, - Sort: sort, - } - return ctr.GetAll(c, allParams) - } - - // get list with pagination - pagination := MustGetPagination(c) - listParams := &GetListParams{ - Query: query, - Sort: sort, - Pagination: pagination, - } - return ctr.GetWithPagination(c, listParams) +type PostParams[T any] struct { + Data T `json:"data"` } -func (ctr *BaseController[T]) Post(c *gin.Context) (response *Response[T], err error) { - var model T - if err := c.ShouldBindJSON(&model); err != nil { - return GetErrorDataResponse[T](err) - } +func (ctr *BaseController[T]) Post(c *gin.Context, params *PostParams[T]) (response *Response[T], err error) { u := GetUserFromContext(c) - m := any(&model).(interfaces.Model) + m := any(¶ms.Data).(interfaces.Model) m.SetId(primitive.NewObjectID()) m.SetCreated(u.Id) m.SetUpdated(u.Id) col := ctr.modelSvc.GetCol() res, err := col.GetCollection().InsertOne(col.GetContext(), m) if err != nil { - return GetErrorDataResponse[T](err) + return GetErrorResponse[T](err) } result, err := ctr.modelSvc.GetById(res.InsertedID.(primitive.ObjectID)) if err != nil { - return GetErrorDataResponse[T](err) + return GetErrorResponse[T](err) } - return GetSuccessDataResponse(*result) + return GetDataResponse(*result) } -func (ctr *BaseController[T]) PutById(c *gin.Context) (response *Response[T], err error) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - return GetErrorDataResponse[T](err) - } +type PutByIdParams[T any] struct { + Id string `path:"id" description:"The ID of the item to update"` + Data T `json:"data"` +} - var model T - if err := c.ShouldBindJSON(&model); err != nil { - return GetErrorDataResponse[T](err) +func (ctr *BaseController[T]) PutById(c *gin.Context, params *PutByIdParams[T]) (response *Response[T], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) + if err != nil { + return GetErrorResponse[T](errors.BadRequestf("invalid id format: %v", err)) } u := GetUserFromContext(c) - m := any(&model).(interfaces.Model) + m := any(¶ms.Data).(interfaces.Model) m.SetUpdated(u.Id) + if m.GetId().IsZero() { + m.SetId(id) + } - if err := ctr.modelSvc.ReplaceById(id, model); err != nil { - return GetErrorDataResponse[T](err) + if err := ctr.modelSvc.ReplaceById(id, params.Data); err != nil { + return GetErrorResponse[T](err) } result, err := ctr.modelSvc.GetById(id) if err != nil { - return GetErrorDataResponse[T](err) + return GetErrorResponse[T](err) } - return GetSuccessDataResponse(*result) + return GetDataResponse(*result) } -func (ctr *BaseController[T]) PatchList(c *gin.Context) (res *Response[T], err error) { - type Payload struct { - Ids []primitive.ObjectID `json:"ids"` - Update bson.M `json:"update"` +type PatchParams struct { + Ids []string `json:"ids" description:"The IDs of the items to update" validate:"required"` + Update bson.M `json:"update" description:"The update object" validate:"required"` +} + +func (ctr *BaseController[T]) PatchList(c *gin.Context, params *PatchParams) (res *Response[T], err error) { + var ids []primitive.ObjectID + for _, id := range params.Ids { + objectId, err := primitive.ObjectIDFromHex(id) + if err != nil { + return GetErrorResponse[T](errors.BadRequestf("invalid id format: %v", err)) + } + ids = append(ids, objectId) } - var payload Payload - if err := c.ShouldBindJSON(&payload); err != nil { - return GetErrorDataResponse[T](err) - } + // Get user from context for updated_by + u := GetUserFromContext(c) // query query := bson.M{ "_id": bson.M{ - "$in": payload.Ids, + "$in": ids, }, } + // Add updated_by and updated_ts to the update object + updateObj := params.Update + updateObj["updated_by"] = u.Id + updateObj["updated_ts"] = time.Now() + // update - if err := ctr.modelSvc.UpdateMany(query, bson.M{"$set": payload.Update}); err != nil { - return GetErrorDataResponse[T](err) + if err := ctr.modelSvc.UpdateMany(query, bson.M{"$set": updateObj}); err != nil { + return GetErrorResponse[T](err) } // Return an empty response with success status var emptyModel T - return GetSuccessDataResponse(emptyModel) + return GetDataResponse(emptyModel) } -func (ctr *BaseController[T]) DeleteById(c *gin.Context) (res *Response[T], err error) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +type DeleteByIdParams struct { + Id string `path:"id" description:"The ID of the item to get"` +} + +func (ctr *BaseController[T]) DeleteById(c *gin.Context, params *DeleteByIdParams) (res *Response[T], err error) { + params.Id = c.Param("id") + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - return GetErrorDataResponse[T](err) + return GetErrorResponse[T](errors.BadRequestf("invalid id format: %v", err)) } if err := ctr.modelSvc.DeleteById(id); err != nil { - return GetErrorDataResponse[T](err) + return GetErrorResponse[T](err) } var emptyModel T - return GetSuccessDataResponse(emptyModel) + return GetDataResponse(emptyModel) } -func (ctr *BaseController[T]) DeleteList(c *gin.Context) (res *Response[T], err error) { - type Payload struct { - Ids []string `json:"ids"` - } - - var payload Payload - if err := c.ShouldBindJSON(&payload); err != nil { - return GetErrorDataResponse[T](err) - } +type DeleteListParams struct { + Ids []string `json:"ids" description:"The IDs of the items to delete"` +} +func (ctr *BaseController[T]) DeleteList(_ *gin.Context, params *DeleteListParams) (res *Response[T], err error) { var ids []primitive.ObjectID - for _, id := range payload.Ids { + for _, id := range params.Ids { objectId, err := primitive.ObjectIDFromHex(id) if err != nil { - return GetErrorDataResponse[T](err) + return GetErrorResponse[T](err) } ids = append(ids, objectId) } @@ -198,16 +192,19 @@ func (ctr *BaseController[T]) DeleteList(c *gin.Context) (res *Response[T], err "$in": ids, }, }); err != nil { - return GetErrorDataResponse[T](err) + return GetErrorResponse[T](err) } var emptyModel T - return GetSuccessDataResponse(emptyModel) + return GetDataResponse(emptyModel) } // GetAll retrieves all items based on filter and sort -func (ctr *BaseController[T]) GetAll(_ *gin.Context, params *GetAllParams) (response *ListResponse[T], err error) { - query := params.Query +func (ctr *BaseController[T]) GetAll(params *GetListParams) (response *ListResponse[T], err error) { + query, err := GetFilterQueryFromListParams(params) + if err != nil { + return GetErrorListResponse[T](errors.BadRequestf("invalid request parameters: %v", err)) + } sort := params.Sort if sort == nil { sort = bson.D{{"_id", -1}} @@ -225,29 +222,23 @@ func (ctr *BaseController[T]) GetAll(_ *gin.Context, params *GetAllParams) (resp return nil, err } - return GetSuccessListResponse(models, total) + return GetListResponse(models, total) } // GetWithPagination retrieves items with pagination -func (ctr *BaseController[T]) GetWithPagination(_ *gin.Context, params *GetListParams) (response *ListResponse[T], err error) { - // params - pagination := params.Pagination - query := params.Query - sort := params.Sort - - if pagination == nil { - pagination = GetDefaultPagination() +func (ctr *BaseController[T]) GetWithPagination(params *GetListParams) (response *ListResponse[T], err error) { + query, err := GetFilterQueryFromListParams(params) + if err != nil { + return GetErrorListResponse[T](errors.BadRequestf("invalid request parameters: %v", err)) } - - // get list models, err := ctr.modelSvc.GetMany(query, &mongo.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, + Sort: params.Sort, + Skip: params.Size * (params.Page - 1), + Limit: params.Size, }) if err != nil { if errors.Is(err, mongo2.ErrNoDocuments) { - return GetSuccessListResponse[T](nil, 0) + return GetListResponse[T](nil, 0) } else { return nil, err } @@ -260,36 +251,7 @@ func (ctr *BaseController[T]) GetWithPagination(_ *gin.Context, params *GetListP } // response - return GetSuccessListResponse(models, total) -} - -// getAll is kept for backward compatibility -func (ctr *BaseController[T]) getAll(c *gin.Context) (response *ListResponse[T], err error) { - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - - params := &GetAllParams{ - Query: query, - Sort: sort, - } - - return ctr.GetAll(c, params) -} - -// getList is kept for backward compatibility -func (ctr *BaseController[T]) getList(c *gin.Context) (response *ListResponse[T], err error) { - // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - - params := &GetListParams{ - Query: query, - Sort: sort, - Pagination: pagination, - } - - return ctr.GetWithPagination(c, params) + return GetListResponse(models, total) } func NewController[T any](actions ...Action) *BaseController[T] { diff --git a/core/controllers/base_file.go b/core/controllers/base_file.go index 403fe89d..fc8657ed 100644 --- a/core/controllers/base_file.go +++ b/core/controllers/base_file.go @@ -4,48 +4,54 @@ import ( "errors" "fmt" "github.com/crawlab-team/crawlab/core/fs" + "github.com/crawlab-team/crawlab/core/interfaces" "github.com/gin-gonic/gin" "io" "os" "sync" ) -func GetBaseFileListDir(rootPath string, c *gin.Context) { - path := c.Query("path") +type GetBaseFileListDirParams struct { + Path string `path:"path"` +} + +func GetBaseFileListDir(rootPath string, params *GetBaseFileListDirParams) (response *Response[[]interfaces.FsFileInfo], err error) { + path := params.Path fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[[]interfaces.FsFileInfo](err) } files, err := fsSvc.List(path) if err != nil { if !errors.Is(err, os.ErrNotExist) { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]interfaces.FsFileInfo](err) } } - HandleSuccessWithData(c, files) + //HandleSuccessWithData(c, files) + return GetDataResponse[[]interfaces.FsFileInfo](files) } -func GetBaseFileFile(rootPath string, c *gin.Context) { - path := c.Query("path") +type GetBaseFileFileParams struct { + Path string `path:"path"` +} + +func GetBaseFileFile(rootPath string, params *GetBaseFileFileParams) (response *Response[string], err error) { + path := params.Path fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[string](err) } data, err := fsSvc.GetFile(path) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[string](err) } - HandleSuccessWithData(c, string(data)) + return GetDataResponse[string](string(data)) } func GetBaseFileFileInfo(rootPath string, c *gin.Context) { diff --git a/core/controllers/base_test.go b/core/controllers/base_test.go index 85cb4b34..e3715938 100644 --- a/core/controllers/base_test.go +++ b/core/controllers/base_test.go @@ -4,19 +4,30 @@ import ( "bytes" "context" "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "testing" + "time" + + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/middlewares" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/user" + "github.com/loopfz/gadgeto/tonic" "github.com/spf13/viper" - "net/http" - "net/http/httptest" - "testing" + "github.com/stretchr/testify/require" + "github.com/wI2L/fizz" - "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" ) func init() { @@ -26,7 +37,12 @@ func init() { // TestModel is a simple struct to be used as a model in tests type TestModel models.TestModel +//type TestModel struct { +// Name string `json:"name" bson:"name"` +//} + var TestToken string +var TestUserId primitive.ObjectID // SetupTestDB sets up the test database func SetupTestDB() { @@ -50,11 +66,12 @@ func SetupTestDB() { panic(err) } TestToken = token + TestUserId = u.Id } // SetupRouter sets up the gin router for testing -func SetupRouter() *gin.Engine { - router := gin.Default() +func SetupRouter() *fizz.Fizz { + router := fizz.New() return router } @@ -77,7 +94,7 @@ func TestBaseController_GetById(t *testing.T) { // Set up the router router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.GET("/testmodels/:id", ctr.GetById) + router.GET("/testmodels/:id", nil, tonic.Handler(ctr.GetById, 200)) // Create a test request req, _ := http.NewRequest("GET", "/testmodels/"+id.Hex(), nil) @@ -106,30 +123,34 @@ func TestBaseController_Post(t *testing.T) { // Set up the router router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.POST("/testmodels", ctr.Post) + router.POST("/testmodels", nil, tonic.Handler(ctr.Post, 200)) // Create a test request testModel := TestModel{Name: "test"} - jsonValue, _ := json.Marshal(testModel) + requestBody := controllers.PostParams[TestModel]{ + Data: testModel, + } + jsonValue, _ := json.Marshal(requestBody) req, _ := http.NewRequest("POST", "/testmodels", bytes.NewBuffer(jsonValue)) req.Header.Set("Authorization", TestToken) + req.Header.Set("Content-Type", "application/json") w := httptest.NewRecorder() // Serve the request router.ServeHTTP(w, req) // Check the response - assert.Equal(t, http.StatusOK, w.Code) + require.Equal(t, http.StatusOK, w.Code) var response controllers.Response[TestModel] err := json.Unmarshal(w.Body.Bytes(), &response) - assert.NoError(t, err) - assert.Equal(t, "test", response.Data.Name) + require.NoError(t, err) + require.Equal(t, "test", response.Data.Name) // Check if the document was inserted into the database result, err := service.NewModelService[TestModel]().GetById(response.Data.Id) - assert.NoError(t, err) - assert.Equal(t, "test", result.Name) + require.NoError(t, err) + require.Equal(t, "test", result.Name) } func TestBaseController_DeleteById(t *testing.T) { @@ -146,7 +167,7 @@ func TestBaseController_DeleteById(t *testing.T) { // Set up the router router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.DELETE("/testmodels/:id", ctr.DeleteById) + router.DELETE("/testmodels/:id", nil, tonic.Handler(ctr.DeleteById, 200)) // Create a test request req, _ := http.NewRequest("DELETE", "/testmodels/"+id.Hex(), nil) @@ -163,3 +184,321 @@ func TestBaseController_DeleteById(t *testing.T) { _, err = service.NewModelService[TestModel]().GetById(id) assert.Error(t, err) } + +func TestBaseController_GetList(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + // Insert test documents + modelSvc := service.NewModelService[TestModel]() + for i := 0; i < 15; i++ { + _, err := modelSvc.InsertOne(TestModel{Name: fmt.Sprintf("test%d", i)}) + assert.NoError(t, err) + } + + // Initialize the controller + ctr := controllers.NewController[TestModel]() + + // Set up the router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.GET("/testmodels/list", nil, tonic.Handler(ctr.GetList, 200)) + + // Test case 1: Get with pagination + t.Run("test_get_with_pagination", func(t *testing.T) { + var testData = []struct { + Page int + ExpectedDataCount int + ExpectedTotalCount int + }{ + {1, 10, 15}, + {2, 5, 15}, + } + for _, data := range testData { + params := url.Values{} + params.Add("page", strconv.Itoa(data.Page)) + params.Add("size", "10") + requestUrl := url.URL{Path: "/testmodels/list", RawQuery: params.Encode()} + req, _ := http.NewRequest("GET", requestUrl.String(), nil) + req.Header.Set("Authorization", TestToken) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + // Serve the request + router.ServeHTTP(w, req) + + // Check the response + assert.Equal(t, http.StatusOK, w.Code) + + var response controllers.ListResponse[TestModel] + err := json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, data.ExpectedDataCount, len(response.Data)) + assert.Equal(t, data.ExpectedTotalCount, response.Total) + } + }) + + // Test case 2: Get all + t.Run("test_get_all", func(t *testing.T) { + params := url.Values{} + params.Add("all", "true") + requestUrl := url.URL{Path: "/testmodels/list", RawQuery: params.Encode()} + req, _ := http.NewRequest("GET", requestUrl.String(), nil) + req.Header.Set("Authorization", TestToken) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + // Serve the request + router.ServeHTTP(w, req) + + // Check the response + assert.Equal(t, http.StatusOK, w.Code) + + var response controllers.ListResponse[TestModel] + err := json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, 15, len(response.Data)) + assert.Equal(t, 15, response.Total) + }) + + // Test case 3: Get with query filter + t.Run("test_get_with_query_filter", func(t *testing.T) { + cond := []entity.Condition{ + {Key: "name", Op: "eq", Value: "test1"}, + } + condBytes, err := json.Marshal(cond) + require.Nil(t, err) + params := url.Values{} + params.Add("conditions", string(condBytes)) + params.Add("page", "1") + params.Add("size", "10") + requestUrl := url.URL{Path: "/testmodels/list", RawQuery: params.Encode()} + req, _ := http.NewRequest("GET", requestUrl.String(), nil) + req.Header.Set("Authorization", TestToken) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + // Serve the request + router.ServeHTTP(w, req) + + // Check the response + assert.Equal(t, http.StatusOK, w.Code) + + var response controllers.ListResponse[TestModel] + err = json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, 1, len(response.Data)) + assert.Equal(t, 1, response.Total) + }) +} + +func TestBaseController_PutById(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + // Insert a test document + id, err := service.NewModelService[TestModel]().InsertOne(TestModel{Name: "test"}) + assert.NoError(t, err) + + // Initialize the controller + ctr := controllers.NewController[TestModel]() + + // Set up the router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.PUT("/testmodels/:id", nil, tonic.Handler(ctr.PutById, 200)) + + // Create a test request + updatedModel := TestModel{Name: "updated"} + requestParams := controllers.PutByIdParams[TestModel]{ + Data: updatedModel, + } + jsonValue, _ := json.Marshal(requestParams) + req, _ := http.NewRequest("PUT", "/testmodels/"+id.Hex(), bytes.NewBuffer(jsonValue)) + req.Header.Set("Authorization", TestToken) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + // Serve the request + router.ServeHTTP(w, req) + + // Check the response + assert.Equal(t, http.StatusOK, w.Code) + + var response controllers.Response[TestModel] + err = json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, "updated", response.Data.Name) + + // Check if the document was updated in the database + result, err := service.NewModelService[TestModel]().GetById(id) + assert.NoError(t, err) + assert.Equal(t, "updated", result.Name) + + // Test with invalid ID + req, _ = http.NewRequest("PUT", "/testmodels/invalid-id", bytes.NewBuffer(jsonValue)) + req.Header.Set("Authorization", TestToken) + req.Header.Set("Content-Type", "application/json") + w = httptest.NewRecorder() + + // Serve the request + router.ServeHTTP(w, req) + + // Check the response + assert.Equal(t, http.StatusBadRequest, w.Code) +} + +func TestBaseController_PatchList(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + // Insert test documents + modelSvc := service.NewModelService[TestModel]() + var ids []primitive.ObjectID + for i := 0; i < 3; i++ { + id, err := modelSvc.InsertOne(TestModel{Name: fmt.Sprintf("test%d", i)}) + assert.NoError(t, err) + ids = append(ids, id) + } + + // Initialize the controller + ctr := controllers.NewController[TestModel]() + + // Set up the router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.PATCH("/testmodels", nil, tonic.Handler(ctr.PatchList, 200)) + + // Create a test request + t.Run("test_patch_list", func(t *testing.T) { + var idStrings []string + for _, id := range ids { + idStrings = append(idStrings, id.Hex()) + } + requestBody := controllers.PatchParams{ + Ids: idStrings, + Update: bson.M{"name": "patched"}, + } + jsonValue, _ := json.Marshal(requestBody) + req, _ := http.NewRequest("PATCH", "/testmodels", bytes.NewBuffer(jsonValue)) + req.Header.Set("Authorization", TestToken) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + // Get the user ID + userId := TestUserId + + // Record time before the update + beforeUpdate := time.Now() + + time.Sleep(100 * time.Millisecond) + + // Serve the request + router.ServeHTTP(w, req) + + // Check the response + assert.Equal(t, http.StatusOK, w.Code) + + time.Sleep(100 * time.Millisecond) + + // Record time after the update + afterUpdate := time.Now() + + // Check if the documents were updated in the database + for _, id := range ids { + result, err := modelSvc.GetById(id) + assert.NoError(t, err) + assert.Equal(t, "patched", result.Name) + + // Verify updated_by is set to the current user's ID + assert.Equal(t, userId, result.UpdatedBy) + + // Verify updated_ts is set to a recent timestamp + assert.GreaterOrEqual(t, result.UpdatedAt.UnixMilli(), beforeUpdate.UnixMilli()) + assert.LessOrEqual(t, result.UpdatedAt.UnixMilli(), afterUpdate.UnixMilli()) + } + }) + + // Test with invalid ID + t.Run("test_patch_list_with_invalid_id", func(t *testing.T) { + requestBody := controllers.PatchParams{ + Ids: []string{"invalid-id"}, + Update: bson.M{"name": "patched"}, + } + jsonValue, _ := json.Marshal(requestBody) + req, _ := http.NewRequest("PATCH", "/testmodels", bytes.NewBuffer(jsonValue)) + req.Header.Set("Authorization", TestToken) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + // Serve the request + router.ServeHTTP(w, req) + + // Check the response + assert.Equal(t, http.StatusBadRequest, w.Code) + }) +} + +func TestBaseController_DeleteList(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + // Insert test documents + modelSvc := service.NewModelService[TestModel]() + var ids []primitive.ObjectID + for i := 0; i < 3; i++ { + id, err := modelSvc.InsertOne(TestModel{Name: fmt.Sprintf("test%d", i)}) + assert.NoError(t, err) + ids = append(ids, id) + } + + // Initialize the controller + ctr := controllers.NewController[TestModel]() + + // Set up the router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.DELETE("/testmodels", nil, tonic.Handler(ctr.DeleteList, 200)) + + // Create a test request + var idStrings []string + for _, id := range ids { + idStrings = append(idStrings, id.Hex()) + } + requestBody := controllers.DeleteListParams{ + Ids: idStrings, + } + jsonValue, _ := json.Marshal(requestBody) + req, _ := http.NewRequest("DELETE", "/testmodels", bytes.NewBuffer(jsonValue)) + req.Header.Set("Authorization", TestToken) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + // Serve the request + router.ServeHTTP(w, req) + + // Check the response + assert.Equal(t, http.StatusOK, w.Code) + + // Check if the documents were deleted from the database + for _, id := range ids { + _, err := modelSvc.GetById(id) + assert.Error(t, err) + } + + // Test with invalid ID + requestBody = controllers.DeleteListParams{ + Ids: []string{"invalid-id"}, + } + jsonValue, _ = json.Marshal(requestBody) + req, _ = http.NewRequest("DELETE", "/testmodels", bytes.NewBuffer(jsonValue)) + req.Header.Set("Authorization", TestToken) + req.Header.Set("Content-Type", "application/json") + w = httptest.NewRecorder() + + // Serve the request + router.ServeHTTP(w, req) + + // Check the response + assert.Equal(t, http.StatusBadRequest, w.Code) +} diff --git a/core/controllers/project.go b/core/controllers/project.go index 0bd21595..8ec44378 100644 --- a/core/controllers/project.go +++ b/core/controllers/project.go @@ -1,34 +1,31 @@ package controllers import ( - "errors" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/mongo" "github.com/gin-gonic/gin" + "github.com/juju/errors" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" ) -func GetProjectList(c *gin.Context) { - // get all list - all := MustGetFilterAll(c) - if all { - NewController[models.Project]().getAll(c) - return +func GetProjectList(c *gin.Context, params *GetListParams) (response *ListResponse[models.Project], err error) { + if params.All { + return NewController[models.Project]().GetAll(params) } - // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) + query, err := GetFilterQueryFromListParams(params) + if err != nil { + return GetErrorListResponse[models.Project](errors.BadRequestf("invalid request parameters: %v", err)) + } // get list projects, err := service.NewModelService[models.Project]().GetMany(query, &mongo.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, + Sort: params.Sort, + Skip: params.Size * (params.Page - 1), + Limit: params.Size, }) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { @@ -82,5 +79,5 @@ func GetProjectList(c *gin.Context) { projects[i].Spiders = cache[p.Id] } - HandleSuccessWithListData(c, projects, total) + return GetListResponse[models.Project](projects, total) } diff --git a/core/controllers/router.go b/core/controllers/router.go index 9fa346fd..1b7846a2 100644 --- a/core/controllers/router.go +++ b/core/controllers/router.go @@ -29,13 +29,13 @@ func GetGlobalFizzWrapper() *openapi.FizzWrapper { // NewRouterGroups initializes the router groups with their respective middleware func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) { // Create OpenAPI wrapper - wrapper := openapi.NewFizzWrapper(app) + globalWrapper = openapi.NewFizzWrapper(app) return &RouterGroups{ AuthGroup: app.Group("/", middlewares.AuthorizationMiddleware()), SyncAuthGroup: app.Group("/", middlewares.SyncAuthorizationMiddleware()), AnonymousGroup: app.Group("/"), - Wrapper: wrapper, + Wrapper: globalWrapper, } } @@ -232,33 +232,6 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetProjectList, }, }...)) - RegisterController(groups.AuthGroup, "/schedules", NewController[models.Schedule]([]Action{ - { - Method: http.MethodPost, - Path: "", - HandlerFunc: PostSchedule, - }, - { - Method: http.MethodPut, - Path: "/:id", - HandlerFunc: PutScheduleById, - }, - { - Method: http.MethodPost, - Path: "/:id/enable", - HandlerFunc: PostScheduleEnable, - }, - { - Method: http.MethodPost, - Path: "/:id/disable", - HandlerFunc: PostScheduleDisable, - }, - { - Method: http.MethodPost, - Path: "/:id/run", - HandlerFunc: PostScheduleRun, - }, - }...)) RegisterController(groups.AuthGroup, "/spiders", NewController[models.Spider]([]Action{ { Method: http.MethodGet, @@ -351,6 +324,35 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetSpiderResults, }, }...)) + groups.AnonymousGroup.GET("/openapi.json", GetOpenAPI) + return + RegisterController(groups.AuthGroup, "/schedules", NewController[models.Schedule]([]Action{ + { + Method: http.MethodPost, + Path: "", + HandlerFunc: PostSchedule, + }, + { + Method: http.MethodPut, + Path: "/:id", + HandlerFunc: PutScheduleById, + }, + { + Method: http.MethodPost, + Path: "/:id/enable", + HandlerFunc: PostScheduleEnable, + }, + { + Method: http.MethodPost, + Path: "/:id/disable", + HandlerFunc: PostScheduleDisable, + }, + { + Method: http.MethodPost, + Path: "/:id/run", + HandlerFunc: PostScheduleRun, + }, + }...)) RegisterController(groups.AuthGroup, "/tasks", NewController[models.Task]([]Action{ { Method: http.MethodGet, diff --git a/core/controllers/router_test.go b/core/controllers/router_test.go index e222ce8c..94a40bb0 100644 --- a/core/controllers/router_test.go +++ b/core/controllers/router_test.go @@ -60,7 +60,7 @@ func TestRegisterController_Routes(t *testing.T) { func TestInitRoutes_ProjectsRoute(t *testing.T) { router := gin.Default() - controllers.InitRoutes(router) + _ = controllers.InitRoutes(router) // Check if the projects route is registered routes := router.Routes() diff --git a/core/controllers/spider.go b/core/controllers/spider.go index 07aeb47c..06008862 100644 --- a/core/controllers/spider.go +++ b/core/controllers/spider.go @@ -1,59 +1,55 @@ package controllers import ( - "errors" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/models" - mongo2 "github.com/crawlab-team/crawlab/core/mongo" - "github.com/crawlab-team/crawlab/core/spider" "math" "os" "path/filepath" "sync" + "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/fs" "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" + mongo2 "github.com/crawlab-team/crawlab/core/mongo" + "github.com/crawlab-team/crawlab/core/spider" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/utils" "github.com/gin-gonic/gin" + "github.com/juju/errors" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" ) -func GetSpiderById(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +// GetSpiderById handles getting a spider by ID +func GetSpiderById(_ *gin.Context, params *GetByIdParams) (response *Response[models.Spider], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[models.Spider](errors.BadRequestf("invalid id format")) } s, err := service.NewModelService[models.Spider]().GetById(id) if errors.Is(err, mongo.ErrNoDocuments) { - HandleErrorNotFound(c, err) - return + return GetErrorResponse[models.Spider](errors.NotFoundf("spider not found")) } if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Spider](err) } // stat s.Stat, err = service.NewModelService[models.SpiderStat]().GetById(s.Id) if err != nil { if !errors.Is(err, mongo.ErrNoDocuments) { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Spider](err) } } - // data collection (compatible to old version) # TODO: remove in the future + // data collection (compatible to old version) if s.ColName == "" && !s.ColId.IsZero() { col, err := service.NewModelService[models.DataCollection]().GetById(s.ColId) if err != nil { if !errors.Is(err, mongo.ErrNoDocuments) { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Spider](err) } } else { s.ColName = col.Name @@ -65,55 +61,51 @@ func GetSpiderById(c *gin.Context) { s.Git, err = service.NewModelService[models.Git]().GetById(s.GitId) if err != nil { if !errors.Is(err, mongo.ErrNoDocuments) { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Spider](err) } } } - HandleSuccessWithData(c, s) + return GetDataResponse(*s) } -func GetSpiderList(c *gin.Context, params *GetListParams) { +// GetSpiderList handles getting a list of spiders with optional stats +func GetSpiderList(c *gin.Context, params *GetListParams) (response *ListResponse[models.Spider], err error) { // get all list - all := MustGetFilterAll(c) + all := params.All if all { - NewController[models.Spider]().getAll(c) - return + return NewController[models.Spider]().GetAll(params) } // get list withStats := c.Query("stats") if withStats == "" { - NewController[models.Spider]().GetList(c, params) - return + return NewController[models.Spider]().GetList(c, params) } // get list with stats - getSpiderListWithStats(c) + return getSpiderListWithStats(params) } -func getSpiderListWithStats(c *gin.Context) { - // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - +func getSpiderListWithStats(params *GetListParams) (response *ListResponse[models.Spider], err error) { + query, err := GetFilterQueryFromListParams(params) + if err != nil { + return GetErrorListResponse[models.Spider](errors.BadRequestf("invalid request parameters: %v", err)) + } // get list spiders, err := service.NewModelService[models.Spider]().GetMany(query, &mongo2.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, + Sort: params.Sort, + Skip: params.Size * (params.Page - 1), + Limit: params.Size, }) if err != nil { - if err.Error() != mongo.ErrNoDocuments.Error() { - HandleErrorInternalServerError(c, err) + if !errors.Is(err, mongo.ErrNoDocuments) { + return GetErrorListResponse[models.Spider](err) } - return + return GetListResponse[models.Spider]([]models.Spider{}, 0) } if len(spiders) == 0 { - HandleSuccessWithListData(c, []models.Spider{}, 0) - return + return GetListResponse[models.Spider]([]models.Spider{}, 0) } // ids @@ -129,15 +121,13 @@ func getSpiderListWithStats(c *gin.Context) { // total count total, err := service.NewModelService[models.Spider]().Count(query) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.Spider](err) } // stat list spiderStats, err := service.NewModelService[models.SpiderStat]().GetMany(bson.M{"_id": bson.M{"$in": ids}}, nil) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.Spider](err) } // cache stat list to dict @@ -170,15 +160,13 @@ func getSpiderListWithStats(c *gin.Context) { } tasks, err = service.NewModelService[models.Task]().GetMany(queryTask, nil) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.Spider](err) } // task stats list taskStats, err := service.NewModelService[models.TaskStat]().GetMany(queryTask, nil) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.Spider](err) } // cache task stats to dict @@ -201,8 +189,7 @@ func getSpiderListWithStats(c *gin.Context) { if len(gitIds) > 0 && utils.IsPro() { gits, err = service.NewModelService[models.Git]().GetMany(bson.M{"_id": bson.M{"$in": gitIds}}, nil) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.Spider](err) } } @@ -240,16 +227,12 @@ func getSpiderListWithStats(c *gin.Context) { } // response - HandleSuccessWithListData(c, data, total) + return GetListResponse(data, total) } -func PostSpider(c *gin.Context) { - // bind - var s models.Spider - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } +// PostSpider handles creating a new spider +func PostSpider(c *gin.Context, params *PostParams[models.Spider]) (response *Response[models.Spider], err error) { + s := params.Data if s.Mode == "" { s.Mode = constants.RunTypeRandom @@ -266,8 +249,7 @@ func PostSpider(c *gin.Context) { s.SetUpdated(u.Id) id, err := service.NewModelService[models.Spider]().InsertOne(s) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Spider](err) } s.SetId(id) @@ -278,20 +260,17 @@ func PostSpider(c *gin.Context) { st.SetUpdated(u.Id) _, err = service.NewModelService[models.SpiderStat]().InsertOne(st) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Spider](err) } // create folder fsSvc, err := getSpiderFsSvcById(id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Spider](err) } err = fsSvc.CreateDir(".") if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Spider](err) } // create template if available @@ -299,63 +278,52 @@ func PostSpider(c *gin.Context) { if templateSvc := spider.GetSpiderTemplateRegistryService(); templateSvc != nil { err = templateSvc.CreateTemplate(s.Id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Spider](err) } } } - HandleSuccessWithData(c, s) + return GetDataResponse(s) } -func PutSpiderById(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +// PutSpiderById handles updating a spider by ID +func PutSpiderById(c *gin.Context, params *PutByIdParams[models.Spider]) (response *Response[models.Spider], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // bind - var s models.Spider - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[models.Spider](errors.BadRequestf("invalid id format")) } u := GetUserFromContext(c) - modelSvc := service.NewModelService[models.Spider]() - // save - s.SetUpdated(u.Id) - err = modelSvc.ReplaceById(id, s) - if err != nil { - HandleErrorInternalServerError(c, err) - return + params.Data.SetUpdated(u.Id) + if params.Data.Id.IsZero() { + params.Data.SetId(id) } - _s, err := modelSvc.GetById(id) + err = modelSvc.ReplaceById(id, params.Data) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Spider](err) } - s = *_s - HandleSuccessWithData(c, s) + s, err := modelSvc.GetById(id) + if err != nil { + return GetErrorResponse[models.Spider](err) + } + + return GetDataResponse(*s) } -func DeleteSpiderById(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +// DeleteSpiderById handles deleting a spider by ID +func DeleteSpiderById(_ *gin.Context, params *DeleteByIdParams) (response *Response[models.Spider], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[models.Spider](errors.BadRequestf("invalid id format")) } - // spider s, err := service.NewModelService[models.Spider]().GetById(id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Spider](errors.NotFoundf("spider not found")) } if err := mongo2.RunTransaction(func(context mongo.SessionContext) (err error) { @@ -416,14 +384,13 @@ func DeleteSpiderById(c *gin.Context) { return nil }); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Spider](err) } if !s.GitId.IsZero() { go func() { // delete spider directory - fsSvc, err := getSpiderFsSvcById(id) + fsSvc, err := getSpiderFsSvcById(s.Id) if err != nil { logger.Errorf("failed to get spider fs service: %v", err) return @@ -436,34 +403,39 @@ func DeleteSpiderById(c *gin.Context) { }() } - HandleSuccess(c) + return GetDataResponse(models.Spider{}) } -func DeleteSpiderList(c *gin.Context) { - var payload struct { - Ids []primitive.ObjectID `json:"ids"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return +type DeleteSpiderListParams struct { + Ids []string `json:"ids" validate:"required"` +} + +// DeleteSpiderList handles deleting multiple spiders +func DeleteSpiderList(_ *gin.Context, params *DeleteSpiderListParams) (response *Response[models.Spider], err error) { + var ids []primitive.ObjectID + for _, id := range params.Ids { + _id, err := primitive.ObjectIDFromHex(id) + if err != nil { + return GetErrorResponse[models.Spider](errors.BadRequestf("invalid id format")) + } + ids = append(ids, _id) } // Fetch spiders before deletion spiders, err := service.NewModelService[models.Spider]().GetMany(bson.M{ "_id": bson.M{ - "$in": payload.Ids, + "$in": ids, }, }, nil) if err != nil { - HandleErrorInternalServerError(c, err) - return + return nil, err } if err := mongo2.RunTransaction(func(context mongo.SessionContext) (err error) { // delete spiders if err := service.NewModelService[models.Spider]().DeleteMany(bson.M{ "_id": bson.M{ - "$in": payload.Ids, + "$in": ids, }, }); err != nil { return err @@ -472,14 +444,14 @@ func DeleteSpiderList(c *gin.Context) { // delete spider stats if err := service.NewModelService[models.SpiderStat]().DeleteMany(bson.M{ "_id": bson.M{ - "$in": payload.Ids, + "$in": ids, }, }); err != nil { return err } // related tasks - tasks, err := service.NewModelService[models.Task]().GetMany(bson.M{"spider_id": bson.M{"$in": payload.Ids}}, nil) + tasks, err := service.NewModelService[models.Task]().GetMany(bson.M{"spider_id": bson.M{"$in": ids}}, nil) if err != nil { return err } @@ -521,8 +493,7 @@ func DeleteSpiderList(c *gin.Context) { return nil }); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Spider](err) } // Delete spider directories @@ -554,25 +525,25 @@ func DeleteSpiderList(c *gin.Context) { wg.Wait() }() - HandleSuccess(c) + return GetDataResponse(models.Spider{}) } -func GetSpiderListDir(c *gin.Context) { +func GetSpiderListDir(c *gin.Context, params *GetBaseFileListDirParams) (response *Response[[]interfaces.FsFileInfo], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { HandleErrorForbidden(c, err) return } - GetBaseFileListDir(rootPath, c) + return GetBaseFileListDir(rootPath, params) } -func GetSpiderFile(c *gin.Context) { +func GetSpiderFile(c *gin.Context, params *GetBaseFileFileParams) (response *Response[string], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { HandleErrorForbidden(c, err) return } - GetBaseFileFile(rootPath, c) + return GetBaseFileFile(rootPath, params) } func GetSpiderFileInfo(c *gin.Context) { @@ -648,18 +619,16 @@ func PostSpiderExport(c *gin.Context) { PostBaseFileExport(rootPath, c) } -func PostSpiderRun(c *gin.Context) { +func PostSpiderRun(c *gin.Context) (response *Response[[]primitive.ObjectID], err error) { id, err := primitive.ObjectIDFromHex(c.Param("id")) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](errors.BadRequestf("invalid id format")) } // options var opts interfaces.SpiderRunOptions if err := c.ShouldBindJSON(&opts); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } // user @@ -670,11 +639,11 @@ func PostSpiderRun(c *gin.Context) { // schedule tasks taskIds, err := admin.GetSpiderAdminService().Schedule(id, &opts) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } HandleSuccessWithData(c, taskIds) + return GetDataResponse(taskIds) } func GetSpiderResults(c *gin.Context) { @@ -733,17 +702,13 @@ func getSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsService, err er } func getSpiderRootPathByContext(c *gin.Context) (rootPath string, err error) { - // spider id id, err := primitive.ObjectIDFromHex(c.Param("id")) if err != nil { return "", err } - - // spider s, err := service.NewModelService[models.Spider]().GetById(id) if err != nil { return "", err } - return utils.GetSpiderRootPath(s) } diff --git a/core/controllers/spider_test.go b/core/controllers/spider_test.go index 10c2f4e0..d4614222 100644 --- a/core/controllers/spider_test.go +++ b/core/controllers/spider_test.go @@ -3,11 +3,13 @@ package controllers_test import ( "bytes" "encoding/json" - "github.com/crawlab-team/crawlab/core/models/models" "net/http" "net/http/httptest" "testing" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/loopfz/gadgeto/tonic" + "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/middlewares" "github.com/crawlab-team/crawlab/core/models/service" @@ -24,9 +26,9 @@ func TestCreateSpider(t *testing.T) { gin.SetMode(gin.TestMode) - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.POST("/spiders", controllers.PostSpider) + router.POST("/spiders", nil, tonic.Handler(controllers.PostSpider, 200)) payload := models.Spider{ Name: "Test Spider", @@ -54,9 +56,9 @@ func TestGetSpiderById(t *testing.T) { gin.SetMode(gin.TestMode) - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.GET("/spiders/:id", controllers.GetSpiderById) + router.GET("/spiders/:id", nil, tonic.Handler(controllers.GetSpiderById, 200)) model := models.Spider{ Name: "Test Spider", @@ -89,9 +91,9 @@ func TestUpdateSpiderById(t *testing.T) { gin.SetMode(gin.TestMode) - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.PUT("/spiders/:id", controllers.PutSpiderById) + router.PUT("/spiders/:id", nil, tonic.Handler(controllers.PutSpiderById, 200)) model := models.Spider{ Name: "Test Spider", @@ -110,7 +112,10 @@ func TestUpdateSpiderById(t *testing.T) { ColName: "test_spider", } payload.SetId(id) - jsonValue, _ := json.Marshal(payload) + requestBody := controllers.PutByIdParams[models.Spider]{ + Data: payload, + } + jsonValue, _ := json.Marshal(requestBody) req, _ := http.NewRequest("PUT", "/spiders/"+spiderId, bytes.NewBuffer(jsonValue)) req.Header.Set("Authorization", TestToken) resp := httptest.NewRecorder() @@ -136,9 +141,9 @@ func TestDeleteSpiderById(t *testing.T) { gin.SetMode(gin.TestMode) - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.DELETE("/spiders/:id", controllers.DeleteSpiderById) + router.DELETE("/spiders/:id", nil, tonic.Handler(controllers.DeleteSpiderById, 200)) model := models.Spider{ Name: "Test Spider", @@ -186,9 +191,9 @@ func TestDeleteSpiderList(t *testing.T) { gin.SetMode(gin.TestMode) - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.DELETE("/spiders", controllers.DeleteSpiderList) + router.DELETE("/spiders", nil, tonic.Handler(controllers.DeleteSpiderList, 200)) modelList := []models.Spider{ { diff --git a/core/controllers/user.go b/core/controllers/user.go index 2449b26b..184a0dd2 100644 --- a/core/controllers/user.go +++ b/core/controllers/user.go @@ -1,11 +1,11 @@ package controllers import ( - "errors" - "fmt" - "github.com/crawlab-team/crawlab/core/mongo" "regexp" + "github.com/crawlab-team/crawlab/core/mongo" + "github.com/juju/errors" + "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" @@ -15,34 +15,31 @@ import ( mongo2 "go.mongodb.org/mongo-driver/mongo" ) -func GetUserById(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +func GetUserById(c *gin.Context, params *GetByIdParams) (response *Response[models.User], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { HandleErrorBadRequest(c, err) return } - getUserById(id, c) + return getUserById(id) } -func GetUserList(c *gin.Context) { - // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - - // get users +func GetUserList(_ *gin.Context, params *GetListParams) (response *ListResponse[models.User], err error) { + query, err := GetFilterQueryFromListParams(params) + if err != nil { + return GetErrorListResponse[models.User](err) + } users, err := service.NewModelService[models.User]().GetMany(query, &mongo.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, + Sort: params.Sort, + Skip: params.Size * (params.Page - 1), + Limit: params.Size, }) if err != nil { if errors.Is(err, mongo2.ErrNoDocuments) { - HandleSuccessWithListData(c, nil, 0) + return GetListResponse[models.User](nil, 0) } else { - HandleErrorInternalServerError(c, err) + return GetErrorListResponse[models.User](err) } - return } // get roles @@ -58,8 +55,7 @@ func GetUserList(c *gin.Context) { "_id": bson.M{"$in": roleIds}, }, nil) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.User](err) } rolesMap := make(map[primitive.ObjectID]models.Role) for _, role := range roles { @@ -80,149 +76,124 @@ func GetUserList(c *gin.Context) { // total count total, err := service.NewModelService[models.User]().Count(query) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.User](err) } // response - HandleSuccessWithListData(c, users, total) + return GetListResponse[models.User](users, total) } -func PostUser(c *gin.Context) { - var payload struct { - Username string `json:"username"` - Password string `json:"password"` - Role string `json:"role"` - RoleId primitive.ObjectID `json:"role_id"` - Email string `json:"email"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } +type PostUserParams struct { + Username string `json:"username" validate:"required"` + Password string `json:"password" validate:"required"` + Role string `json:"role"` + RoleId primitive.ObjectID `json:"role_id"` + Email string `json:"email"` +} +func PostUser(c *gin.Context, params *PostUserParams) (response *Response[models.User], err error) { // Validate email format - if payload.Email != "" { + if params.Email != "" { emailRegex := regexp.MustCompile(`^[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}$`) - if !emailRegex.MatchString(payload.Email) { - HandleErrorBadRequest(c, fmt.Errorf("invalid email format")) - return + if !emailRegex.MatchString(params.Email) { + return GetErrorResponse[models.User](errors.BadRequestf("invalid email format")) } } - if !payload.RoleId.IsZero() { - _, err := service.NewModelService[models.Role]().GetById(payload.RoleId) + if !params.RoleId.IsZero() { + _, err := service.NewModelService[models.Role]().GetById(params.RoleId) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[models.User](errors.BadRequestf("role not found: %v", err)) } } u := GetUserFromContext(c) model := models.User{ - Username: payload.Username, - Password: utils.EncryptMd5(payload.Password), - Role: payload.Role, - RoleId: payload.RoleId, - Email: payload.Email, + Username: params.Username, + Password: utils.EncryptMd5(params.Password), + Role: params.Role, + RoleId: params.RoleId, + Email: params.Email, } model.SetCreated(u.Id) model.SetUpdated(u.Id) id, err := service.NewModelService[models.User]().InsertOne(model) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.User](err) } result, err := service.NewModelService[models.User]().GetById(id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.User](err) } - HandleSuccessWithData(c, result) + return GetDataResponse[models.User](*result) } -func PutUserById(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +func PutUserById(c *gin.Context, params *PutByIdParams[models.User]) (response *Response[models.User], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[models.User](errors.BadRequestf("invalid user id: %v", err)) } - putUser(id, c) + return putUser(id, GetUserFromContext(c).Id, params.Data) } -func PostUserChangePassword(c *gin.Context) { - // get id +type PostUserChangePasswordParams struct { + Id string `path:"id"` + Password string `json:"password" validate:"required"` +} + +func PostUserChangePassword(c *gin.Context, params *PostUserChangePasswordParams) (response *Response[models.User], err error) { id, err := primitive.ObjectIDFromHex(c.Param("id")) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[models.User](errors.BadRequestf("invalid user id: %v", err)) } - - postUserChangePassword(id, c) + return postUserChangePassword(id, GetUserFromContext(c).Id, params.Password) } -func DeleteUserById(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +func DeleteUserById(_ *gin.Context, params *DeleteByIdParams) (response *Response[models.User], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[models.User](errors.BadRequestf("invalid user id: %v", err)) } user, err := service.NewModelService[models.User]().GetById(id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.User](err) } if user.RootAdmin { - HandleErrorForbidden(c, errors.New("root admin cannot be deleted")) - return + return GetErrorResponse[models.User](errors.New("root admin cannot be deleted")) } if err := service.NewModelService[models.User]().DeleteById(id); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.User](err) } - HandleSuccess(c) + return GetDataResponse[models.User](models.User{}) } -func DeleteUserList(c *gin.Context) { - type Payload struct { - Ids []string `json:"ids"` - } - - var payload Payload - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - +func DeleteUserList(_ *gin.Context, params *DeleteListParams) (response *Response[models.User], err error) { // Convert string IDs to ObjectIDs var ids []primitive.ObjectID - for _, id := range payload.Ids { + for _, id := range params.Ids { objectId, err := primitive.ObjectIDFromHex(id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[models.User](errors.BadRequestf("invalid user id: %v", err)) } ids = append(ids, objectId) } // Check if root admin is in the list - _, err := service.NewModelService[models.User]().GetOne(bson.M{ + _, err = service.NewModelService[models.User]().GetOne(bson.M{ "_id": bson.M{ "$in": ids, }, "root_admin": true, }, nil) if err == nil { - HandleErrorForbidden(c, errors.New("root admin cannot be deleted")) - return + return GetErrorResponse[models.User](errors.New("root admin cannot be deleted")) } if !errors.Is(err, mongo2.ErrNoDocuments) { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.User](err) } // Delete users @@ -231,34 +202,43 @@ func DeleteUserList(c *gin.Context) { "$in": ids, }, }); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.User](err) } - HandleSuccess(c) + return GetDataResponse[models.User](models.User{}) } -func GetUserMe(c *gin.Context) { +func GetUserMe(c *gin.Context) (response *Response[models.User], err error) { u := GetUserFromContext(c) - getUserByIdWithRoutes(u.Id, c) + return getUserByIdWithRoutes(u.Id) } -func PutUserMe(c *gin.Context) { +type PutUserMeParams struct { + Data models.User `json:"data"` +} + +func PutUserMe(c *gin.Context, params *PutUserMeParams) (response *Response[models.User], err error) { u := GetUserFromContext(c) - putUser(u.Id, c) + return putUser(u.Id, u.Id, params.Data) } -func PostUserMeChangePassword(c *gin.Context) { +type PostUserMeChangePasswordParams struct { + Password string `json:"password" validate:"required"` +} + +func PostUserMeChangePassword(c *gin.Context, params *PostUserMeChangePasswordParams) (response *Response[models.User], err error) { u := GetUserFromContext(c) - postUserChangePassword(u.Id, c) + return postUserChangePassword(u.Id, u.Id, params.Password) } -func getUserById(userId primitive.ObjectID, c *gin.Context) { +func getUserById(userId primitive.ObjectID) (response *Response[models.User], err error) { // get user user, err := service.NewModelService[models.User]().GetById(userId) if err != nil { - HandleErrorInternalServerError(c, err) - return + if errors.Is(err, mongo2.ErrNoDocuments) { + return GetErrorResponse[models.User](errors.BadRequestf("user not found: %v", err)) + } + return GetErrorResponse[models.User](err) } // get role @@ -266,61 +246,58 @@ func getUserById(userId primitive.ObjectID, c *gin.Context) { if !user.RoleId.IsZero() { role, err := service.NewModelService[models.Role]().GetById(user.RoleId) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.User](errors.BadRequestf("role not found: %v", err)) } user.Role = role.Name user.RootAdminRole = role.RootAdmin } } - HandleSuccessWithData(c, user) + return GetDataResponse[models.User](*user) } -func getUserByIdWithRoutes(userId primitive.ObjectID, c *gin.Context) { +func getUserByIdWithRoutes(userId primitive.ObjectID) (response *Response[models.User], err error) { if !utils.IsPro() { - getUserById(userId, c) - return + return getUserById(userId) } // get user user, err := service.NewModelService[models.User]().GetById(userId) if err != nil { - HandleErrorInternalServerError(c, err) - return + if errors.Is(err, mongo2.ErrNoDocuments) { + return GetErrorResponse[models.User](errors.BadRequestf("user not found: %v", err)) + } + return GetErrorResponse[models.User](err) } // get role if !user.RoleId.IsZero() { role, err := service.NewModelService[models.Role]().GetById(user.RoleId) if err != nil { - HandleErrorInternalServerError(c, err) - return + if errors.Is(err, mongo2.ErrNoDocuments) { + return GetErrorResponse[models.User](errors.BadRequestf("role not found: %v", err)) + } + return GetErrorResponse[models.User](err) } user.Role = role.Name user.RootAdminRole = role.RootAdmin user.Routes = role.Routes } - HandleSuccessWithData(c, user) + return GetDataResponse[models.User](*user) } -func putUser(userId primitive.ObjectID, c *gin.Context) { - // get payload - var user models.User - if err := c.ShouldBindJSON(&user); err != nil { - HandleErrorBadRequest(c, err) - return - } - +func putUser(userId, by primitive.ObjectID, user models.User) (response *Response[models.User], err error) { // model service modelSvc := service.NewModelService[models.User]() // update user userDb, err := modelSvc.GetById(userId) if err != nil { - HandleErrorInternalServerError(c, err) - return + if errors.Is(err, mongo2.ErrNoDocuments) { + return GetErrorResponse[models.User](errors.BadRequestf("user not found: %v", err)) + } + return GetErrorResponse[models.User](err) } // if root admin, disallow changing username and role @@ -332,53 +309,34 @@ func putUser(userId primitive.ObjectID, c *gin.Context) { // disallow changing password user.Password = userDb.Password - // current user - u := GetUserFromContext(c) - // update user - user.SetUpdated(u.Id) + user.SetUpdated(by) if user.Id.IsZero() { user.Id = userId } if err := modelSvc.ReplaceById(userId, user); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.User](err) } // handle success - HandleSuccess(c) + return GetDataResponse[models.User](user) } -func postUserChangePassword(userId primitive.ObjectID, c *gin.Context) { - // get payload - var payload struct { - Password string `json:"password"` +func postUserChangePassword(userId, by primitive.ObjectID, password string) (response *Response[models.User], err error) { + if len(password) < 5 { + return GetErrorResponse[models.User](errors.BadRequestf("password must be at least 5 characters")) } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - if len(payload.Password) < 5 { - HandleErrorBadRequest(c, errors.New("password must be at least 5 characters")) - return - } - - // current user - u := GetUserFromContext(c) // update password userDb, err := service.NewModelService[models.User]().GetById(userId) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.User](err) } - userDb.SetUpdated(u.Id) - userDb.Password = utils.EncryptMd5(payload.Password) + userDb.SetUpdated(by) + userDb.Password = utils.EncryptMd5(password) if err := service.NewModelService[models.User]().ReplaceById(userDb.Id, *userDb); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.User](err) } - // handle success - HandleSuccess(c) + return GetDataResponse[models.User](models.User{}) } diff --git a/core/controllers/user_test.go b/core/controllers/user_test.go index 075bd9bd..45b4e2fb 100644 --- a/core/controllers/user_test.go +++ b/core/controllers/user_test.go @@ -1,6 +1,8 @@ package controllers_test import ( + "bytes" + "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -12,7 +14,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/user" - "github.com/gin-gonic/gin" + "github.com/loopfz/gadgeto/tonic" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.mongodb.org/mongo-driver/bson" @@ -36,9 +38,9 @@ func TestGetUserById_Success(t *testing.T) { require.Nil(t, err) u.SetId(id) - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.GET("/users/:id", controllers.GetUserById) + router.GET("/users/:id", nil, tonic.Handler(controllers.GetUserById, 200)) // Test valid ID req, err := http.NewRequest(http.MethodGet, "/users/"+id.Hex(), nil) @@ -79,9 +81,9 @@ func TestGetUserList_Success(t *testing.T) { assert.Nil(t, err) } - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.GET("/users", controllers.GetUserList) + router.GET("/users", nil, tonic.Handler(controllers.GetUserList, 200)) // Test default pagination req, err := http.NewRequest(http.MethodGet, "/users", nil) @@ -108,9 +110,9 @@ func TestPostUser_Success(t *testing.T) { SetupTestDB() defer CleanupTestDB() - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.POST("/users", controllers.PostUser) + router.POST("/users", nil, tonic.Handler(controllers.PostUser, 200)) // Test creating a new user with valid data reqBody := strings.NewReader(`{ @@ -161,9 +163,9 @@ func TestPutUserById_Success(t *testing.T) { require.Nil(t, err) u.SetId(id) - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.PUT("/users/:id", controllers.PutUserById) + router.PUT("/users/:id", nil, tonic.Handler(controllers.PutUserById, 200)) // Test case 1: Regular user update reqBody := strings.NewReader(`{ @@ -214,9 +216,9 @@ func TestPostUserChangePassword_Success(t *testing.T) { require.Nil(t, err) u.SetId(id) - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.POST("/users/:id/change-password", controllers.PostUserChangePassword) + router.POST("/users/:id/change-password", nil, tonic.Handler(controllers.PostUserChangePassword, 200)) // Add validation for minimum password length // Test case 1: Valid password @@ -252,9 +254,9 @@ func TestGetUserMe_Success(t *testing.T) { require.Nil(t, err) u.SetId(id) - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.GET("/users/me", controllers.GetUserMe) + router.GET("/users/me", nil, tonic.Handler(controllers.GetUserMe, 200)) req, _ := http.NewRequest(http.MethodGet, "/users/me", nil) req.Header.Set("Content-Type", "application/json") @@ -288,23 +290,26 @@ func TestPutUserMe_Success(t *testing.T) { require.Nil(t, err) // Create router - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.PUT("/users/me", controllers.PutUserMe) + router.PUT("/users/me", nil, tonic.Handler(controllers.PutUserMe, 200)) // Test valid update - reqBody := strings.NewReader(`{ - "username": "updateduser", - "email": "updated@example.com" - }`) - req, err := http.NewRequest(http.MethodPut, "/users/me", reqBody) + reqParams := controllers.PutUserMeParams{ + Data: models.User{ + Username: "updateduser", + Email: "updated@example.com", + }, + } + jsonValue, _ := json.Marshal(reqParams) + req, err := http.NewRequest(http.MethodPut, "/users/me", bytes.NewBuffer(jsonValue)) assert.Nil(t, err) req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", token) w := httptest.NewRecorder() router.ServeHTTP(w, req) - assert.Equal(t, http.StatusOK, w.Code) + assert.Equalf(t, http.StatusOK, w.Code, "response body: %s", w.Body.String()) // Verify the update updatedUser, err := modelSvc.GetById(id) @@ -338,9 +343,9 @@ func TestPostUserMeChangePassword_Success(t *testing.T) { require.Nil(t, err) // Create router - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.POST("/users/me/change-password", controllers.PostUserMeChangePassword) + router.POST("/users/me/change-password", nil, tonic.Handler(controllers.PostUserMeChangePassword, 200)) // Test valid password change password := "newValidPassword123" @@ -388,9 +393,9 @@ func TestDeleteUserById_Success(t *testing.T) { require.Nil(t, err) u.SetId(id) - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.DELETE("/users/:id", controllers.DeleteUserById) + router.DELETE("/users/:id", nil, tonic.Handler(controllers.DeleteUserById, 200)) // Test deleting normal user req, err := http.NewRequest(http.MethodDelete, "/users/"+id.Hex(), nil) @@ -423,7 +428,7 @@ func TestDeleteUserById_Success(t *testing.T) { w = httptest.NewRecorder() router.ServeHTTP(w, req) - assert.Equal(t, http.StatusForbidden, w.Code) + assert.Equalf(t, http.StatusBadRequest, w.Code, "response body: %s", w.Body.String()) // Test deleting with invalid ID req, err = http.NewRequest(http.MethodDelete, "/users/invalid-id", nil) @@ -460,9 +465,9 @@ func TestDeleteUserList_Success(t *testing.T) { } } - router := gin.Default() + router := SetupRouter() router.Use(middlewares.AuthorizationMiddleware()) - router.DELETE("/users", controllers.DeleteUserList) + router.DELETE("/users", nil, tonic.Handler(controllers.DeleteUserList, 200)) // Test deleting normal users reqBody := strings.NewReader(fmt.Sprintf(`{"ids":["%s","%s"]}`, @@ -492,7 +497,7 @@ func TestDeleteUserList_Success(t *testing.T) { w = httptest.NewRecorder() router.ServeHTTP(w, req) - assert.Equal(t, http.StatusForbidden, w.Code) + assert.Equalf(t, http.StatusBadRequest, w.Code, "response body: %s", w.Body.String()) // Test with mix of valid and invalid ids reqBody = strings.NewReader(fmt.Sprintf(`{"ids":["%s","invalid-id"]}`, normalUserIds[0].Hex())) diff --git a/core/controllers/utils.go b/core/controllers/utils.go new file mode 100644 index 00000000..cca52bac --- /dev/null +++ b/core/controllers/utils.go @@ -0,0 +1,331 @@ +package controllers + +import ( + "encoding/json" + "errors" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/mongo" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/crawlab/trace" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "net/http" + "reflect" +) + +var logger = utils.NewLogger("Controllers") + +func GetUserFromContext(c *gin.Context) (u *models.User) { + value, ok := c.Get(constants.UserContextKey) + if !ok { + return nil + } + u, ok = value.(*models.User) + if !ok { + return nil + } + return u +} + +func GetFilterQueryFromListParams(params *GetListParams) (q bson.M, err error) { + if params.Conditions == "" { + return nil, nil + } + conditions, err := GetFilterFromConditionString(params.Conditions) + if err != nil { + return nil, err + } + return utils.FilterToQuery(conditions) +} + +func GetFilterQueryFromConditionString(condStr string) (q bson.M, err error) { + conditions, err := GetFilterFromConditionString(condStr) + if err != nil { + return nil, err + } + return utils.FilterToQuery(conditions) +} + +func GetFilterFromConditionString(condStr string) (f *entity.Filter, err error) { + var conditions []*entity.Condition + if err := json.Unmarshal([]byte(condStr), &conditions); err != nil { + return nil, err + } + + for i, cond := range conditions { + v := reflect.ValueOf(cond.Value) + switch v.Kind() { + case reflect.String: + item := cond.Value.(string) + // attempt to convert object id + id, err := primitive.ObjectIDFromHex(item) + if err == nil { + conditions[i].Value = id + } else { + conditions[i].Value = item + } + case reflect.Slice, reflect.Array: + var items []interface{} + for i := 0; i < v.Len(); i++ { + vItem := v.Index(i) + item := vItem.Interface() + + // string + stringItem, ok := item.(string) + if ok { + id, err := primitive.ObjectIDFromHex(stringItem) + if err == nil { + items = append(items, id) + } else { + items = append(items, stringItem) + } + continue + } + + // default + items = append(items, item) + } + conditions[i].Value = items + default: + return nil, errors.New("invalid type") + } + } + + return &entity.Filter{ + IsOr: false, + Conditions: conditions, + }, nil +} + +// GetFilter Get entity.Filter from gin.Context +func GetFilter(c *gin.Context) (f *entity.Filter, err error) { + condStr := c.Query(constants.FilterQueryFieldConditions) + return GetFilterFromConditionString(condStr) +} + +// GetFilterQuery Get bson.M from gin.Context +func GetFilterQuery(c *gin.Context) (q bson.M, err error) { + f, err := GetFilter(c) + if err != nil { + return nil, err + } + + if f == nil { + return nil, nil + } + + // TODO: implement logic OR + + return utils.FilterToQuery(f) +} + +func MustGetFilterQuery(c *gin.Context) (q bson.M) { + q, err := GetFilterQuery(c) + if err != nil { + return nil + } + return q +} + +func getResultListQuery(c *gin.Context) (q mongo.ListQuery) { + f, err := GetFilter(c) + if err != nil { + return q + } + for _, cond := range f.Conditions { + q = append(q, mongo.ListQueryCondition{ + Key: cond.Key, + Op: cond.Op, + Value: utils.NormalizeObjectId(cond.Value), + }) + } + return q +} + +func GetDefaultPagination() (p *entity.Pagination) { + return &entity.Pagination{ + Page: constants.PaginationDefaultPage, + Size: constants.PaginationDefaultSize, + } +} + +func GetPagination(c *gin.Context) (p *entity.Pagination, err error) { + var _p entity.Pagination + if err := c.ShouldBindQuery(&_p); err != nil { + return GetDefaultPagination(), err + } + if _p.Page == 0 { + _p.Page = constants.PaginationDefaultPage + } + if _p.Size == 0 { + _p.Size = constants.PaginationDefaultSize + } + return &_p, nil +} + +func MustGetPagination(c *gin.Context) (p *entity.Pagination) { + p, err := GetPagination(c) + if err != nil || p == nil { + return GetDefaultPagination() + } + return p +} + +// GetSorts Get entity.Sort from gin.Context +func GetSorts(c *gin.Context) (sorts []entity.Sort, err error) { + // bind + sortStr := c.Query(constants.SortQueryField) + if err := json.Unmarshal([]byte(sortStr), &sorts); err != nil { + return nil, err + } + return sorts, nil +} + +// GetSortsOption Get entity.Sort from gin.Context +func GetSortsOption(c *gin.Context) (sort bson.D, err error) { + sorts, err := GetSorts(c) + if err != nil { + return nil, err + } + + if sorts == nil || len(sorts) == 0 { + return bson.D{{"_id", -1}}, nil + } + + return SortsToOption(sorts) +} + +func MustGetSortOption(c *gin.Context) (sort bson.D) { + sort, err := GetSortsOption(c) + if err != nil { + return nil + } + return sort +} + +// SortsToOption Translate entity.Sort to bson.D +func SortsToOption(sorts []entity.Sort) (sort bson.D, err error) { + sort = bson.D{} + for _, s := range sorts { + switch s.Direction { + case constants.ASCENDING: + sort = append(sort, bson.E{Key: s.Key, Value: 1}) + case constants.DESCENDING: + sort = append(sort, bson.E{Key: s.Key, Value: -1}) + } + } + if len(sort) == 0 { + sort = bson.D{{"_id", -1}} + } + return sort, nil +} + +type Response[T any] struct { + Status string `json:"status"` + Message string `json:"message"` + Data T `json:"data"` + Error string `json:"error"` +} + +type ListResponse[T any] struct { + Status string `json:"status"` + Message string `json:"message"` + Total int `json:"total"` + Data []T `json:"data"` + Error string `json:"error"` +} + +func GetDataResponse[T any](model T) (res *Response[T], err error) { + return &Response[T]{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageSuccess, + Data: model, + }, nil +} + +func GetListResponse[T any](models []T, total int) (res *ListResponse[T], err error) { + return &ListResponse[T]{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageSuccess, + Data: models, + Total: total, + }, nil +} + +func GetErrorResponse[T any](err error) (res *Response[T], err2 error) { + return &Response[T]{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageError, + Error: err.Error(), + }, err +} + +func GetErrorListResponse[T any](err error) (res *ListResponse[T], err2 error) { + return &ListResponse[T]{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageError, + Error: err.Error(), + }, err +} + +func handleError(statusCode int, c *gin.Context, err error) { + if utils.IsDev() { + trace.PrintError(err) + } + c.AbortWithStatusJSON(statusCode, entity.Response{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageError, + Error: err.Error(), + }) +} + +func HandleError(statusCode int, c *gin.Context, err error) { + handleError(statusCode, c, err) +} + +func HandleErrorBadRequest(c *gin.Context, err error) { + HandleError(http.StatusBadRequest, c, err) +} + +func HandleErrorForbidden(c *gin.Context, err error) { + HandleError(http.StatusForbidden, c, err) +} + +func HandleErrorUnauthorized(c *gin.Context, err error) { + HandleError(http.StatusUnauthorized, c, err) +} + +func HandleErrorNotFound(c *gin.Context, err error) { + HandleError(http.StatusNotFound, c, err) +} + +func HandleErrorInternalServerError(c *gin.Context, err error) { + HandleError(http.StatusInternalServerError, c, err) +} + +func HandleSuccess(c *gin.Context) { + c.AbortWithStatusJSON(http.StatusOK, entity.Response{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageSuccess, + }) +} + +func HandleSuccessWithData(c *gin.Context, data interface{}) { + c.AbortWithStatusJSON(http.StatusOK, entity.Response{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageSuccess, + Data: data, + }) +} + +func HandleSuccessWithListData(c *gin.Context, data interface{}, total int) { + c.AbortWithStatusJSON(http.StatusOK, entity.ListResponse{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageSuccess, + Data: data, + Total: total, + }) +} diff --git a/core/controllers/utils_context.go b/core/controllers/utils_context.go deleted file mode 100644 index 506c7549..00000000 --- a/core/controllers/utils_context.go +++ /dev/null @@ -1,19 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/gin-gonic/gin" -) - -func GetUserFromContext(c *gin.Context) (u *models.User) { - value, ok := c.Get(constants.UserContextKey) - if !ok { - return nil - } - u, ok = value.(*models.User) - if !ok { - return nil - } - return u -} diff --git a/core/controllers/utils_filter.go b/core/controllers/utils_filter.go deleted file mode 100644 index 7bc5e168..00000000 --- a/core/controllers/utils_filter.go +++ /dev/null @@ -1,141 +0,0 @@ -package controllers - -import ( - "encoding/json" - errors2 "errors" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/mongo" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "reflect" - "strings" -) - -// GetFilter Get entity.Filter from gin.Context -func GetFilter(c *gin.Context) (f *entity.Filter, err error) { - // bind - condStr := c.Query(constants.FilterQueryFieldConditions) - var conditions []*entity.Condition - if err := json.Unmarshal([]byte(condStr), &conditions); err != nil { - return nil, err - } - - // attempt to convert object id - for i, cond := range conditions { - v := reflect.ValueOf(cond.Value) - switch v.Kind() { - case reflect.String: - item := cond.Value.(string) - id, err := primitive.ObjectIDFromHex(item) - if err == nil { - conditions[i].Value = id - } else { - conditions[i].Value = item - } - case reflect.Slice, reflect.Array: - var items []interface{} - for i := 0; i < v.Len(); i++ { - vItem := v.Index(i) - item := vItem.Interface() - - // string - stringItem, ok := item.(string) - if ok { - id, err := primitive.ObjectIDFromHex(stringItem) - if err == nil { - items = append(items, id) - } else { - items = append(items, stringItem) - } - continue - } - - // default - items = append(items, item) - } - conditions[i].Value = items - default: - return nil, errors2.New("invalid type") - } - } - - return &entity.Filter{ - IsOr: false, - Conditions: conditions, - }, nil -} - -// GetFilterQuery Get bson.M from gin.Context -func GetFilterQuery(c *gin.Context) (q bson.M, err error) { - f, err := GetFilter(c) - if err != nil { - return nil, err - } - - if f == nil { - return nil, nil - } - - // TODO: implement logic OR - - return utils.FilterToQuery(f) -} - -func MustGetFilterQuery(c *gin.Context) (q bson.M) { - q, err := GetFilterQuery(c) - if err != nil { - return nil - } - return q -} - -// GetFilterAll Get all from gin.Context -func GetFilterAll(c *gin.Context) (res bool, err error) { - resStr := c.Query(constants.FilterQueryFieldAll) - switch strings.ToUpper(resStr) { - case "1": - return true, nil - case "0": - return false, nil - case "Y": - return true, nil - case "N": - return false, nil - case "T": - return true, nil - case "F": - return false, nil - case "TRUE": - return true, nil - case "FALSE": - return false, nil - default: - return false, errors2.New("invalid value") - } -} - -func MustGetFilterAll(c *gin.Context) (res bool) { - res, err := GetFilterAll(c) - if err != nil { - return false - } - return res -} - -func getResultListQuery(c *gin.Context) (q mongo.ListQuery) { - f, err := GetFilter(c) - if err != nil { - return q - } - for _, cond := range f.Conditions { - q = append(q, mongo.ListQueryCondition{ - Key: cond.Key, - Op: cond.Op, - Value: utils.NormalizeObjectId(cond.Value), - }) - } - return q -} diff --git a/core/controllers/utils_http.go b/core/controllers/utils_http.go deleted file mode 100644 index 54ac2987..00000000 --- a/core/controllers/utils_http.go +++ /dev/null @@ -1,110 +0,0 @@ -package controllers - -import ( - "net/http" - - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" -) - -type Response[T any] struct { - Status string `json:"status"` - Message string `json:"message"` - Data T `json:"data"` - Error string `json:"error"` -} - -type ListResponse[T any] struct { - Status string `json:"status"` - Message string `json:"message"` - Total int `json:"total"` - Data []T `json:"data"` - Error string `json:"error"` -} - -func GetSuccessDataResponse[T any](model T) (res *Response[T], err error) { - return &Response[T]{ - Status: constants.HttpResponseStatusOk, - Message: constants.HttpResponseMessageSuccess, - Data: model, - }, nil -} - -func GetSuccessListResponse[T any](models []T, total int) (res *ListResponse[T], err error) { - return &ListResponse[T]{ - Status: constants.HttpResponseStatusOk, - Message: constants.HttpResponseMessageSuccess, - Data: models, - Total: total, - }, nil -} - -func GetErrorDataResponse[T any](err error) (res *Response[T], err2 error) { - return &Response[T]{ - Status: constants.HttpResponseStatusOk, - Message: constants.HttpResponseMessageError, - Error: err.Error(), - }, err -} - -func handleError(statusCode int, c *gin.Context, err error) { - if utils.IsDev() { - trace.PrintError(err) - } - c.AbortWithStatusJSON(statusCode, entity.Response{ - Status: constants.HttpResponseStatusOk, - Message: constants.HttpResponseMessageError, - Error: err.Error(), - }) -} - -func HandleError(statusCode int, c *gin.Context, err error) { - handleError(statusCode, c, err) -} - -func HandleErrorBadRequest(c *gin.Context, err error) { - HandleError(http.StatusBadRequest, c, err) -} - -func HandleErrorForbidden(c *gin.Context, err error) { - HandleError(http.StatusForbidden, c, err) -} - -func HandleErrorUnauthorized(c *gin.Context, err error) { - HandleError(http.StatusUnauthorized, c, err) -} - -func HandleErrorNotFound(c *gin.Context, err error) { - HandleError(http.StatusNotFound, c, err) -} - -func HandleErrorInternalServerError(c *gin.Context, err error) { - HandleError(http.StatusInternalServerError, c, err) -} - -func HandleSuccess(c *gin.Context) { - c.AbortWithStatusJSON(http.StatusOK, entity.Response{ - Status: constants.HttpResponseStatusOk, - Message: constants.HttpResponseMessageSuccess, - }) -} - -func HandleSuccessWithData(c *gin.Context, data interface{}) { - c.AbortWithStatusJSON(http.StatusOK, entity.Response{ - Status: constants.HttpResponseStatusOk, - Message: constants.HttpResponseMessageSuccess, - Data: data, - }) -} - -func HandleSuccessWithListData(c *gin.Context, data interface{}, total int) { - c.AbortWithStatusJSON(http.StatusOK, entity.ListResponse{ - Status: constants.HttpResponseStatusOk, - Message: constants.HttpResponseMessageSuccess, - Data: data, - Total: total, - }) -} diff --git a/core/controllers/utils_logger.go b/core/controllers/utils_logger.go deleted file mode 100644 index 002ff743..00000000 --- a/core/controllers/utils_logger.go +++ /dev/null @@ -1,5 +0,0 @@ -package controllers - -import "github.com/crawlab-team/crawlab/core/utils" - -var logger = utils.NewLogger("Controllers") diff --git a/core/controllers/utils_pagination.go b/core/controllers/utils_pagination.go deleted file mode 100644 index 4f148ecf..00000000 --- a/core/controllers/utils_pagination.go +++ /dev/null @@ -1,36 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/gin-gonic/gin" -) - -func GetDefaultPagination() (p *entity.Pagination) { - return &entity.Pagination{ - Page: constants.PaginationDefaultPage, - Size: constants.PaginationDefaultSize, - } -} - -func GetPagination(c *gin.Context) (p *entity.Pagination, err error) { - var _p entity.Pagination - if err := c.ShouldBindQuery(&_p); err != nil { - return GetDefaultPagination(), err - } - if _p.Page == 0 { - _p.Page = constants.PaginationDefaultPage - } - if _p.Size == 0 { - _p.Size = constants.PaginationDefaultSize - } - return &_p, nil -} - -func MustGetPagination(c *gin.Context) (p *entity.Pagination) { - p, err := GetPagination(c) - if err != nil || p == nil { - return GetDefaultPagination() - } - return p -} diff --git a/core/controllers/utils_sort.go b/core/controllers/utils_sort.go deleted file mode 100644 index 86e68e96..00000000 --- a/core/controllers/utils_sort.go +++ /dev/null @@ -1,58 +0,0 @@ -package controllers - -import ( - "encoding/json" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" -) - -// GetSorts Get entity.Sort from gin.Context -func GetSorts(c *gin.Context) (sorts []entity.Sort, err error) { - // bind - sortStr := c.Query(constants.SortQueryField) - if err := json.Unmarshal([]byte(sortStr), &sorts); err != nil { - return nil, err - } - return sorts, nil -} - -// GetSortsOption Get entity.Sort from gin.Context -func GetSortsOption(c *gin.Context) (sort bson.D, err error) { - sorts, err := GetSorts(c) - if err != nil { - return nil, err - } - - if sorts == nil || len(sorts) == 0 { - return bson.D{{"_id", -1}}, nil - } - - return SortsToOption(sorts) -} - -func MustGetSortOption(c *gin.Context) (sort bson.D) { - sort, err := GetSortsOption(c) - if err != nil { - return nil - } - return sort -} - -// SortsToOption Translate entity.Sort to bson.D -func SortsToOption(sorts []entity.Sort) (sort bson.D, err error) { - sort = bson.D{} - for _, s := range sorts { - switch s.Direction { - case constants.ASCENDING: - sort = append(sort, bson.E{Key: s.Key, Value: 1}) - case constants.DESCENDING: - sort = append(sort, bson.E{Key: s.Key, Value: -1}) - } - } - if len(sort) == 0 { - sort = bson.D{{"_id", -1}} - } - return sort, nil -} diff --git a/core/controllers/ws_writer.go b/core/controllers/ws_writer.go deleted file mode 100644 index dcc096c3..00000000 --- a/core/controllers/ws_writer.go +++ /dev/null @@ -1,54 +0,0 @@ -package controllers - -import ( - "github.com/gin-gonic/gin" - "github.com/gorilla/websocket" - "io" - http2 "net/http" -) - -type WsWriter struct { - io.Writer - io.Closer - conn *websocket.Conn -} - -func (w *WsWriter) Write(data []byte) (n int, err error) { - logger.Infof("websocket write: %s", string(data)) - err = w.conn.WriteMessage(websocket.TextMessage, data) - if err != nil { - return 0, err - } - return len(data), nil -} - -func (w *WsWriter) Close() (err error) { - return w.conn.Close() -} - -func (w *WsWriter) CloseWithText(text string) { - _ = w.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, text)) -} - -func (w *WsWriter) CloseWithError(err error) { - _ = w.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseInternalServerErr, err.Error())) -} - -func NewWsWriter(c *gin.Context) (writer *WsWriter, err error) { - upgrader := websocket.Upgrader{ - ReadBufferSize: 1024, - WriteBufferSize: 1024, - CheckOrigin: func(r *http2.Request) bool { - return true - }, - } - - conn, err := upgrader.Upgrade(c.Writer, c.Request, nil) - if err != nil { - logger.Errorf("websocket open connection error: %v", err) - } - - return &WsWriter{ - conn: conn, - }, nil -} diff --git a/core/go.mod b/core/go.mod index c44dee01..4f0f91f6 100644 --- a/core/go.mod +++ b/core/go.mod @@ -1,11 +1,12 @@ module github.com/crawlab-team/crawlab/core -go 1.23 +go 1.23.7 replace ( github.com/crawlab-team/crawlab/grpc => ../grpc github.com/crawlab-team/crawlab/trace => ../trace github.com/crawlab-team/crawlab/vcs => ../vcs + github.com/wI2L/fizz => github.com/crawlab-team/fizz v0.0.0-20250312082934-630a98f9d791 ) require ( @@ -22,22 +23,22 @@ require ( github.com/golang-jwt/jwt/v5 v5.2.1 github.com/gomarkdown/markdown v0.0.0-20241105142532-d03b89096d81 github.com/google/uuid v1.6.0 - github.com/gorilla/websocket v1.5.3 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-uuid v1.0.3 github.com/juju/errors v1.0.0 + github.com/loopfz/gadgeto v0.9.0 github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.0 github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/cobra v1.3.0 github.com/spf13/viper v1.19.0 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/wI2L/fizz v0.22.0 go.mongodb.org/mongo-driver v1.15.1 golang.org/x/oauth2 v0.23.0 - golang.org/x/text v0.21.0 + golang.org/x/text v0.23.0 google.golang.org/api v0.189.0 google.golang.org/grpc v1.69.2 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df @@ -55,16 +56,16 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect - github.com/bytedance/sonic v1.11.6 // indirect - github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/bytedance/sonic v1.13.1 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/cloudflare/circl v1.3.7 // indirect - github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/gabriel-vasile/mimetype v1.4.3 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.8 // indirect + github.com/gin-contrib/sse v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-git/go-git/v5 v5.12.0 // indirect @@ -73,8 +74,8 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.20.0 // indirect - github.com/goccy/go-json v0.10.2 // indirect + github.com/go-playground/validator/v10 v10.25.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/gofrs/uuid v3.2.0+incompatible // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect @@ -88,16 +89,15 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.17.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/loopfz/gadgeto v0.9.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -127,14 +127,14 @@ require ( go.opentelemetry.io/otel/trace v1.31.0 // indirect go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/arch v0.8.0 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/arch v0.15.0 // indirect + golang.org/x/crypto v0.36.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.28.0 // indirect + golang.org/x/net v0.37.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 // indirect - google.golang.org/protobuf v1.36.1 // indirect + google.golang.org/protobuf v1.36.5 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/go-playground/validator.v9 v9.30.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/core/go.sum b/core/go.sum index 0c7649e5..33aa2bc2 100644 --- a/core/go.sum +++ b/core/go.sum @@ -99,8 +99,12 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic v1.13.1 h1:Jyd5CIvdFnkOWuKXr+wm4Nyk2h0yAFsr8ucJgEasO3g= +github.com/bytedance/sonic v1.13.1/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -119,6 +123,8 @@ github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vc github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -136,6 +142,8 @@ github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crawlab-team/fizz v0.0.0-20250312082934-630a98f9d791 h1:K849MMP329dseKSpMqX3XDTccpWxghMwWyrDclKzkdM= +github.com/crawlab-team/fizz v0.0.0-20250312082934-630a98f9d791/go.mod h1:CMxMR1amz8id9wr2YUpONf+F/F9hW1cqRXxVNNuWVxE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= @@ -171,12 +179,16 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= +github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/cors v1.3.0/go.mod h1:artPvLlhkF7oG06nK8v3U8TNz6IeX+w1uzCSEId5/Vc= github.com/gin-contrib/sse v0.0.0-20190125020943-a7658810eb74/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= +github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= @@ -223,9 +235,13 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.25.0 h1:5Dh7cjvzR7BRZadnsVOzPhWsrwUr0nmsZJxEAnFLNO8= +github.com/go-playground/validator/v10 v10.25.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -323,8 +339,6 @@ github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pf github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -383,6 +397,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/errors v1.0.0 h1:yiq7kjCLll1BiaRuNY53MGI0+EQ3rF6GB+wvboZDefM= +github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5Qe8= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -395,6 +411,8 @@ github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ib github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -471,6 +489,8 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -561,6 +581,7 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= @@ -587,8 +608,6 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/wI2L/fizz v0.22.0 h1:mgRA+uUdESvgsIeBFkMSS/MEIQ4EZ4I2xyRxnCqkhJY= -github.com/wI2L/fizz v0.22.0/go.mod h1:CMxMR1amz8id9wr2YUpONf+F/F9hW1cqRXxVNNuWVxE= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -647,6 +666,8 @@ go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= +golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -666,6 +687,8 @@ golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2Uz golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -756,6 +779,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -790,6 +815,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -871,6 +898,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -879,6 +908,7 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -895,6 +925,8 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1112,6 +1144,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= diff --git a/core/models/models/base.go b/core/models/models/base.go index 1bdb06a6..96f278ed 100644 --- a/core/models/models/base.go +++ b/core/models/models/base.go @@ -7,7 +7,7 @@ import ( "go.mongodb.org/mongo-driver/bson/primitive" ) -type BaseModel[T any] struct { +type BaseModel struct { Id primitive.ObjectID `json:"_id" bson:"_id"` CreatedAt time.Time `json:"created_ts,omitempty" bson:"created_ts,omitempty"` CreatedBy primitive.ObjectID `json:"created_by,omitempty" bson:"created_by,omitempty"` @@ -15,52 +15,52 @@ type BaseModel[T any] struct { UpdatedBy primitive.ObjectID `json:"updated_by,omitempty" bson:"updated_by,omitempty"` } -func (m *BaseModel[T]) GetId() primitive.ObjectID { +func (m *BaseModel) GetId() primitive.ObjectID { return m.Id } -func (m *BaseModel[T]) SetId(id primitive.ObjectID) { +func (m *BaseModel) SetId(id primitive.ObjectID) { m.Id = id } -func (m *BaseModel[T]) GetCreatedAt() time.Time { +func (m *BaseModel) GetCreatedAt() time.Time { return m.CreatedAt } -func (m *BaseModel[T]) SetCreatedAt(t time.Time) { +func (m *BaseModel) SetCreatedAt(t time.Time) { m.CreatedAt = t } -func (m *BaseModel[T]) GetCreatedBy() primitive.ObjectID { +func (m *BaseModel) GetCreatedBy() primitive.ObjectID { return m.CreatedBy } -func (m *BaseModel[T]) SetCreatedBy(id primitive.ObjectID) { +func (m *BaseModel) SetCreatedBy(id primitive.ObjectID) { m.CreatedBy = id } -func (m *BaseModel[T]) GetUpdatedAt() time.Time { +func (m *BaseModel) GetUpdatedAt() time.Time { return m.UpdatedAt } -func (m *BaseModel[T]) SetUpdatedAt(t time.Time) { +func (m *BaseModel) SetUpdatedAt(t time.Time) { m.UpdatedAt = t } -func (m *BaseModel[T]) GetUpdatedBy() primitive.ObjectID { +func (m *BaseModel) GetUpdatedBy() primitive.ObjectID { return m.UpdatedBy } -func (m *BaseModel[T]) SetUpdatedBy(id primitive.ObjectID) { +func (m *BaseModel) SetUpdatedBy(id primitive.ObjectID) { m.UpdatedBy = id } -func (m *BaseModel[T]) SetCreated(id primitive.ObjectID) { +func (m *BaseModel) SetCreated(id primitive.ObjectID) { m.SetCreatedAt(time.Now()) m.SetCreatedBy(id) } -func (m *BaseModel[T]) SetUpdated(id primitive.ObjectID) { +func (m *BaseModel) SetUpdated(id primitive.ObjectID) { m.SetUpdatedAt(time.Now()) m.SetUpdatedBy(id) } diff --git a/core/models/models/data_collection.go b/core/models/models/data_collection.go index 587b5db9..89cfbb5a 100644 --- a/core/models/models/data_collection.go +++ b/core/models/models/data_collection.go @@ -5,11 +5,11 @@ import ( ) type DataCollection struct { - any `collection:"data_collections"` - BaseModel[DataCollection] `bson:",inline"` - Name string `json:"name" bson:"name"` - Fields []entity.DataField `json:"fields" bson:"fields"` - Dedup struct { + any `collection:"data_collections"` + BaseModel `bson:",inline"` + Name string `json:"name" bson:"name"` + Fields []entity.DataField `json:"fields" bson:"fields"` + Dedup struct { Enabled bool `json:"enabled" bson:"enabled"` Keys []string `json:"keys" bson:"keys"` Type string `json:"type" bson:"type"` diff --git a/core/models/models/database.go b/core/models/models/database.go index c510b0f7..a4faa23e 100644 --- a/core/models/models/database.go +++ b/core/models/models/database.go @@ -5,23 +5,23 @@ import ( ) type Database struct { - any `collection:"databases"` - BaseModel[Database] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - DataSource string `json:"data_source" bson:"data_source"` - Host string `json:"host" bson:"host"` - Port int `json:"port" bson:"port"` - URI string `json:"uri,omitempty" bson:"uri,omitempty"` - Database string `json:"database,omitempty" bson:"database,omitempty"` - Username string `json:"username,omitempty" bson:"username,omitempty"` - Password string `json:"password,omitempty" bson:"-"` - EncryptedPassword string `json:"-,omitempty" bson:"encrypted_password,omitempty"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Active bool `json:"active" bson:"active"` - ActiveAt time.Time `json:"active_ts" bson:"active_ts"` - IsDefault bool `json:"is_default" bson:"-"` + any `collection:"databases"` + BaseModel `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + DataSource string `json:"data_source" bson:"data_source"` + Host string `json:"host" bson:"host"` + Port int `json:"port" bson:"port"` + URI string `json:"uri,omitempty" bson:"uri,omitempty"` + Database string `json:"database,omitempty" bson:"database,omitempty"` + Username string `json:"username,omitempty" bson:"username,omitempty"` + Password string `json:"password,omitempty" bson:"-"` + EncryptedPassword string `json:"-,omitempty" bson:"encrypted_password,omitempty"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Active bool `json:"active" bson:"active"` + ActiveAt time.Time `json:"active_ts" bson:"active_ts"` + IsDefault bool `json:"is_default" bson:"-"` MongoParams *struct { AuthSource string `json:"auth_source,omitempty" bson:"auth_source,omitempty"` diff --git a/core/models/models/database_metric.go b/core/models/models/database_metric.go index 195615a0..689d0365 100644 --- a/core/models/models/database_metric.go +++ b/core/models/models/database_metric.go @@ -3,22 +3,22 @@ package models import "go.mongodb.org/mongo-driver/bson/primitive" type DatabaseMetric struct { - any `collection:"database_metrics"` - BaseModel[DatabaseMetric] `bson:",inline"` - DatabaseId primitive.ObjectID `json:"database_id" bson:"database_id"` - CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` - TotalMemory uint64 `json:"total_memory" bson:"total_memory"` - AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` - UsedMemory uint64 `json:"used_memory" bson:"used_memory"` - UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` - TotalDisk uint64 `json:"total_disk" bson:"total_disk"` - AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` - UsedDisk uint64 `json:"used_disk" bson:"used_disk"` - UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` - Connections int `json:"connections" bson:"connections"` - QueryPerSecond float64 `json:"query_per_second" bson:"query_per_second"` - TotalQuery uint64 `json:"total_query,omitempty" bson:"total_query,omitempty"` - CacheHitRatio float64 `json:"cache_hit_ratio" bson:"cache_hit_ratio"` - ReplicationLag float64 `json:"replication_lag" bson:"replication_lag"` - LockWaitTime float64 `json:"lock_wait_time" bson:"lock_wait_time"` + any `collection:"database_metrics"` + BaseModel `bson:",inline"` + DatabaseId primitive.ObjectID `json:"database_id" bson:"database_id"` + CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` + TotalMemory uint64 `json:"total_memory" bson:"total_memory"` + AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` + UsedMemory uint64 `json:"used_memory" bson:"used_memory"` + UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` + TotalDisk uint64 `json:"total_disk" bson:"total_disk"` + AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` + UsedDisk uint64 `json:"used_disk" bson:"used_disk"` + UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` + Connections int `json:"connections" bson:"connections"` + QueryPerSecond float64 `json:"query_per_second" bson:"query_per_second"` + TotalQuery uint64 `json:"total_query,omitempty" bson:"total_query,omitempty"` + CacheHitRatio float64 `json:"cache_hit_ratio" bson:"cache_hit_ratio"` + ReplicationLag float64 `json:"replication_lag" bson:"replication_lag"` + LockWaitTime float64 `json:"lock_wait_time" bson:"lock_wait_time"` } diff --git a/core/models/models/dependency.go b/core/models/models/dependency.go index 4099cda6..6514bc20 100644 --- a/core/models/models/dependency.go +++ b/core/models/models/dependency.go @@ -5,15 +5,15 @@ import ( ) type Dependency struct { - any `collection:"dependencies"` - BaseModel[Dependency] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - Type string `json:"type" bson:"type"` - Version string `json:"version" bson:"version"` - Status string `json:"status" bson:"status"` - Error string `json:"error,omitempty" bson:"error,omitempty"` - NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-"` - Versions []string `json:"versions,omitempty" bson:"-"` + any `collection:"dependencies"` + BaseModel `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Type string `json:"type" bson:"type"` + Version string `json:"version" bson:"version"` + Status string `json:"status" bson:"status"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-"` + Versions []string `json:"versions,omitempty" bson:"-"` } diff --git a/core/models/models/dependency_config.go b/core/models/models/dependency_config.go index d05a5f34..29c4c8b6 100644 --- a/core/models/models/dependency_config.go +++ b/core/models/models/dependency_config.go @@ -1,14 +1,14 @@ package models type DependencyConfig struct { - any `collection:"dependency_configs"` - BaseModel[DependencyConfig] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - ExecCmd string `json:"exec_cmd" bson:"exec_cmd"` - PkgCmd string `json:"pkg_cmd" bson:"pkg_cmd"` - PkgSrcURL string `json:"pkg_src_url" bson:"pkg_src_url"` - Setup bool `json:"setup" bson:"-"` - TotalDependencies int `json:"total_dependencies" bson:"-"` - SearchReady bool `json:"search_ready" bson:"-"` + any `collection:"dependency_configs"` + BaseModel `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + ExecCmd string `json:"exec_cmd" bson:"exec_cmd"` + PkgCmd string `json:"pkg_cmd" bson:"pkg_cmd"` + PkgSrcURL string `json:"pkg_src_url" bson:"pkg_src_url"` + Setup bool `json:"setup" bson:"-"` + TotalDependencies int `json:"total_dependencies" bson:"-"` + SearchReady bool `json:"search_ready" bson:"-"` } diff --git a/core/models/models/dependency_config_setup.go b/core/models/models/dependency_config_setup.go index 20808a85..386141ae 100644 --- a/core/models/models/dependency_config_setup.go +++ b/core/models/models/dependency_config_setup.go @@ -3,15 +3,15 @@ package models import "go.mongodb.org/mongo-driver/bson/primitive" type DependencyConfigSetup struct { - any `collection:"dependency_config_setups"` - BaseModel[DependencyConfigSetup] `bson:",inline"` - DependencyConfigId primitive.ObjectID `json:"dependency_config_id" bson:"dependency_config_id"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - Version string `json:"version" bson:"version"` - Drivers []DependencyDriver `json:"versions,omitempty" bson:"versions,omitempty"` - Status string `json:"status" bson:"status"` - Error string `json:"error,omitempty" bson:"error,omitempty"` - Node *Node `json:"node,omitempty" bson:"-"` + any `collection:"dependency_config_setups"` + BaseModel `bson:",inline"` + DependencyConfigId primitive.ObjectID `json:"dependency_config_id" bson:"dependency_config_id"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Version string `json:"version" bson:"version"` + Drivers []DependencyDriver `json:"versions,omitempty" bson:"versions,omitempty"` + Status string `json:"status" bson:"status"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + Node *Node `json:"node,omitempty" bson:"-"` } type DependencyDriver struct { Name string `json:"name" bson:"name"` diff --git a/core/models/models/dependency_log.go b/core/models/models/dependency_log.go index 415245ce..e4e13976 100644 --- a/core/models/models/dependency_log.go +++ b/core/models/models/dependency_log.go @@ -3,8 +3,8 @@ package models import "go.mongodb.org/mongo-driver/bson/primitive" type DependencyLog struct { - any `collection:"dependency_logs"` - BaseModel[DependencyLog] `bson:",inline"` - TargetId primitive.ObjectID `json:"target_id" bson:"target_id"` - Content string `json:"content" bson:"content"` + any `collection:"dependency_logs"` + BaseModel `bson:",inline"` + TargetId primitive.ObjectID `json:"target_id" bson:"target_id"` + Content string `json:"content" bson:"content"` } diff --git a/core/models/models/dependency_pypi_project.go b/core/models/models/dependency_pypi_project.go index f0d20a92..ec3947a3 100644 --- a/core/models/models/dependency_pypi_project.go +++ b/core/models/models/dependency_pypi_project.go @@ -1,9 +1,9 @@ package models type DependencyPypiProject struct { - any `collection:"dependency_pypi_projects"` - BaseModel[DependencyPypiProject] `bson:",inline"` - Name string `json:"name" bson:"name"` - Version string `json:"version" bson:"version"` - LastSerial int `json:"_last-serial" bson:"last_serial"` + any `collection:"dependency_pypi_projects"` + BaseModel `bson:",inline"` + Name string `json:"name" bson:"name"` + Version string `json:"version" bson:"version"` + LastSerial int `json:"_last-serial" bson:"last_serial"` } diff --git a/core/models/models/dependency_repo.go b/core/models/models/dependency_repo.go index d205dce0..d3ca1263 100644 --- a/core/models/models/dependency_repo.go +++ b/core/models/models/dependency_repo.go @@ -1,10 +1,10 @@ package models type DependencyRepo struct { - any `collection:"dependency_repos"` - BaseModel[DependencyRepo] `bson:",inline"` - Name string `json:"name" bson:"name"` - Type string `json:"type" bson:"type"` - LatestVersion string `json:"latest_version" bson:"latest_version"` - AllVersions []string `json:"all_versions" bson:"all_versions"` + any `collection:"dependency_repos"` + BaseModel `bson:",inline"` + Name string `json:"name" bson:"name"` + Type string `json:"type" bson:"type"` + LatestVersion string `json:"latest_version" bson:"latest_version"` + AllVersions []string `json:"all_versions" bson:"all_versions"` } diff --git a/core/models/models/environment.go b/core/models/models/environment.go index 889d412c..3faf8547 100644 --- a/core/models/models/environment.go +++ b/core/models/models/environment.go @@ -1,8 +1,8 @@ package models type Environment struct { - any `collection:"environments"` - BaseModel[Environment] `bson:",inline"` - Key string `json:"key" bson:"key"` - Value string `json:"value" bson:"value"` + any `collection:"environments"` + BaseModel `bson:",inline"` + Key string `json:"key" bson:"key"` + Value string `json:"value" bson:"value"` } diff --git a/core/models/models/git.go b/core/models/models/git.go index 45a3365f..c5babffd 100644 --- a/core/models/models/git.go +++ b/core/models/models/git.go @@ -6,20 +6,20 @@ import ( ) type Git struct { - any `collection:"gits"` - BaseModel[Git] `bson:",inline"` - Url string `json:"url" bson:"url"` - Name string `json:"name" bson:"name"` - AuthType string `json:"auth_type" bson:"auth_type"` - Username string `json:"username" bson:"username"` - Password string `json:"password" bson:"password"` - CurrentBranch string `json:"current_branch" bson:"current_branch"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Spiders []Spider `json:"spiders,omitempty" bson:"-"` - Refs []vcs.GitRef `json:"refs" bson:"refs"` - RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"` - CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"` + any `collection:"gits"` + BaseModel `bson:",inline"` + Url string `json:"url" bson:"url"` + Name string `json:"name" bson:"name"` + AuthType string `json:"auth_type" bson:"auth_type"` + Username string `json:"username" bson:"username"` + Password string `json:"password" bson:"password"` + CurrentBranch string `json:"current_branch" bson:"current_branch"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Spiders []Spider `json:"spiders,omitempty" bson:"-"` + Refs []vcs.GitRef `json:"refs" bson:"refs"` + RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"` + CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"` // settings AutoPull bool `json:"auto_pull" bson:"auto_pull"` diff --git a/core/models/models/llm_provider.go b/core/models/models/llm_provider.go index 1364c54c..d723eb91 100644 --- a/core/models/models/llm_provider.go +++ b/core/models/models/llm_provider.go @@ -2,17 +2,17 @@ package models // LLMProvider represents a language model provider such as OpenAI, Anthropic, etc. type LLMProvider struct { - any `collection:"llm_providers"` - BaseModel[LLMProvider] `bson:",inline"` - Key string `json:"key" bson:"key"` // Provider key (e.g., "openai", "anthropic", "gemini") - Name string `json:"name" bson:"name"` // Display name for UI - Enabled bool `json:"enabled" bson:"enabled"` // Whether this provider is enabled - ApiKey string `json:"api_key" bson:"api_key"` // API key for the provider - ApiBaseUrl string `json:"api_base_url" bson:"api_base_url"` // API base URL for the provider - DeploymentName string `json:"deployment_name" bson:"deployment_name"` // Deployment name for the provider - ApiVersion string `json:"api_version" bson:"api_version"` // API version for the provider - Models []string `json:"models" bson:"models"` // Models supported by this provider - Unset bool `json:"unset" bson:"-"` // Whether the provider is unset + any `collection:"llm_providers"` + BaseModel `bson:",inline"` + Key string `json:"key" bson:"key"` // Provider key (e.g., "openai", "anthropic", "gemini") + Name string `json:"name" bson:"name"` // Display name for UI + Enabled bool `json:"enabled" bson:"enabled"` // Whether this provider is enabled + ApiKey string `json:"api_key" bson:"api_key"` // API key for the provider + ApiBaseUrl string `json:"api_base_url" bson:"api_base_url"` // API base URL for the provider + DeploymentName string `json:"deployment_name" bson:"deployment_name"` // Deployment name for the provider + ApiVersion string `json:"api_version" bson:"api_version"` // API version for the provider + Models []string `json:"models" bson:"models"` // Models supported by this provider + Unset bool `json:"unset" bson:"-"` // Whether the provider is unset } func (p *LLMProvider) IsUnset() bool { diff --git a/core/models/models/metric.go b/core/models/models/metric.go index 8f72eb45..a9494754 100644 --- a/core/models/models/metric.go +++ b/core/models/models/metric.go @@ -6,7 +6,7 @@ import ( type Metric struct { any `collection:"metrics"` - BaseModel[Metric] `bson:",inline"` + BaseModel `bson:",inline"` Type string `json:"type" bson:"type"` NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` diff --git a/core/models/models/node.go b/core/models/models/node.go index b132fa06..2adcdb23 100644 --- a/core/models/models/node.go +++ b/core/models/models/node.go @@ -5,19 +5,19 @@ import ( ) type Node struct { - any `collection:"nodes"` - BaseModel[Node] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Ip string `json:"ip" bson:"ip"` - Mac string `json:"mac" bson:"mac"` - Hostname string `json:"hostname" bson:"hostname"` - Description string `json:"description" bson:"description"` - IsMaster bool `json:"is_master" bson:"is_master"` - Status string `json:"status" bson:"status"` - Enabled bool `json:"enabled" bson:"enabled"` - Active bool `json:"active" bson:"active"` - ActiveAt time.Time `json:"active_at" bson:"active_ts"` - CurrentRunners int `json:"current_runners" bson:"current_runners"` - MaxRunners int `json:"max_runners" bson:"max_runners"` + any `collection:"nodes"` + BaseModel `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Ip string `json:"ip" bson:"ip"` + Mac string `json:"mac" bson:"mac"` + Hostname string `json:"hostname" bson:"hostname"` + Description string `json:"description" bson:"description"` + IsMaster bool `json:"is_master" bson:"is_master"` + Status string `json:"status" bson:"status"` + Enabled bool `json:"enabled" bson:"enabled"` + Active bool `json:"active" bson:"active"` + ActiveAt time.Time `json:"active_at" bson:"active_ts"` + CurrentRunners int `json:"current_runners" bson:"current_runners"` + MaxRunners int `json:"max_runners" bson:"max_runners"` } diff --git a/core/models/models/notification_alert.go b/core/models/models/notification_alert.go index 65004964..0fb4c145 100644 --- a/core/models/models/notification_alert.go +++ b/core/models/models/notification_alert.go @@ -3,17 +3,17 @@ package models import "go.mongodb.org/mongo-driver/bson/primitive" type NotificationAlert struct { - any `collection:"notification_alerts"` - BaseModel[NotificationAlert] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - HasMetricTarget bool `json:"has_metric_target" bson:"has_metric_target"` - MetricTargetId primitive.ObjectID `json:"metric_target_id,omitempty" bson:"metric_target_id,omitempty"` - MetricName string `json:"metric_name" bson:"metric_name"` - Operator string `json:"operator" bson:"operator"` - LastingSeconds int `json:"lasting_seconds" bson:"lasting_seconds"` - TargetValue float32 `json:"target_value" bson:"target_value"` - Level string `json:"level" bson:"level"` - TemplateKey string `json:"template_key,omitempty" bson:"template_key,omitempty"` + any `collection:"notification_alerts"` + BaseModel `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + HasMetricTarget bool `json:"has_metric_target" bson:"has_metric_target"` + MetricTargetId primitive.ObjectID `json:"metric_target_id,omitempty" bson:"metric_target_id,omitempty"` + MetricName string `json:"metric_name" bson:"metric_name"` + Operator string `json:"operator" bson:"operator"` + LastingSeconds int `json:"lasting_seconds" bson:"lasting_seconds"` + TargetValue float32 `json:"target_value" bson:"target_value"` + Level string `json:"level" bson:"level"` + TemplateKey string `json:"template_key,omitempty" bson:"template_key,omitempty"` } diff --git a/core/models/models/notification_channel.go b/core/models/models/notification_channel.go index d8ad1cbc..972aca09 100644 --- a/core/models/models/notification_channel.go +++ b/core/models/models/notification_channel.go @@ -1,18 +1,18 @@ package models type NotificationChannel struct { - any `collection:"notification_channels"` - BaseModel[NotificationChannel] `bson:",inline"` - Type string `json:"type" bson:"type"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Provider string `json:"provider" bson:"provider"` - SMTPServer string `json:"smtp_server,omitempty" bson:"smtp_server,omitempty"` - SMTPPort int `json:"smtp_port,omitempty" bson:"smtp_port,omitempty"` - SMTPUsername string `json:"smtp_username,omitempty" bson:"smtp_username,omitempty"` - SMTPPassword string `json:"smtp_password,omitempty" bson:"smtp_password,omitempty"` - WebhookUrl string `json:"webhook_url,omitempty" bson:"webhook_url,omitempty"` - TelegramBotToken string `json:"telegram_bot_token,omitempty" bson:"telegram_bot_token,omitempty"` - TelegramChatId string `json:"telegram_chat_id,omitempty" bson:"telegram_chat_id,omitempty"` - GoogleOAuth2Json string `json:"google_oauth2_json,omitempty" bson:"google_oauth2_json,omitempty"` + any `collection:"notification_channels"` + BaseModel `bson:",inline"` + Type string `json:"type" bson:"type"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Provider string `json:"provider" bson:"provider"` + SMTPServer string `json:"smtp_server,omitempty" bson:"smtp_server,omitempty"` + SMTPPort int `json:"smtp_port,omitempty" bson:"smtp_port,omitempty"` + SMTPUsername string `json:"smtp_username,omitempty" bson:"smtp_username,omitempty"` + SMTPPassword string `json:"smtp_password,omitempty" bson:"smtp_password,omitempty"` + WebhookUrl string `json:"webhook_url,omitempty" bson:"webhook_url,omitempty"` + TelegramBotToken string `json:"telegram_bot_token,omitempty" bson:"telegram_bot_token,omitempty"` + TelegramChatId string `json:"telegram_chat_id,omitempty" bson:"telegram_chat_id,omitempty"` + GoogleOAuth2Json string `json:"google_oauth2_json,omitempty" bson:"google_oauth2_json,omitempty"` } diff --git a/core/models/models/notification_request.go b/core/models/models/notification_request.go index 480cef85..355f05aa 100644 --- a/core/models/models/notification_request.go +++ b/core/models/models/notification_request.go @@ -3,20 +3,20 @@ package models import "go.mongodb.org/mongo-driver/bson/primitive" type NotificationRequest struct { - any `collection:"notification_requests"` - BaseModel[NotificationRequest] `bson:",inline"` - Status string `json:"status" bson:"status"` - Error string `json:"error,omitempty" bson:"error,omitempty"` - Title string `json:"title" bson:"title"` - Content string `json:"content" bson:"content"` - SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` - SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"` - MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty"` - MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` - MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` - SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` - ChannelId primitive.ObjectID `json:"channel_id" bson:"channel_id"` - Setting *NotificationSetting `json:"setting,omitempty" bson:"-"` - Channel *NotificationChannel `json:"channel,omitempty" bson:"-"` - Test bool `json:"test,omitempty" bson:"test,omitempty"` + any `collection:"notification_requests"` + BaseModel `bson:",inline"` + Status string `json:"status" bson:"status"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + Title string `json:"title" bson:"title"` + Content string `json:"content" bson:"content"` + SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` + SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"` + MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty"` + MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` + MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` + SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` + ChannelId primitive.ObjectID `json:"channel_id" bson:"channel_id"` + Setting *NotificationSetting `json:"setting,omitempty" bson:"-"` + Channel *NotificationChannel `json:"channel,omitempty" bson:"-"` + Test bool `json:"test,omitempty" bson:"test,omitempty"` } diff --git a/core/models/models/notification_setting.go b/core/models/models/notification_setting.go index 80826bec..b937c446 100644 --- a/core/models/models/notification_setting.go +++ b/core/models/models/notification_setting.go @@ -3,11 +3,11 @@ package models import "go.mongodb.org/mongo-driver/bson/primitive" type NotificationSetting struct { - any `collection:"notification_settings"` - BaseModel[NotificationSetting] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` + any `collection:"notification_settings"` + BaseModel `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` Title string `json:"title,omitempty" bson:"title,omitempty"` Template string `json:"template" bson:"template"` diff --git a/core/models/models/permission.go b/core/models/models/permission.go index 90882835..e59007d2 100644 --- a/core/models/models/permission.go +++ b/core/models/models/permission.go @@ -3,12 +3,12 @@ package models import "go.mongodb.org/mongo-driver/bson/primitive" type Permission struct { - any `collection:"permissions"` - BaseModel[Permission] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - Type string `json:"type" bson:"type"` - Routes []string `json:"routes" bson:"routes"` + any `collection:"permissions"` + BaseModel `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + Type string `json:"type" bson:"type"` + Routes []string `json:"routes" bson:"routes"` } diff --git a/core/models/models/project.go b/core/models/models/project.go index e487ef7b..e3f6a1ca 100644 --- a/core/models/models/project.go +++ b/core/models/models/project.go @@ -1,9 +1,9 @@ package models type Project struct { - any `collection:"projects"` - BaseModel[Project] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Spiders int `json:"spiders" bson:"-"` + any `collection:"projects"` + BaseModel `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Spiders int `json:"spiders" bson:"-"` } diff --git a/core/models/models/role.go b/core/models/models/role.go index 2a132f69..ea6b39e9 100644 --- a/core/models/models/role.go +++ b/core/models/models/role.go @@ -1,12 +1,12 @@ package models type Role struct { - any `collection:"roles"` - BaseModel[Role] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Routes []string `json:"routes" bson:"routes"` - RootAdmin bool `json:"-" bson:"root_admin,omitempty"` - IsRootAdmin bool `json:"root_admin" bson:"-"` - Users int `json:"users" bson:"-"` + any `collection:"roles"` + BaseModel `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Routes []string `json:"routes" bson:"routes"` + RootAdmin bool `json:"-" bson:"root_admin,omitempty"` + IsRootAdmin bool `json:"root_admin" bson:"-"` + Users int `json:"users" bson:"-"` } diff --git a/core/models/models/schedule.go b/core/models/models/schedule.go index 22df35da..b4bc232b 100644 --- a/core/models/models/schedule.go +++ b/core/models/models/schedule.go @@ -6,17 +6,17 @@ import ( ) type Schedule struct { - any `collection:"schedules"` - BaseModel[Schedule] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` - Cron string `json:"cron" bson:"cron"` - EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` - Cmd string `json:"cmd" bson:"cmd"` - Param string `json:"param" bson:"param"` - Mode string `json:"mode" bson:"mode"` - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` - Priority int `json:"priority" bson:"priority"` - Enabled bool `json:"enabled" bson:"enabled"` + any `collection:"schedules"` + BaseModel `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` + Cron string `json:"cron" bson:"cron"` + EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` + Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` + Mode string `json:"mode" bson:"mode"` + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` + Priority int `json:"priority" bson:"priority"` + Enabled bool `json:"enabled" bson:"enabled"` } diff --git a/core/models/models/setting.go b/core/models/models/setting.go index 6f61f1c3..c66bdbcd 100644 --- a/core/models/models/setting.go +++ b/core/models/models/setting.go @@ -5,8 +5,8 @@ import ( ) type Setting struct { - any `collection:"settings"` - BaseModel[Setting] `bson:",inline"` - Key string `json:"key" bson:"key"` - Value bson.M `json:"value" bson:"value"` + any `collection:"settings"` + BaseModel `bson:",inline"` + Key string `json:"key" bson:"key"` + Value bson.M `json:"value" bson:"value"` } diff --git a/core/models/models/spider.go b/core/models/models/spider.go index 916a6f53..14b008cd 100644 --- a/core/models/models/spider.go +++ b/core/models/models/spider.go @@ -5,23 +5,23 @@ import ( ) type Spider struct { - any `collection:"spiders"` - BaseModel[Spider] `bson:",inline"` - Name string `json:"name" bson:"name"` // spider name - ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id (deprecated) # TODO: remove this field in the future - ColName string `json:"col_name,omitempty" bson:"col_name"` // data collection name - DbName string `json:"db_name,omitempty" bson:"db_name"` // database name - DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id - DataSource *Database `json:"data_source,omitempty" bson:"-"` // data source - Description string `json:"description" bson:"description"` // description - ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id - Mode string `json:"mode" bson:"mode"` // default Task.Mode - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds - GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id - GitRootPath string `json:"git_root_path" bson:"git_root_path"` - Git *Git `json:"git,omitempty" bson:"-"` - Template string `json:"template,omitempty" bson:"template,omitempty"` // spider template - TemplateParams *SpiderTemplateParams `json:"template_params,omitempty" bson:"template_params,omitempty"` + any `collection:"spiders"` + BaseModel `bson:",inline"` + Name string `json:"name" bson:"name"` // spider name + ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id (deprecated) # TODO: remove this field in the future + ColName string `json:"col_name,omitempty" bson:"col_name"` // data collection name + DbName string `json:"db_name,omitempty" bson:"db_name"` // database name + DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id + DataSource *Database `json:"data_source,omitempty" bson:"-"` // data source + Description string `json:"description" bson:"description"` // description + ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id + Mode string `json:"mode" bson:"mode"` // default Task.Mode + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds + GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id + GitRootPath string `json:"git_root_path" bson:"git_root_path"` + Git *Git `json:"git,omitempty" bson:"-"` + Template string `json:"template,omitempty" bson:"template,omitempty"` // spider template + TemplateParams *SpiderTemplateParams `json:"template_params,omitempty" bson:"template_params,omitempty"` // stats Stat *SpiderStat `json:"stat,omitempty" bson:"-"` diff --git a/core/models/models/spider_stat.go b/core/models/models/spider_stat.go index a3c9f631..c122a540 100644 --- a/core/models/models/spider_stat.go +++ b/core/models/models/spider_stat.go @@ -6,7 +6,7 @@ import ( type SpiderStat struct { any `collection:"spider_stats"` - BaseModel[SpiderStat] `bson:",inline"` + BaseModel `bson:",inline"` LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"` LastTask *Task `json:"last_task,omitempty" bson:"-"` Tasks int `json:"tasks" bson:"tasks"` diff --git a/core/models/models/task.go b/core/models/models/task.go index 69b9c78b..00e723c2 100644 --- a/core/models/models/task.go +++ b/core/models/models/task.go @@ -5,23 +5,23 @@ import ( ) type Task struct { - any `collection:"tasks"` - BaseModel[Task] `bson:",inline"` - SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` - Status string `json:"status" bson:"status"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - Cmd string `json:"cmd" bson:"cmd"` - Param string `json:"param" bson:"param"` - Error string `json:"error" bson:"error"` - Pid int `json:"pid" bson:"pid"` - ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` - Type string `json:"type" bson:"type"` - Mode string `json:"mode" bson:"mode"` - Priority int `json:"priority" bson:"priority"` - NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-"` - Stat *TaskStat `json:"stat,omitempty" bson:"-"` - Spider *Spider `json:"spider,omitempty" bson:"-"` - Schedule *Schedule `json:"schedule,omitempty" bson:"-"` - Node *Node `json:"node,omitempty" bson:"-"` - UserId primitive.ObjectID `json:"-" bson:"-"` + any `collection:"tasks"` + BaseModel `bson:",inline"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` + Status string `json:"status" bson:"status"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` + Error string `json:"error" bson:"error"` + Pid int `json:"pid" bson:"pid"` + ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` + Type string `json:"type" bson:"type"` + Mode string `json:"mode" bson:"mode"` + Priority int `json:"priority" bson:"priority"` + NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-"` + Stat *TaskStat `json:"stat,omitempty" bson:"-"` + Spider *Spider `json:"spider,omitempty" bson:"-"` + Schedule *Schedule `json:"schedule,omitempty" bson:"-"` + Node *Node `json:"node,omitempty" bson:"-"` + UserId primitive.ObjectID `json:"-" bson:"-"` } diff --git a/core/models/models/task_stat.go b/core/models/models/task_stat.go index 900e8c24..4be7b233 100644 --- a/core/models/models/task_stat.go +++ b/core/models/models/task_stat.go @@ -5,12 +5,12 @@ import ( ) type TaskStat struct { - any `collection:"task_stats"` - BaseModel[TaskStat] `bson:",inline"` - StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` - EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` - WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond - RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond - TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond - ResultCount int64 `json:"result_count" bson:"result_count"` + any `collection:"task_stats"` + BaseModel `bson:",inline"` + StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` + EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond + ResultCount int64 `json:"result_count" bson:"result_count"` } diff --git a/core/models/models/test.go b/core/models/models/test.go index bd31b833..c5f42e8f 100644 --- a/core/models/models/test.go +++ b/core/models/models/test.go @@ -1,7 +1,7 @@ package models type TestModel struct { - any `collection:"testmodels"` - BaseModel[TestModel] `bson:",inline"` - Name string `json:"name" bson:"name"` + any `collection:"testmodels"` + BaseModel `bson:",inline"` + Name string `json:"name" bson:"name"` } diff --git a/core/models/models/token.go b/core/models/models/token.go index 5657777c..f421aa2d 100644 --- a/core/models/models/token.go +++ b/core/models/models/token.go @@ -1,8 +1,8 @@ package models type Token struct { - any `collection:"tokens"` - BaseModel[Token] `bson:",inline"` - Name string `json:"name" bson:"name"` - Token string `json:"token" bson:"token"` + any `collection:"tokens"` + BaseModel `bson:",inline"` + Name string `json:"name" bson:"name"` + Token string `json:"token" bson:"token"` } diff --git a/core/models/models/user.go b/core/models/models/user.go index c96bcf6e..f9681cfc 100644 --- a/core/models/models/user.go +++ b/core/models/models/user.go @@ -3,16 +3,16 @@ package models import "go.mongodb.org/mongo-driver/bson/primitive" type User struct { - any `collection:"users"` - BaseModel[User] `bson:",inline"` - Username string `json:"username" bson:"username"` - Password string `json:"-" bson:"password"` - Role string `json:"role" bson:"role"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - FirstName string `json:"first_name" bson:"first_name"` - LastName string `json:"last_name" bson:"last_name"` - Email string `json:"email" bson:"email"` - RootAdmin bool `json:"root_admin,omitempty" bson:"root_admin"` - RootAdminRole bool `json:"root_admin_role,omitempty" bson:"-"` - Routes []string `json:"routes,omitempty" bson:"-"` + any `collection:"users"` + BaseModel `bson:",inline"` + Username string `json:"username" bson:"username"` + Password string `json:"-" bson:"password"` + Role string `json:"role" bson:"role"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + FirstName string `json:"first_name" bson:"first_name"` + LastName string `json:"last_name" bson:"last_name"` + Email string `json:"email" bson:"email"` + RootAdmin bool `json:"root_admin,omitempty" bson:"root_admin"` + RootAdminRole bool `json:"root_admin_role,omitempty" bson:"-"` + Routes []string `json:"routes,omitempty" bson:"-"` } diff --git a/core/models/models/user_role.go b/core/models/models/user_role.go index ca54f520..976d5a57 100644 --- a/core/models/models/user_role.go +++ b/core/models/models/user_role.go @@ -5,8 +5,8 @@ import ( ) type UserRole struct { - any `collection:"user_roles"` - BaseModel[UserRole] `bson:",inline"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - UserId primitive.ObjectID `json:"user_id" bson:"user_id"` + any `collection:"user_roles"` + BaseModel `bson:",inline"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + UserId primitive.ObjectID `json:"user_id" bson:"user_id"` } diff --git a/core/models/service/base_service_test.go b/core/models/service/base_service_test.go index b9743370..e876587d 100644 --- a/core/models/service/base_service_test.go +++ b/core/models/service/base_service_test.go @@ -17,9 +17,9 @@ import ( ) type TestModel struct { - Id primitive.ObjectID `bson:"_id,omitempty" collection:"testmodels"` - models.BaseModel[TestModel] `bson:",inline"` - Name string `bson:"name"` + Id primitive.ObjectID `bson:"_id,omitempty" collection:"testmodels"` + models.BaseModel `bson:",inline"` + Name string `bson:"name"` } func setupTestDB() { diff --git a/core/openapi/wrapper.go b/core/openapi/wrapper.go index e8259ba9..8f8214a1 100644 --- a/core/openapi/wrapper.go +++ b/core/openapi/wrapper.go @@ -2,7 +2,8 @@ package openapi import ( "fmt" - + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/utils" "github.com/loopfz/gadgeto/tonic" "github.com/gin-gonic/gin" @@ -11,8 +12,9 @@ import ( // FizzWrapper wraps an existing Gin Engine to add OpenAPI functionality type FizzWrapper struct { - fizz *fizz.Fizz - gin *gin.Engine + fizz *fizz.Fizz + gin *gin.Engine + logger interfaces.Logger } // NewFizzWrapper creates a new wrapper around an existing Gin Engine @@ -20,10 +22,10 @@ type FizzWrapper struct { func NewFizzWrapper(engine *gin.Engine) *FizzWrapper { // Create a new Fizz instance using the existing Gin engine f := fizz.NewFromEngine(engine) - return &FizzWrapper{ - fizz: f, - gin: engine, + fizz: f, + gin: engine, + logger: utils.NewLogger("FizzWrapper"), } } diff --git a/go.work b/go.work index ea0e5963..900ac09c 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.23 +go 1.23.7 use ( backend diff --git a/go.work.sum b/go.work.sum index 001314c3..9c0840c0 100644 --- a/go.work.sum +++ b/go.work.sum @@ -973,8 +973,6 @@ github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9 h1:hJix6idebFclqlfZCHE7EUX7uqLCyb70nHNHH1XKGBg= -github.com/juju/errors v1.0.0 h1:yiq7kjCLll1BiaRuNY53MGI0+EQ3rF6GB+wvboZDefM= -github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5Qe8= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 h1:UUHMLvzt/31azWTN/ifGWef4WUqvXk0iRqdhdy/2uzI= github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2 h1:Pp8RxiF4rSoXP9SED26WCfNB28/dwTDpPXS8XMJR8rc= github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= @@ -1086,6 +1084,7 @@ github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmq github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE= github.com/tj/go-buffer v1.1.0 h1:Lo2OsPHlIxXF24zApe15AbK3bJLAOvkkxEA6Ux4c47M= github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2 h1:eGaGNxrtoZf/mBURsnNQKDR7u50Klgcf2eFDQEnc8Bc= @@ -1172,6 +1171,7 @@ golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZP golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= @@ -1202,6 +1202,7 @@ golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= @@ -1236,12 +1237,15 @@ golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2 h1:IRJeR9r1pYWsHKTRe/IInb7lYvbBVIqOgsX/u0mbOWY= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457 h1:zf5N6UOrA487eEFacMePxjXAJctxKmyjKUsjA11Uzuk= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= From 23cad00d92594693e922b886b8da87675be26497 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 13 Mar 2025 17:37:30 +0800 Subject: [PATCH 03/13] refactor: streamline controller methods and enhance parameter handling - Refactored multiple controller methods to accept structured parameters for improved clarity and maintainability. - Consolidated error handling and response generation across various endpoints. - Updated function signatures to eliminate unnecessary context parameters and enhance type safety. - Improved consistency in response formatting and error handling across controllers. - Enhanced file handling methods to support multipart file uploads and directory operations more effectively. --- core/controllers/base_file.go | 265 ++++++++++++------------------- core/controllers/login.go | 44 +++-- core/controllers/router.go | 4 +- core/controllers/schedule.go | 175 ++++++++++---------- core/controllers/setting.go | 85 ++++------ core/controllers/spider.go | 202 ++++++++++++++++------- core/controllers/stats.go | 67 +++++--- core/controllers/sync.go | 36 +++-- core/controllers/system_info.go | 6 +- core/controllers/task.go | 273 ++++++++++++++++---------------- core/controllers/token.go | 66 ++++---- core/controllers/user.go | 21 ++- 12 files changed, 636 insertions(+), 608 deletions(-) diff --git a/core/controllers/base_file.go b/core/controllers/base_file.go index fc8657ed..1384d1f0 100644 --- a/core/controllers/base_file.go +++ b/core/controllers/base_file.go @@ -3,21 +3,17 @@ package controllers import ( "errors" "fmt" + "io" + "mime/multipart" + "os" + "sync" + "github.com/crawlab-team/crawlab/core/fs" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/gin-gonic/gin" - "io" - "os" - "sync" ) -type GetBaseFileListDirParams struct { - Path string `path:"path"` -} - -func GetBaseFileListDir(rootPath string, params *GetBaseFileListDirParams) (response *Response[[]interfaces.FsFileInfo], err error) { - path := params.Path - +func GetBaseFileListDir(rootPath, path string) (response *Response[[]interfaces.FsFileInfo], err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { return GetErrorResponse[[]interfaces.FsFileInfo](err) @@ -30,17 +26,10 @@ func GetBaseFileListDir(rootPath string, params *GetBaseFileListDirParams) (resp } } - //HandleSuccessWithData(c, files) - return GetDataResponse[[]interfaces.FsFileInfo](files) + return GetDataResponse(files) } -type GetBaseFileFileParams struct { - Path string `path:"path"` -} - -func GetBaseFileFile(rootPath string, params *GetBaseFileFileParams) (response *Response[string], err error) { - path := params.Path - +func GetBaseFileContent(rootPath, path string) (response *Response[string], err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { return GetErrorResponse[string](err) @@ -51,94 +40,75 @@ func GetBaseFileFile(rootPath string, params *GetBaseFileFileParams) (response * return GetErrorResponse[string](err) } - return GetDataResponse[string](string(data)) + return GetDataResponse(string(data)) } -func GetBaseFileFileInfo(rootPath string, c *gin.Context) { - path := c.Query("path") - +func GetBaseFileInfo(rootPath, path string) (response *Response[interfaces.FsFileInfo], err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[interfaces.FsFileInfo](err) } info, err := fsSvc.GetFileInfo(path) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[interfaces.FsFileInfo](err) } - HandleSuccessWithData(c, info) + return GetDataResponse(info) } -func PostBaseFileSaveFile(rootPath string, c *gin.Context) { - fsSvc, err := fs.GetBaseFileFsSvc(rootPath) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - if c.GetHeader("Content-Type") == "application/json" { - var payload struct { - Path string `json:"path"` - Data string `json:"data"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := fsSvc.Save(payload.Path, []byte(payload.Data)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } else { - path, ok := c.GetPostForm("path") - if !ok { - HandleErrorBadRequest(c, errors.New("missing required field 'path'")) - return - } - file, err := c.FormFile("file") - if err != nil { - HandleErrorBadRequest(c, err) - return - } - f, err := file.Open() - if err != nil { - HandleErrorBadRequest(c, err) - return - } - fileData, err := io.ReadAll(f) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := fsSvc.Save(path, fileData); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - HandleSuccess(c) +type PostBaseFileSaveOneParams struct { + Path string `json:"path" form:"path"` + Data string `json:"data"` } -func PostBaseFileSaveFiles(rootPath string, c *gin.Context) { +func PostBaseFileSaveOne(rootPath, path, data string) (response *Response[any], err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[any](err) } - form, err := c.MultipartForm() - if err != nil { - HandleErrorBadRequest(c, err) - return + if err := fsSvc.Save(path, []byte(data)); err != nil { + return GetErrorResponse[any](err) } + + return GetDataResponse(any(data)) +} + +func PostBaseFileSaveOneForm(rootPath, path string, file *multipart.FileHeader) (response *Response[any], err error) { + fsSvc, err := fs.GetBaseFileFsSvc(rootPath) + if err != nil { + return GetErrorResponse[any](err) + } + + f, err := file.Open() + if err != nil { + return GetErrorResponse[any](err) + } + + fileData, err := io.ReadAll(f) + if err != nil { + return GetErrorResponse[any](err) + } + + if err := fsSvc.Save(path, fileData); err != nil { + return GetErrorResponse[any](err) + } + + return GetDataResponse[any](nil) +} + +func PostBaseFileSaveMany(rootPath string, form *multipart.Form) (response *Response[any], err error) { + fsSvc, err := fs.GetBaseFileFsSvc(rootPath) + if err != nil { + return GetErrorResponse[any](err) + } + wg := sync.WaitGroup{} wg.Add(len(form.File)) for path := range form.File { go func(path string) { - file, err := c.FormFile(path) + file := form.File[path][0] if err != nil { logger.Warnf("invalid file header: %s", path) logger.Error(err.Error()) @@ -170,125 +140,84 @@ func PostBaseFileSaveFiles(rootPath string, c *gin.Context) { } wg.Wait() - HandleSuccess(c) + return GetDataResponse[any](nil) } -func PostBaseFileSaveDir(rootPath string, c *gin.Context) { - var payload struct { - Path string `json:"path"` - NewPath string `json:"new_path"` - Data string `json:"data"` +func PostBaseFileSaveDir(rootPath, path string) (response *Response[any], err error) { + fsSvc, err := fs.GetBaseFileFsSvc(rootPath) + if err != nil { + return GetErrorResponse[any](err) } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return + + if err := fsSvc.CreateDir(path); err != nil { + return GetErrorResponse[any](err) + } + + return GetDataResponse[any](nil) +} + +func PostBaseFileRename(rootPath, path, newPath string) (response *Response[any], err error) { + fsSvc, err := fs.GetBaseFileFsSvc(rootPath) + if err != nil { + return GetErrorResponse[any](err) + } + + if err := fsSvc.Rename(path, newPath); err != nil { + return GetErrorResponse[any](err) + } + + return GetDataResponse[any](nil) +} + +func DeleteBaseFile(rootPath, path string) (response *Response[any], err error) { + if path == "~" { + path = "." } fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[any](err) } - if err := fsSvc.CreateDir(payload.Path); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func PostBaseFileRenameFile(rootPath string, c *gin.Context) { - var payload struct { - Path string `json:"path"` - NewPath string `json:"new_path"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - fsSvc, err := fs.GetBaseFileFsSvc(rootPath) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := fsSvc.Rename(payload.Path, payload.NewPath); err != nil { - HandleErrorInternalServerError(c, err) - return - } -} - -func DeleteBaseFileFile(rootPath string, c *gin.Context) { - var payload struct { - Path string `json:"path"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - if payload.Path == "~" { - payload.Path = "." - } - - fsSvc, err := fs.GetBaseFileFsSvc(rootPath) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := fsSvc.Delete(payload.Path); err != nil { - HandleErrorInternalServerError(c, err) - return + if err := fsSvc.Delete(path); err != nil { + return GetErrorResponse[any](err) } _, err = fsSvc.GetFileInfo(".") if err != nil { _ = fsSvc.CreateDir("/") } - HandleSuccess(c) + return GetDataResponse[any](nil) } -func PostBaseFileCopyFile(rootPath string, c *gin.Context) { - var payload struct { - Path string `json:"path"` - NewPath string `json:"new_path"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - +func PostBaseFileCopy(rootPath, path, newPath string) (response *Response[any], err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[any](err) } - if err := fsSvc.Copy(payload.Path, payload.NewPath); err != nil { - HandleErrorInternalServerError(c, err) - return + if err := fsSvc.Copy(path, newPath); err != nil { + return GetErrorResponse[any](err) } - HandleSuccess(c) + return GetDataResponse[any](nil) } -func PostBaseFileExport(rootPath string, c *gin.Context) { +func PostBaseFileExport(rootPath string, c *gin.Context) (err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - HandleErrorBadRequest(c, err) - return + return err } // zip file path zipFilePath, err := fsSvc.Export() if err != nil { - HandleErrorInternalServerError(c, err) - return + return err } // download c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilePath)) c.File(zipFilePath) + + return nil } diff --git a/core/controllers/login.go b/core/controllers/login.go index 082ff200..1a174b12 100644 --- a/core/controllers/login.go +++ b/core/controllers/login.go @@ -7,30 +7,26 @@ import ( "github.com/gin-gonic/gin" ) -func PostLogin(c *gin.Context) { - var payload struct { - Username string `json:"username"` - Password string `json:"password"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - userSvc, err := user.GetUserService() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - token, loggedInUser, err := userSvc.Login(payload.Username, payload.Password) - if err != nil { - HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) - return - } - c.Set(constants.UserContextKey, loggedInUser) - HandleSuccessWithData(c, token) +type PostLoginParams struct { + Username string `json:"username" validate:"required"` + Password string `json:"password" validate:"required"` } -func PostLogout(c *gin.Context) { - c.Set(constants.UserContextKey, nil) - HandleSuccess(c) +func PostLogin(c *gin.Context, params *PostLoginParams) (response *Response[string], err error) { + userSvc, err := user.GetUserService() + if err != nil { + return GetErrorResponse[string](err) + } + + token, loggedInUser, err := userSvc.Login(params.Username, params.Password) + if err != nil { + return GetErrorResponse[string](errors.ErrorUserUnauthorized) + } + + c.Set(constants.UserContextKey, loggedInUser) + return GetDataResponse(token) +} + +func PostLogout(_ *gin.Context) (response *Response[any], err error) { + return GetDataResponse[any](nil) } diff --git a/core/controllers/router.go b/core/controllers/router.go index 1b7846a2..95aa7805 100644 --- a/core/controllers/router.go +++ b/core/controllers/router.go @@ -271,7 +271,7 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodGet, Path: "/:id/files/get", - HandlerFunc: GetSpiderFile, + HandlerFunc: GetSpiderFileContent, }, { Method: http.MethodGet, @@ -324,8 +324,6 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetSpiderResults, }, }...)) - groups.AnonymousGroup.GET("/openapi.json", GetOpenAPI) - return RegisterController(groups.AuthGroup, "/schedules", NewController[models.Schedule]([]Action{ { Method: http.MethodPost, diff --git a/core/controllers/schedule.go b/core/controllers/schedule.go index a3a0056d..4a5bf363 100644 --- a/core/controllers/schedule.go +++ b/core/controllers/schedule.go @@ -2,22 +2,23 @@ package controllers import ( errors2 "errors" + "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/schedule" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/gin-gonic/gin" + "github.com/juju/errors" "go.mongodb.org/mongo-driver/bson/primitive" ) -func PostSchedule(c *gin.Context) { - var s models.Schedule - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } +type PostScheduleParams struct { + Data models.Schedule `json:"data"` +} +func PostSchedule(c *gin.Context, params *PostScheduleParams) (response *Response[models.Schedule], err error) { + s := params.Data u := GetUserFromContext(c) modelSvc := service.NewModelService[models.Schedule]() @@ -26,141 +27,155 @@ func PostSchedule(c *gin.Context) { s.SetUpdated(u.Id) id, err := modelSvc.InsertOne(s) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Schedule](err) } s.Id = id if s.Enabled { scheduleSvc := schedule.GetScheduleService() if err := scheduleSvc.Enable(s, u.Id); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Schedule](err) } } - HandleSuccessWithData(c, s) + return GetDataResponse(s) } -func PutScheduleById(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +type PutScheduleByIdParams struct { + Id string `path:"id"` + Data models.Schedule `json:"data"` +} + +func PutScheduleById(c *gin.Context, params *PutScheduleByIdParams) (response *Response[models.Schedule], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return - } - var s models.Schedule - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[models.Schedule](errors.BadRequestf("invalid schedule id: %v", err)) } + + s := params.Data if s.Id != id { - HandleErrorBadRequest(c, errors2.New("id in path does not match id in body")) - return + return GetErrorResponse[models.Schedule](errors2.New("id in path does not match id in body")) } modelSvc := service.NewModelService[models.Schedule]() err = modelSvc.ReplaceById(id, s) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Schedule](err) } scheduleSvc := schedule.GetScheduleService() - u := GetUserFromContext(c) if s.Enabled { if err := scheduleSvc.Enable(s, u.Id); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Schedule](err) } } else { if err := scheduleSvc.Disable(s, u.Id); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Schedule](err) } } - HandleSuccessWithData(c, s) + return GetDataResponse(s) } -func PostScheduleEnable(c *gin.Context) { - postScheduleEnableDisableFunc(true)(c) +type PostScheduleEnableDisableParams struct { + Id string `path:"id"` } -func PostScheduleDisable(c *gin.Context) { - postScheduleEnableDisableFunc(false)(c) +func PostScheduleEnable(c *gin.Context, params *PostScheduleEnableDisableParams) (response *Response[any], err error) { + userId := GetUserFromContext(c).Id + return postScheduleEnableDisableFunc(true, userId, params) } -func postScheduleEnableDisableFunc(isEnable bool) func(c *gin.Context) { - return func(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - svc := schedule.GetScheduleService() - s, err := service.NewModelService[models.Schedule]().GetById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - u := GetUserFromContext(c) - if isEnable { - err = svc.Enable(*s, u.Id) - } else { - err = svc.Disable(*s, u.Id) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) - } +func PostScheduleDisable(c *gin.Context, params *PostScheduleEnableDisableParams) (response *Response[any], err error) { + userId := GetUserFromContext(c).Id + return postScheduleEnableDisableFunc(false, userId, params) } -func PostScheduleRun(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +func postScheduleEnableDisableFunc(isEnable bool, userId primitive.ObjectID, params *PostScheduleEnableDisableParams) (response *Response[any], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[any](errors.BadRequestf("invalid schedule id: %v", err)) } - // options - var opts interfaces.SpiderRunOptions - if err := c.ShouldBindJSON(&opts); err != nil { - HandleErrorInternalServerError(c, err) - return + svc := schedule.GetScheduleService() + s, err := service.NewModelService[models.Schedule]().GetById(id) + if err != nil { + return GetErrorResponse[any](err) } - if opts.ScheduleId.IsZero() { - opts.ScheduleId = id + + if isEnable { + err = svc.Enable(*s, userId) + } else { + err = svc.Disable(*s, userId) + } + if err != nil { + return GetErrorResponse[any](err) + } + + return GetDataResponse[any](nil) +} + +type PostScheduleRunParams struct { + Id string `path:"id"` + Mode string `json:"mode"` + NodeIds []string `json:"node_ids"` + Cmd string `json:"cmd"` + Param string `json:"param"` + Priority int `json:"priority"` +} + +func PostScheduleRun(c *gin.Context, params *PostScheduleRunParams) (response *Response[[]primitive.ObjectID], err error) { + userId := GetUserFromContext(c).Id + return postScheduleRunFunc(params, userId) +} + +func postScheduleRunFunc(params *PostScheduleRunParams, userId primitive.ObjectID) (response *Response[[]primitive.ObjectID], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) + if err != nil { + return GetErrorResponse[[]primitive.ObjectID](errors.BadRequestf("invalid schedule id: %v", err)) + } + + var nodeIds []primitive.ObjectID + for _, nodeId := range params.NodeIds { + nodeId, err := primitive.ObjectIDFromHex(nodeId) + if err != nil { + return GetErrorResponse[[]primitive.ObjectID](errors.BadRequestf("invalid node id: %v", err)) + } + nodeIds = append(nodeIds, nodeId) + } + + opts := interfaces.SpiderRunOptions{ + Mode: params.Mode, + NodeIds: nodeIds, + Cmd: params.Cmd, + Param: params.Param, + Priority: params.Priority, + ScheduleId: id, + UserId: userId, } // schedule sch, err := service.NewModelService[models.Schedule]().GetById(id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } // spider s, err := service.NewModelService[models.Spider]().GetById(sch.SpiderId) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } // user - if u := GetUserFromContext(c); u != nil { - opts.UserId = u.GetId() - } + opts.UserId = userId // schedule tasks taskIds, err := admin.GetSpiderAdminService().Schedule(s.Id, &opts) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } - HandleSuccessWithData(c, taskIds) + return GetDataResponse(taskIds) } diff --git a/core/controllers/setting.go b/core/controllers/setting.go index de691ecb..f04bb56b 100644 --- a/core/controllers/setting.go +++ b/core/controllers/setting.go @@ -1,7 +1,6 @@ package controllers import ( - "errors" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/gin-gonic/gin" @@ -9,84 +8,70 @@ import ( "go.mongodb.org/mongo-driver/mongo" ) -func GetSetting(c *gin.Context) { - // key - key := c.Param("key") - - // setting - s, err := service.NewModelService[models.Setting]().GetOne(bson.M{"key": key}, nil) - if err != nil { - if errors.Is(err, mongo.ErrNoDocuments) { - HandleSuccess(c) - return - } - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, s) +type GetSettingParams struct { + Key string `path:"key" validate:"required"` } -func PostSetting(c *gin.Context) { - // key - key := c.Param("key") - - // settings - var s models.Setting - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorInternalServerError(c, err) - return +func GetSetting(_ *gin.Context, params *GetSettingParams) (response *Response[models.Setting], err error) { + // setting + s, err := service.NewModelService[models.Setting]().GetOne(bson.M{"key": params.Key}, nil) + if err != nil { + if err == mongo.ErrNoDocuments { + return GetDataResponse(models.Setting{}) + } + return GetErrorResponse[models.Setting](err) } + return GetDataResponse(*s) +} + +type PostSettingParams struct { + Key string `path:"key" validate:"required"` + Data models.Setting `json:"data"` +} + +func PostSetting(c *gin.Context, params *PostSettingParams) (response *Response[models.Setting], err error) { + s := params.Data if s.Key == "" { - s.Key = key + s.Key = params.Key } u := GetUserFromContext(c) - s.SetCreated(u.Id) s.SetUpdated(u.Id) id, err := service.NewModelService[models.Setting]().InsertOne(s) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Setting](err) } s.Id = id - HandleSuccessWithData(c, s) + return GetDataResponse(s) } -func PutSetting(c *gin.Context) { - // key - key := c.Param("key") - - // settings - var s models.Setting - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorInternalServerError(c, err) - return - } +type PutSettingParams struct { + Key string `path:"key" validate:"required"` + Data models.Setting `json:"data"` +} +func PutSetting(c *gin.Context, params *PutSettingParams) (response *Response[models.Setting], err error) { modelSvc := service.NewModelService[models.Setting]() // setting - _s, err := modelSvc.GetOne(bson.M{"key": key}, nil) + existingSetting, err := modelSvc.GetOne(bson.M{"key": params.Key}, nil) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Setting](err) } u := GetUserFromContext(c) // save - _s.Value = s.Value - _s.SetUpdated(u.Id) - err = modelSvc.ReplaceOne(bson.M{"key": key}, *_s) + existingSetting.Value = params.Data.Value + existingSetting.SetUpdated(u.Id) + err = modelSvc.ReplaceOne(bson.M{"key": params.Key}, *existingSetting) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Setting](err) } - HandleSuccess(c) + return GetDataResponse(*existingSetting) } diff --git a/core/controllers/spider.go b/core/controllers/spider.go index 06008862..8bbbb5c3 100644 --- a/core/controllers/spider.go +++ b/core/controllers/spider.go @@ -2,6 +2,7 @@ package controllers import ( "math" + "mime/multipart" "os" "path/filepath" "sync" @@ -528,105 +529,189 @@ func DeleteSpiderList(_ *gin.Context, params *DeleteSpiderListParams) (response return GetDataResponse(models.Spider{}) } -func GetSpiderListDir(c *gin.Context, params *GetBaseFileListDirParams) (response *Response[[]interfaces.FsFileInfo], err error) { - rootPath, err := getSpiderRootPathByContext(c) - if err != nil { - HandleErrorForbidden(c, err) - return - } - return GetBaseFileListDir(rootPath, params) +type GetSpiderListDirParams struct { + Id string `path:"id"` + Path string `query:"path"` } -func GetSpiderFile(c *gin.Context, params *GetBaseFileFileParams) (response *Response[string], err error) { +func GetSpiderListDir(c *gin.Context, params *GetSpiderListDirParams) (response *Response[[]interfaces.FsFileInfo], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[[]interfaces.FsFileInfo](err) } - return GetBaseFileFile(rootPath, params) + return GetBaseFileListDir(rootPath, params.Path) } -func GetSpiderFileInfo(c *gin.Context) { +type GetSpiderFileContentParams struct { + Id string `path:"id"` + Path string `query:"path"` +} + +func GetSpiderFileContent(c *gin.Context, params *GetSpiderFileContentParams) (response *Response[string], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[string](err) } - GetBaseFileFileInfo(rootPath, c) + return GetBaseFileContent(rootPath, params.Path) } -func PostSpiderSaveFile(c *gin.Context) { +type GetSpiderFileInfoParams struct { + Id string `path:"id"` + Path string `query:"path"` +} + +func GetSpiderFileInfo(c *gin.Context, params *GetSpiderFileInfoParams) (response *Response[interfaces.FsFileInfo], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[interfaces.FsFileInfo](err) } - PostBaseFileSaveFile(rootPath, c) + return GetBaseFileInfo(rootPath, params.Path) } -func PostSpiderSaveFiles(c *gin.Context) { +type PostSpiderSaveFileParams struct { + Id string `path:"id"` + Path string `json:"path"` + Data string `json:"data"` + File *multipart.FileHeader `form:"file"` +} + +func PostSpiderSaveFile(c *gin.Context, params *PostSpiderSaveFileParams) (response *Response[any], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[any](err) + } + if c.GetHeader("Content-Type") == "application/json" { + return PostBaseFileSaveOne(rootPath, params.Path, params.Data) + } else { + return PostBaseFileSaveOneForm(rootPath, params.Path, params.File) } - targetDirectory := c.PostForm("targetDirectory") - PostBaseFileSaveFiles(filepath.Join(rootPath, targetDirectory), c) } -func PostSpiderSaveDir(c *gin.Context) { +type PostSpiderSaveFilesParams struct { + Id string `path:"id"` + TargetDirectory string `form:"targetDirectory"` +} + +func PostSpiderSaveFiles(c *gin.Context, params *PostSpiderSaveFilesParams) (response *Response[any], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[any](err) } - PostBaseFileSaveDir(rootPath, c) + form, err := c.MultipartForm() + if err != nil { + return GetErrorResponse[any](err) + } + return PostBaseFileSaveMany(filepath.Join(rootPath, params.TargetDirectory), form) } -func PostSpiderRenameFile(c *gin.Context) { +type PostSpiderSaveDirParams struct { + Id string `path:"id"` + Path string `json:"path"` +} + +func PostSpiderSaveDir(c *gin.Context, params *PostSpiderSaveDirParams) (response *Response[any], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[any](err) } - PostBaseFileRenameFile(rootPath, c) + return PostBaseFileSaveDir(rootPath, params.Path) } -func DeleteSpiderFile(c *gin.Context) { +type PostSpiderRenameFileParams struct { + Id string `path:"id"` + Path string `json:"path"` + NewPath string `json:"newPath"` +} + +func PostSpiderRenameFile(c *gin.Context, params *PostSpiderRenameFileParams) (response *Response[any], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[any](err) } - DeleteBaseFileFile(rootPath, c) + return PostBaseFileRename(rootPath, params.Path, params.NewPath) } -func PostSpiderCopyFile(c *gin.Context) { +type DeleteSpiderFileParams struct { + Id string `path:"id"` + Path string `json:"path"` +} + +func DeleteSpiderFile(c *gin.Context, params *DeleteSpiderFileParams) (response *Response[any], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[any](err) } - PostBaseFileCopyFile(rootPath, c) + return DeleteBaseFile(rootPath, params.Path) } -func PostSpiderExport(c *gin.Context) { +type PostSpiderCopyFileParams struct { + Id string `path:"id"` + Path string `json:"path"` + NewPath string `json:"new_path"` +} + +func PostSpiderCopyFile(c *gin.Context, params *PostSpiderCopyFileParams) (response *Response[any], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[any](err) } - PostBaseFileExport(rootPath, c) + return PostBaseFileCopy(rootPath, params.Path, params.NewPath) } -func PostSpiderRun(c *gin.Context) (response *Response[[]primitive.ObjectID], err error) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +type PostSpiderExportParams struct { + Id string `path:"id"` +} + +func PostSpiderExport(c *gin.Context, params *PostSpiderExportParams) (err error) { + rootPath, err := getSpiderRootPathByContext(c) + if err != nil { + return err + } + return PostBaseFileExport(rootPath, c) +} + +type PostSpiderRunParams struct { + Id string `path:"id"` + Mode string `json:"mode"` + NodeIds []string `json:"node_ids"` + Cmd string `json:"cmd"` + Param string `json:"param"` + ScheduleId string `json:"schedule_id"` + Priority int `json:"priority"` +} + +func PostSpiderRun(c *gin.Context, params *PostSpiderRunParams) (response *Response[[]primitive.ObjectID], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { return GetErrorResponse[[]primitive.ObjectID](errors.BadRequestf("invalid id format")) } // options - var opts interfaces.SpiderRunOptions + var nodeIds []primitive.ObjectID + if len(params.NodeIds) > 0 { + for _, id := range params.NodeIds { + nodeId, err := primitive.ObjectIDFromHex(id) + if err != nil { + return GetErrorResponse[[]primitive.ObjectID](errors.BadRequestf("invalid node id format")) + } + nodeIds = append(nodeIds, nodeId) + } + } + var scheduleId primitive.ObjectID + if params.ScheduleId != "" { + scheduleId, err = primitive.ObjectIDFromHex(params.ScheduleId) + if err != nil { + return GetErrorResponse[[]primitive.ObjectID](errors.BadRequestf("invalid schedule id format")) + } + } + opts := interfaces.SpiderRunOptions{ + Mode: params.Mode, + NodeIds: nodeIds, + Cmd: params.Cmd, + Param: params.Param, + ScheduleId: scheduleId, + Priority: params.Priority, + } if err := c.ShouldBindJSON(&opts); err != nil { return GetErrorResponse[[]primitive.ObjectID](err) } @@ -642,21 +727,22 @@ func PostSpiderRun(c *gin.Context) (response *Response[[]primitive.ObjectID], er return GetErrorResponse[[]primitive.ObjectID](err) } - HandleSuccessWithData(c, taskIds) return GetDataResponse(taskIds) } -func GetSpiderResults(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +type GetSpiderResultsParams struct { + Id string `path:"id"` +} + +func GetSpiderResults(c *gin.Context, params *GetSpiderResultsParams) (response *ListResponse[bson.M], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorListResponse[bson.M](errors.BadRequestf("invalid id format")) } s, err := service.NewModelService[models.Spider]().GetById(id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[bson.M](err) } // params @@ -672,17 +758,15 @@ func GetSpiderResults(c *gin.Context) { Limit: pagination.Size, })).All(&results) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[bson.M](err) } total, err := mongo2.GetMongoCol(s.ColName).Count(mongo2.GetMongoQuery(query)) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[bson.M](err) } - HandleSuccessWithListData(c, results, total) + return GetListResponse(results, total) } func getSpiderFsSvc(s *models.Spider) (svc interfaces.FsService, err error) { diff --git a/core/controllers/stats.go b/core/controllers/stats.go index 92ec6452..14a64f4e 100644 --- a/core/controllers/stats.go +++ b/core/controllers/stats.go @@ -1,10 +1,11 @@ package controllers import ( + "time" + "github.com/crawlab-team/crawlab/core/stats" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" - "time" ) var statsDefaultQuery = bson.M{ @@ -13,29 +14,53 @@ var statsDefaultQuery = bson.M{ }, } -func GetStatsOverview(c *gin.Context) { - data, err := stats.GetStatsService().GetOverviewStats(statsDefaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, data) +type GetStatsOverviewParams struct { + Query bson.M `json:"query"` } -func GetStatsDaily(c *gin.Context) { - data, err := stats.GetStatsService().GetDailyStats(statsDefaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return +func GetStatsOverview(_ *gin.Context, params *GetStatsOverviewParams) (response *Response[bson.M], err error) { + query := statsDefaultQuery + if params.Query != nil { + query = params.Query } - HandleSuccessWithData(c, data) + + data, err := stats.GetStatsService().GetOverviewStats(query) + if err != nil { + return GetErrorResponse[bson.M](err) + } + return GetDataResponse(data.(bson.M)) } -func GetStatsTasks(c *gin.Context) { - data, err := stats.GetStatsService().GetTaskStats(statsDefaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, data) +type GetStatsDailyParams struct { + Query bson.M `json:"query"` +} + +func GetStatsDaily(_ *gin.Context, params *GetStatsDailyParams) (response *Response[bson.M], err error) { + query := statsDefaultQuery + if params.Query != nil { + query = params.Query + } + + data, err := stats.GetStatsService().GetDailyStats(query) + if err != nil { + return GetErrorResponse[bson.M](err) + } + return GetDataResponse(data.(bson.M)) +} + +type GetStatsTasksParams struct { + Query bson.M `json:"query"` +} + +func GetStatsTasks(_ *gin.Context, params *GetStatsTasksParams) (response *Response[bson.M], err error) { + query := statsDefaultQuery + if params.Query != nil { + query = params.Query + } + + data, err := stats.GetStatsService().GetTaskStats(query) + if err != nil { + return GetErrorResponse[bson.M](err) + } + return GetDataResponse(data.(bson.M)) } diff --git a/core/controllers/sync.go b/core/controllers/sync.go index e7a77f8e..81555c53 100644 --- a/core/controllers/sync.go +++ b/core/controllers/sync.go @@ -1,30 +1,36 @@ package controllers import ( + "path/filepath" + + "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/utils" "github.com/gin-gonic/gin" - "net/http" - "path/filepath" ) -func GetSyncScan(c *gin.Context) { - id := c.Param("id") - path := c.Query("path") +type GetSyncScanParams struct { + Id string `path:"id" validate:"required"` + Path string `query:"path"` +} +func GetSyncScan(_ *gin.Context, params *GetSyncScanParams) (response *Response[map[string]entity.FsFileInfo], err error) { workspacePath := utils.GetWorkspace() - dirPath := filepath.Join(workspacePath, id, path) + dirPath := filepath.Join(workspacePath, params.Id, params.Path) files, err := utils.ScanDirectory(dirPath) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[map[string]entity.FsFileInfo](err) } - c.AbortWithStatusJSON(http.StatusOK, files) + return GetDataResponse(files) } -func GetSyncDownload(c *gin.Context) { - id := c.Param("id") - path := c.Query("path") - workspacePath := utils.GetWorkspace() - filePath := filepath.Join(workspacePath, id, path) - c.File(filePath) +type GetSyncDownloadParams struct { + Id string `path:"id" validate:"required"` + Path string `query:"path" validate:"required"` +} + +func GetSyncDownload(c *gin.Context, params *GetSyncDownloadParams) (err error) { + workspacePath := utils.GetWorkspace() + filePath := filepath.Join(workspacePath, params.Id, params.Path) + c.File(filePath) + return nil } diff --git a/core/controllers/system_info.go b/core/controllers/system_info.go index 45ff983a..b95b0900 100644 --- a/core/controllers/system_info.go +++ b/core/controllers/system_info.go @@ -6,10 +6,10 @@ import ( "github.com/gin-gonic/gin" ) -func GetSystemInfo(c *gin.Context) { - info := &entity.SystemInfo{ +func GetSystemInfo(c *gin.Context) (response *Response[entity.SystemInfo], err error) { + info := entity.SystemInfo{ Edition: utils.GetEdition(), Version: utils.GetVersion(), } - HandleSuccessWithData(c, info) + return GetDataResponse(info) } diff --git a/core/controllers/task.go b/core/controllers/task.go index e341841f..23299c7f 100644 --- a/core/controllers/task.go +++ b/core/controllers/task.go @@ -2,6 +2,11 @@ package controllers import ( "errors" + "os" + "path/filepath" + "strings" + "sync" + "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" @@ -15,35 +20,31 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" - "os" - "path/filepath" - "strings" - "sync" ) -func GetTaskById(c *gin.Context) { +type GetTaskByIdParams struct { + Id string `path:"id"` +} + +func GetTaskById(_ *gin.Context, params *GetTaskByIdParams) (response *Response[models.Task], err error) { // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[models.Task](err) } // task t, err := service.NewModelService[models.Task]().GetById(id) if errors.Is(err, mongo2.ErrNoDocuments) { - HandleErrorNotFound(c, err) - return + return GetErrorResponse[models.Task](err) } if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Task](err) } // skip if task status is pending if t.Status == constants.TaskStatusPending { - HandleSuccessWithData(c, t) - return + return GetDataResponse(*t) } // spider @@ -64,40 +65,41 @@ func GetTaskById(c *gin.Context) { // task stat t.Stat, _ = service.NewModelService[models.TaskStat]().GetById(id) - HandleSuccessWithData(c, t) + return GetDataResponse(*t) } -func GetTaskList(c *gin.Context, params *GetListParams) { - withStats := c.Query("stats") - if withStats == "" { - NewController[models.Task]().GetList(c, params) - return +type GetTaskListParams struct { + *GetListParams + Stats bool `query:"stats"` +} + +func GetTaskList(c *gin.Context, params *GetTaskListParams) (response *ListResponse[models.Task], err error) { + if params.Stats { + return NewController[models.Task]().GetList(c, params.GetListParams) } - // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) + // get query + query, err := GetFilterQueryFromListParams(params.GetListParams) + if err != nil { + return GetErrorListResponse[models.Task](err) + } // get tasks tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo3.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, + Sort: params.Sort, + Skip: params.Size * (params.Page - 1), + Limit: params.Size, }) if err != nil { if errors.Is(err, mongo2.ErrNoDocuments) { - HandleErrorNotFound(c, err) - } else { - HandleErrorInternalServerError(c, err) + return GetErrorListResponse[models.Task](err) } - return + return GetErrorListResponse[models.Task](err) } // check empty list if len(tasks) == 0 { - HandleSuccessWithListData(c, nil, 0) - return + return GetListResponse[models.Task](nil, 0) } // ids @@ -111,8 +113,7 @@ func GetTaskList(c *gin.Context, params *GetListParams) { // total count total, err := service.NewModelService[models.Task]().Count(query) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.Task](err) } // stat list @@ -122,8 +123,7 @@ func GetTaskList(c *gin.Context, params *GetListParams) { }, }, nil) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.Task](err) } // cache stat list to dict @@ -139,8 +139,7 @@ func GetTaskList(c *gin.Context, params *GetListParams) { }, }, nil) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.Task](err) } // cache spider list to dict @@ -164,15 +163,17 @@ func GetTaskList(c *gin.Context, params *GetListParams) { } } - // response - HandleSuccessWithListData(c, tasks, total) + return GetListResponse(tasks, total) } -func DeleteTaskById(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +type DeleteTaskByIdParams struct { + Id string `path:"id"` +} + +func DeleteTaskById(_ *gin.Context, params *DeleteTaskByIdParams) (response *Response[any], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[any](err) } // delete in db @@ -201,8 +202,7 @@ func DeleteTaskById(c *gin.Context) { return nil }); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[any](err) } // delete task logs @@ -211,23 +211,28 @@ func DeleteTaskById(c *gin.Context) { logger.Warnf("failed to remove task log directory: %s", logPath) } - HandleSuccess(c) + return GetDataResponse[any](nil) } -func DeleteList(c *gin.Context) { - var payload struct { - Ids []primitive.ObjectID `json:"ids"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return +type DeleteTaskListParams struct { + Ids []string `json:"ids"` +} + +func DeleteList(_ *gin.Context, params *DeleteTaskListParams) (response *Response[any], err error) { + var ids []primitive.ObjectID + for _, id := range params.Ids { + id, err := primitive.ObjectIDFromHex(id) + if err != nil { + return GetErrorResponse[any](err) + } + ids = append(ids, id) } if err := mongo3.RunTransaction(func(context mongo2.SessionContext) error { // delete tasks if err := service.NewModelService[models.Task]().DeleteMany(bson.M{ "_id": bson.M{ - "$in": payload.Ids, + "$in": ids, }, }); err != nil { return err @@ -236,7 +241,7 @@ func DeleteList(c *gin.Context) { // delete task stats if err := service.NewModelService[models.Task]().DeleteMany(bson.M{ "_id": bson.M{ - "$in": payload.Ids, + "$in": ids, }, }); err != nil { logger.Warnf("delete task stat error: %s", err.Error()) @@ -245,56 +250,66 @@ func DeleteList(c *gin.Context) { return nil }); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[any](err) } // delete tasks logs wg := sync.WaitGroup{} - wg.Add(len(payload.Ids)) - for _, id := range payload.Ids { - go func(id string) { + wg.Add(len(ids)) + for _, id := range ids { + go func(taskId primitive.ObjectID) { // delete task logs - logPath := filepath.Join(utils.GetTaskLogPath(), id) + logPath := filepath.Join(utils.GetTaskLogPath(), taskId.Hex()) if err := os.RemoveAll(logPath); err != nil { logger.Warnf("failed to remove task log directory: %s", logPath) } wg.Done() - }(id.Hex()) + }(id) } wg.Wait() - HandleSuccess(c) + return GetDataResponse[any](nil) } -func PostTaskRun(c *gin.Context) { - // task - var t models.Task - if err := c.ShouldBindJSON(&t); err != nil { - HandleErrorBadRequest(c, err) - return +type PostTaskRunParams struct { + SpiderId string `json:"spider_id" validate:"required"` + Mode string `json:"mode"` + NodeIds []string `json:"node_ids"` + Cmd string `json:"cmd"` + Param string `json:"param"` + Priority int `json:"priority"` +} + +func PostTaskRun(c *gin.Context, params *PostTaskRunParams) (response *Response[[]primitive.ObjectID], err error) { + spiderId, err := primitive.ObjectIDFromHex(params.SpiderId) + if err != nil { + return GetErrorResponse[[]primitive.ObjectID](err) } - // validate spider id - if t.SpiderId.IsZero() { - HandleErrorBadRequest(c, errors.New("spider id is required")) - return + var nodeIds []primitive.ObjectID + if params.NodeIds != nil { + for _, nodeId := range params.NodeIds { + nodeId, err := primitive.ObjectIDFromHex(nodeId) + if err != nil { + return GetErrorResponse[[]primitive.ObjectID](err) + } + nodeIds = append(nodeIds, nodeId) + } } // spider - s, err := service.NewModelService[models.Spider]().GetById(t.SpiderId) + s, err := service.NewModelService[models.Spider]().GetById(spiderId) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } // options opts := &interfaces.SpiderRunOptions{ - Mode: t.Mode, - NodeIds: t.NodeIds, - Cmd: t.Cmd, - Param: t.Param, - Priority: t.Priority, + Mode: params.Mode, + NodeIds: nodeIds, + Cmd: params.Cmd, + Param: params.Param, + Priority: params.Priority, } // user @@ -306,27 +321,27 @@ func PostTaskRun(c *gin.Context) { adminSvc := admin.GetSpiderAdminService() taskIds, err := adminSvc.Schedule(s.Id, opts) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } - HandleSuccessWithData(c, taskIds) - + return GetDataResponse(taskIds) } -func PostTaskRestart(c *gin.Context) { +type PostTaskRestartParams struct { + Id string `path:"id"` +} + +func PostTaskRestart(c *gin.Context, params *PostTaskRestartParams) (response *Response[[]primitive.ObjectID], err error) { // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } // task t, err := service.NewModelService[models.Task]().GetById(id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } // options @@ -353,89 +368,73 @@ func PostTaskRestart(c *gin.Context) { adminSvc := admin.GetSpiderAdminService() taskIds, err := adminSvc.Schedule(t.SpiderId, opts) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } - HandleSuccessWithData(c, taskIds) + return GetDataResponse(taskIds) } -func PostTaskCancel(c *gin.Context) { - type Payload struct { - Force bool `json:"force,omitempty"` - } +type PostTaskCancelParams struct { + Id string `path:"id"` + Force bool `json:"force,omitempty"` +} +func PostTaskCancel(c *gin.Context, params *PostTaskCancelParams) (response *Response[any], err error) { // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // payload - var p Payload - if err := c.ShouldBindJSON(&p); err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[any](err) } // task t, err := service.NewModelService[models.Task]().GetById(id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[any](err) } // validate if !utils.IsCancellable(t.Status) { - HandleErrorInternalServerError(c, errors.New("task is not cancellable")) - return + return GetErrorResponse[any](errors.New("task is not cancellable")) } u := GetUserFromContext(c) // cancel schedulerSvc := scheduler.GetTaskSchedulerService() - err = schedulerSvc.Cancel(id, u.Id, p.Force) + err = schedulerSvc.Cancel(id, u.Id, params.Force) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[any](err) } - HandleSuccess(c) + return GetDataResponse[any](nil) } -func GetTaskLogs(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } +type GetTaskLogsParams struct { + Id string `path:"id"` + Page int `query:"page"` + Size int `query:"size"` +} - // pagination - p, err := GetPagination(c) +func GetTaskLogs(_ *gin.Context, params *GetTaskLogsParams) (response *ListResponse[string], err error) { + // id + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorListResponse[string](err) } // logs logDriver := log.GetFileLogDriver() - logs, err := logDriver.Find(id.Hex(), "", (p.Page-1)*p.Size, p.Size) + logs, err := logDriver.Find(id.Hex(), "", (params.Page-1)*params.Size, params.Size) if err != nil { if strings.HasSuffix(err.Error(), "Status:404 Not Found") { - HandleSuccess(c) - return + return GetListResponse[string](nil, 0) } - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[string](err) } total, err := logDriver.Count(id.Hex(), "") if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[string](err) } - HandleSuccessWithListData(c, logs, total) + return GetListResponse(logs, total) } diff --git a/core/controllers/token.go b/core/controllers/token.go index bfa079a3..1fab41c2 100644 --- a/core/controllers/token.go +++ b/core/controllers/token.go @@ -1,55 +1,51 @@ package controllers import ( - "errors" - "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/user" "github.com/gin-gonic/gin" + "github.com/juju/errors" mongo2 "go.mongodb.org/mongo-driver/mongo" ) -func PostToken(c *gin.Context) { - var t models.Token - if err := c.ShouldBindJSON(&t); err != nil { - HandleErrorBadRequest(c, err) - return - } +type PostTokenParams struct { + Data models.Token `json:"data"` +} + +func PostToken(c *gin.Context, params *PostTokenParams) (response *Response[models.Token], err error) { + t := params.Data svc, err := user.GetUserService() if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Token](err) } + u := GetUserFromContext(c) t.SetCreated(u.Id) t.SetUpdated(u.Id) t.Token, err = svc.MakeToken(u) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Token](err) } - _, err = service.NewModelService[models.Token]().InsertOne(t) + + id, err := service.NewModelService[models.Token]().InsertOne(t) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Token](err) } - HandleSuccess(c) + t.Id = id + + return GetDataResponse(t) } -func GetTokenList(c *gin.Context) { +func GetTokenList(c *gin.Context, params *GetListParams) (response *ListResponse[models.Token], err error) { // Get current user from context u := GetUserFromContext(c) - // Get pagination, filter query, and sort options - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - - // If query is nil, initialize it - if query == nil { - query = make(map[string]interface{}) + // Get filter query + query, err := GetFilterQueryFromListParams(params) + if err != nil { + return GetErrorListResponse[models.Token](errors.BadRequestf("invalid request parameters: %v", err)) } // Add filter for tokens created by the current user @@ -57,26 +53,22 @@ func GetTokenList(c *gin.Context) { // Get tokens with pagination tokens, err := service.NewModelService[models.Token]().GetMany(query, &mongo.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, + Sort: params.Sort, + Skip: params.Size * (params.Page - 1), + Limit: params.Size, }) if err != nil { - if errors.Is(err, mongo2.ErrNoDocuments) { - HandleSuccessWithListData(c, nil, 0) - } else { - HandleErrorInternalServerError(c, err) + if err == mongo2.ErrNoDocuments { + return GetListResponse([]models.Token{}, 0) } - return + return GetErrorListResponse[models.Token](err) } // Count total tokens for pagination total, err := service.NewModelService[models.Token]().Count(query) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.Token](err) } - // Return tokens with total count - HandleSuccessWithListData(c, tokens, total) + return GetListResponse(tokens, total) } diff --git a/core/controllers/user.go b/core/controllers/user.go index 184a0dd2..435b534f 100644 --- a/core/controllers/user.go +++ b/core/controllers/user.go @@ -15,11 +15,10 @@ import ( mongo2 "go.mongodb.org/mongo-driver/mongo" ) -func GetUserById(c *gin.Context, params *GetByIdParams) (response *Response[models.User], err error) { +func GetUserById(_ *gin.Context, params *GetByIdParams) (response *Response[models.User], err error) { id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[models.User](errors.BadRequestf("invalid user id: %v", err)) } return getUserById(id) } @@ -80,7 +79,7 @@ func GetUserList(_ *gin.Context, params *GetListParams) (response *ListResponse[ } // response - return GetListResponse[models.User](users, total) + return GetListResponse(users, total) } type PostUserParams struct { @@ -126,7 +125,7 @@ func PostUser(c *gin.Context, params *PostUserParams) (response *Response[models return GetErrorResponse[models.User](err) } - return GetDataResponse[models.User](*result) + return GetDataResponse(*result) } func PutUserById(c *gin.Context, params *PutByIdParams[models.User]) (response *Response[models.User], err error) { @@ -168,7 +167,7 @@ func DeleteUserById(_ *gin.Context, params *DeleteByIdParams) (response *Respons return GetErrorResponse[models.User](err) } - return GetDataResponse[models.User](models.User{}) + return GetDataResponse(models.User{}) } func DeleteUserList(_ *gin.Context, params *DeleteListParams) (response *Response[models.User], err error) { @@ -205,7 +204,7 @@ func DeleteUserList(_ *gin.Context, params *DeleteListParams) (response *Respons return GetErrorResponse[models.User](err) } - return GetDataResponse[models.User](models.User{}) + return GetDataResponse(models.User{}) } func GetUserMe(c *gin.Context) (response *Response[models.User], err error) { @@ -253,7 +252,7 @@ func getUserById(userId primitive.ObjectID) (response *Response[models.User], er } } - return GetDataResponse[models.User](*user) + return GetDataResponse(*user) } func getUserByIdWithRoutes(userId primitive.ObjectID) (response *Response[models.User], err error) { @@ -284,7 +283,7 @@ func getUserByIdWithRoutes(userId primitive.ObjectID) (response *Response[models user.Routes = role.Routes } - return GetDataResponse[models.User](*user) + return GetDataResponse(*user) } func putUser(userId, by primitive.ObjectID, user models.User) (response *Response[models.User], err error) { @@ -319,7 +318,7 @@ func putUser(userId, by primitive.ObjectID, user models.User) (response *Respons } // handle success - return GetDataResponse[models.User](user) + return GetDataResponse(user) } func postUserChangePassword(userId, by primitive.ObjectID, password string) (response *Response[models.User], err error) { @@ -338,5 +337,5 @@ func postUserChangePassword(userId, by primitive.ObjectID, password string) (res return GetErrorResponse[models.User](err) } - return GetDataResponse[models.User](models.User{}) + return GetDataResponse(models.User{}) } From 43691d0d8affd8fe13218a817b32781fd5f3802d Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 13 Mar 2025 17:43:55 +0800 Subject: [PATCH 04/13] refactor: enhance controller parameter handling and error responses - Refactored export and filter controller methods to utilize structured parameter types for improved clarity and maintainability. - Consolidated error handling and response generation, ensuring consistent API responses across methods. - Updated function signatures to eliminate unnecessary context parameters, enhancing type safety and readability. - Improved handling of query parameters and conditions, streamlining the logic for better performance and maintainability. --- core/controllers/export.go | 96 +++++++++++++++++++------------------- core/controllers/filter.go | 32 +++++++++---- 2 files changed, 70 insertions(+), 58 deletions(-) diff --git a/core/controllers/export.go b/core/controllers/export.go index e4c022eb..3282cacc 100644 --- a/core/controllers/export.go +++ b/core/controllers/export.go @@ -1,88 +1,88 @@ package controllers import ( - "errors" "fmt" + "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/export" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/gin-gonic/gin" + "github.com/juju/errors" ) -func PostExport(c *gin.Context) { - exportType := c.Param("type") - exportTarget := c.Query("target") - exportFilter, _ := GetFilter(c) +type PostExportParams struct { + Type string `path:"type" validate:"required"` + Target string `query:"target" validate:"required"` + Filter interfaces.Filter `query:"filter"` +} +func PostExport(_ *gin.Context, params *PostExportParams) (response *Response[string], err error) { var exportId string - var err error - switch exportType { + switch params.Type { case constants.ExportTypeCsv: - exportId, err = export.GetCsvService().Export(exportType, exportTarget, exportFilter) + exportId, err = export.GetCsvService().Export(params.Type, params.Target, params.Filter) case constants.ExportTypeJson: - exportId, err = export.GetJsonService().Export(exportType, exportTarget, exportFilter) + exportId, err = export.GetJsonService().Export(params.Type, params.Target, params.Filter) default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - return + return GetErrorResponse[string](errors.BadRequestf("invalid export type: %s", params.Type)) } if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[string](err) } - HandleSuccessWithData(c, exportId) + return GetDataResponse(exportId) } -func GetExport(c *gin.Context) { - exportType := c.Param("type") - exportId := c.Param("id") - - var exp interfaces.Export - var err error - switch exportType { - case constants.ExportTypeCsv: - exp, err = export.GetCsvService().GetExport(exportId) - case constants.ExportTypeJson: - exp, err = export.GetJsonService().GetExport(exportId) - default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, exp) +type GetExportParams struct { + Type string `path:"type" validate:"required"` + Id string `path:"id" validate:"required"` } -func GetExportDownload(c *gin.Context) { - exportType := c.Param("type") - exportId := c.Param("id") - +func GetExport(_ *gin.Context, params *GetExportParams) (response *Response[interfaces.Export], err error) { var exp interfaces.Export - var err error - switch exportType { + switch params.Type { case constants.ExportTypeCsv: - exp, err = export.GetCsvService().GetExport(exportId) + exp, err = export.GetCsvService().GetExport(params.Id) case constants.ExportTypeJson: - exp, err = export.GetJsonService().GetExport(exportId) + exp, err = export.GetJsonService().GetExport(params.Id) default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - return + return GetErrorResponse[interfaces.Export](errors.BadRequestf("invalid export type: %s", params.Type)) } if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[interfaces.Export](err) } - switch exportType { + return GetDataResponse(exp) +} + +type GetExportDownloadParams struct { + Type string `path:"type" validate:"required"` + Id string `path:"id" validate:"required"` +} + +func GetExportDownload(c *gin.Context, params *GetExportDownloadParams) (err error) { + var exp interfaces.Export + switch params.Type { + case constants.ExportTypeCsv: + exp, err = export.GetCsvService().GetExport(params.Id) + case constants.ExportTypeJson: + exp, err = export.GetJsonService().GetExport(params.Id) + default: + return errors.BadRequestf("invalid export type: %s", params.Type) + } + if err != nil { + return err + } + + switch params.Type { case constants.ExportTypeCsv: c.Header("Content-Type", "text/csv") case constants.ExportTypeJson: c.Header("Content-Type", "text/plain") default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) + return errors.BadRequestf("invalid export type: %s", params.Type) } c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", exp.GetDownloadPath())) c.File(exp.GetDownloadPath()) + return nil } diff --git a/core/controllers/filter.go b/core/controllers/filter.go index c7d753d9..d616f2f6 100644 --- a/core/controllers/filter.go +++ b/core/controllers/filter.go @@ -4,23 +4,34 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/mongo" "github.com/gin-gonic/gin" + "github.com/juju/errors" "go.mongodb.org/mongo-driver/bson" mongo2 "go.mongodb.org/mongo-driver/mongo" ) -func GetFilterColFieldOptions(c *gin.Context) { - colName := c.Param("col") - value := c.Param("value") +type GetFilterColFieldOptionsParams struct { + Col string `path:"col" validate:"required"` + Value string `path:"value"` + Label string `path:"label"` + Conditions string `query:"conditions" description:"Filter conditions. Format: [{\"key\":\"name\",\"op\":\"eq\",\"value\":\"test\"}]"` +} + +func GetFilterColFieldOptions(_ *gin.Context, params *GetFilterColFieldOptionsParams) (response *Response[[]entity.FilterSelectOption], err error) { + value := params.Value if value == "" { value = "_id" } - label := c.Param("label") + label := params.Label if label == "" { label = "name" } - query := MustGetFilterQuery(c) + pipelines := mongo2.Pipeline{} - if query != nil { + if params.Conditions != "" { + query, err := GetFilterFromConditionString(params.Conditions) + if err != nil { + return GetErrorResponse[[]entity.FilterSelectOption](errors.Trace(err)) + } pipelines = append(pipelines, bson.D{{"$match", query}}) } pipelines = append( @@ -60,10 +71,11 @@ func GetFilterColFieldOptions(c *gin.Context) { }, }, ) + var options []entity.FilterSelectOption - if err := mongo.GetMongoCol(colName).Aggregate(pipelines, nil).All(&options); err != nil { - HandleErrorInternalServerError(c, err) - return + if err := mongo.GetMongoCol(params.Col).Aggregate(pipelines, nil).All(&options); err != nil { + return GetErrorResponse[[]entity.FilterSelectOption](errors.Trace(err)) } - HandleSuccessWithData(c, options) + + return GetDataResponse(options) } From ce0143ca06c25cafdaf6b4051809fa05dbd89da6 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 13 Mar 2025 18:10:24 +0800 Subject: [PATCH 05/13] refactor: enhance health check function and add comprehensive test coverage - Updated GetHealthFn to return an error for better error handling and clarity. - Introduced a new test file for schedule management, covering various endpoints including creation, retrieval, updating, and deletion of schedules. - Added tests for task management, including task creation, retrieval, updating, and cancellation. - Implemented utility tests for filtering and response generation to ensure consistent API behavior. - Improved logging in the task scheduler service for better traceability. --- core/controllers/health.go | 6 +- core/controllers/schedule_test.go | 376 ++++++++++++++++++++++++++++++ core/controllers/spider_test.go | 5 +- core/controllers/task_test.go | 343 +++++++++++++++++++++++++++ core/controllers/utils_test.go | 178 ++++++++++++++ core/task/scheduler/service.go | 1 + 6 files changed, 906 insertions(+), 3 deletions(-) create mode 100644 core/controllers/schedule_test.go create mode 100644 core/controllers/task_test.go create mode 100644 core/controllers/utils_test.go diff --git a/core/controllers/health.go b/core/controllers/health.go index 8e19adb1..bf687a5e 100644 --- a/core/controllers/health.go +++ b/core/controllers/health.go @@ -1,12 +1,13 @@ package controllers import ( + "errors" "github.com/gin-gonic/gin" "net/http" ) -func GetHealthFn(healthFn func() bool) func(c *gin.Context) { - return func(c *gin.Context) { +func GetHealthFn(healthFn func() bool) func(c *gin.Context) error { + return func(c *gin.Context) (err error) { if healthFn() { c.Writer.Write([]byte("ok")) c.AbortWithStatus(http.StatusOK) @@ -14,5 +15,6 @@ func GetHealthFn(healthFn func() bool) func(c *gin.Context) { } c.Writer.Write([]byte("not ready")) c.AbortWithStatus(http.StatusServiceUnavailable) + return errors.New("not ready") } } diff --git a/core/controllers/schedule_test.go b/core/controllers/schedule_test.go new file mode 100644 index 00000000..9ea635f2 --- /dev/null +++ b/core/controllers/schedule_test.go @@ -0,0 +1,376 @@ +package controllers_test + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/crawlab-team/crawlab/core/controllers" + "github.com/crawlab-team/crawlab/core/middlewares" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/gin-gonic/gin" + "github.com/loopfz/gadgeto/tonic" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +// Helper function to create a test schedule +func createTestSchedule(t *testing.T) (*models.Schedule, primitive.ObjectID) { + // First, create a test spider + spider := models.Spider{ + Name: "Test Spider for Schedule", + ColName: "test_spider_for_schedule", + } + spiderSvc := service.NewModelService[models.Spider]() + spiderId, err := spiderSvc.InsertOne(spider) + require.NoError(t, err) + require.False(t, spiderId.IsZero()) + + // Now create a schedule associated with the spider + schedule := &models.Schedule{ + Name: "Test Schedule", + SpiderId: spiderId, + Cron: "0 0 * * *", // Run daily at midnight + Cmd: "python main.py", + Param: "test param", + Priority: 5, + Enabled: false, // Disabled initially + } + + // Set timestamps + now := time.Now() + schedule.CreatedAt = now + schedule.UpdatedAt = now + + scheduleSvc := service.NewModelService[models.Schedule]() + scheduleId, err := scheduleSvc.InsertOne(*schedule) + require.NoError(t, err) + require.False(t, scheduleId.IsZero()) + + schedule.Id = scheduleId + return schedule, spiderId +} + +// Test PostSchedule endpoint +func TestPostSchedule(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + // Create a test spider first + spider := models.Spider{ + Name: "Test Spider for Schedule Post", + ColName: "test_spider_for_schedule_post", + } + spiderSvc := service.NewModelService[models.Spider]() + spiderId, err := spiderSvc.InsertOne(spider) + require.NoError(t, err) + + // Set up router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.POST("/schedules", nil, tonic.Handler(controllers.PostSchedule, 200)) + + // Create payload + schedule := models.Schedule{ + Name: "Test Schedule for Post", + SpiderId: spiderId, + Cron: "0 0 * * *", + Cmd: "python main.py", + Param: "test schedule param", + Priority: 3, + Enabled: false, + } + + payload := controllers.PostScheduleParams{ + Data: schedule, + } + jsonValue, _ := json.Marshal(payload) + + // Create test request + req, err := http.NewRequest("POST", "/schedules", bytes.NewBuffer(jsonValue)) + req.Header.Set("Authorization", TestToken) + req.Header.Set("Content-Type", "application/json") + require.NoError(t, err) + + // Execute request + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + // Verify response + assert.Equal(t, http.StatusOK, resp.Code) + + var response controllers.Response[models.Schedule] + err = json.Unmarshal(resp.Body.Bytes(), &response) + require.NoError(t, err) + + assert.Equal(t, "ok", response.Status) + assert.False(t, response.Data.Id.IsZero()) + assert.Equal(t, schedule.Name, response.Data.Name) + assert.Equal(t, schedule.SpiderId, response.Data.SpiderId) + assert.Equal(t, schedule.Cron, response.Data.Cron) + assert.Equal(t, schedule.Enabled, response.Data.Enabled) +} + +// Test GetScheduleById endpoint +func TestGetScheduleById(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + // Create a test schedule + schedule, _ := createTestSchedule(t) + + // Set up router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.GET("/schedules/:id", nil, tonic.Handler(controllers.NewController[models.Schedule]().GetById, 200)) + + // Create test request + req, err := http.NewRequest("GET", "/schedules/"+schedule.Id.Hex(), nil) + req.Header.Set("Authorization", TestToken) + require.NoError(t, err) + + // Execute request + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + // Verify response + assert.Equal(t, http.StatusOK, resp.Code) + + var response controllers.Response[models.Schedule] + err = json.Unmarshal(resp.Body.Bytes(), &response) + require.NoError(t, err) + + assert.Equal(t, "ok", response.Status) + assert.Equal(t, schedule.Id, response.Data.Id) + assert.Equal(t, schedule.Name, response.Data.Name) + assert.Equal(t, schedule.Cron, response.Data.Cron) +} + +// Test PutScheduleById endpoint +func TestPutScheduleById(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + // Create a test schedule + schedule, _ := createTestSchedule(t) + + // Set up router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.PUT("/schedules/:id", nil, tonic.Handler(controllers.PutScheduleById, 200)) + + // Update the schedule + updatedSchedule := *schedule + updatedSchedule.Name = "Updated Schedule Name" + updatedSchedule.Cron = "*/5 * * * *" // Every 5 minutes + updatedSchedule.Enabled = true + + payload := controllers.PutScheduleByIdParams{ + Id: schedule.Id.Hex(), + Data: updatedSchedule, + } + jsonValue, _ := json.Marshal(payload) + + // Create test request + req, err := http.NewRequest("PUT", "/schedules/"+schedule.Id.Hex(), bytes.NewBuffer(jsonValue)) + req.Header.Set("Authorization", TestToken) + req.Header.Set("Content-Type", "application/json") + require.NoError(t, err) + + // Execute request + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + // Verify response + assert.Equal(t, http.StatusOK, resp.Code) + + var response controllers.Response[models.Schedule] + err = json.Unmarshal(resp.Body.Bytes(), &response) + require.NoError(t, err) + + assert.Equal(t, "ok", response.Status) + + // Fetch the updated schedule to verify changes + scheduleSvc := service.NewModelService[models.Schedule]() + updatedScheduleFromDB, err := scheduleSvc.GetById(schedule.Id) + require.NoError(t, err) + + assert.Equal(t, "Updated Schedule Name", updatedScheduleFromDB.Name) + assert.Equal(t, "*/5 * * * *", updatedScheduleFromDB.Cron) + assert.True(t, updatedScheduleFromDB.Enabled) +} + +// Test DeleteScheduleById endpoint +func TestDeleteScheduleById(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + // Create a test schedule + schedule, _ := createTestSchedule(t) + + // Set up router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.DELETE("/schedules/:id", nil, tonic.Handler(controllers.NewController[models.Schedule]().DeleteById, 200)) + + // Create test request + req, err := http.NewRequest("DELETE", "/schedules/"+schedule.Id.Hex(), nil) + req.Header.Set("Authorization", TestToken) + require.NoError(t, err) + + // Execute request + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + // Verify response + assert.Equal(t, http.StatusOK, resp.Code) + + // Verify schedule is deleted from database + scheduleSvc := service.NewModelService[models.Schedule]() + _, err = scheduleSvc.GetById(schedule.Id) + assert.Error(t, err) // Should return error as the schedule is deleted +} + +// Test GetScheduleList endpoint +func TestGetScheduleList(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + // Create several test schedules + schedule1, _ := createTestSchedule(t) + schedule2, _ := createTestSchedule(t) + schedule2.Name = "Second Test Schedule" + + scheduleSvc := service.NewModelService[models.Schedule]() + err := scheduleSvc.ReplaceById(schedule2.Id, *schedule2) + require.NoError(t, err) + + // Set up router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.GET("/schedules", nil, tonic.Handler(controllers.NewController[models.Schedule]().GetList, 200)) + + // Create test request + req, err := http.NewRequest("GET", "/schedules?page=1&size=10", nil) + req.Header.Set("Authorization", TestToken) + require.NoError(t, err) + + // Execute request + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + // Verify response + assert.Equal(t, http.StatusOK, resp.Code) + + var response controllers.ListResponse[models.Schedule] + err = json.Unmarshal(resp.Body.Bytes(), &response) + require.NoError(t, err) + + assert.Equal(t, "ok", response.Status) + assert.Equal(t, 2, response.Total) // We created 2 schedules + assert.Equal(t, 2, len(response.Data)) + + // Verify both schedules are in the response + foundSchedule1 := false + foundSchedule2 := false + for _, schedule := range response.Data { + if schedule.Id == schedule1.Id { + foundSchedule1 = true + assert.Equal(t, schedule1.Name, schedule.Name) + } + if schedule.Id == schedule2.Id { + foundSchedule2 = true + assert.Equal(t, "Second Test Schedule", schedule.Name) + } + } + assert.True(t, foundSchedule1, "schedule1 should be in the response") + assert.True(t, foundSchedule2, "schedule2 should be in the response") +} + +// Test EnableSchedule endpoint +func TestEnableSchedule(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + // Create a test schedule (disabled by default) + schedule, _ := createTestSchedule(t) + assert.False(t, schedule.Enabled) + + // Set up router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.POST("/schedules/:id/enable", nil, tonic.Handler(controllers.PostScheduleEnable, 200)) + + // Create test request + req, err := http.NewRequest("POST", "/schedules/"+schedule.Id.Hex()+"/enable", nil) + req.Header.Set("Authorization", TestToken) + require.NoError(t, err) + + // Execute request + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + // Verify response + assert.Equal(t, http.StatusOK, resp.Code) + + // Verify schedule is now enabled + scheduleSvc := service.NewModelService[models.Schedule]() + updatedSchedule, err := scheduleSvc.GetById(schedule.Id) + require.NoError(t, err) + assert.True(t, updatedSchedule.Enabled) +} + +// Test DisableSchedule endpoint +func TestDisableSchedule(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + // Create a test schedule and enable it + schedule, _ := createTestSchedule(t) + schedule.Enabled = true + + scheduleSvc := service.NewModelService[models.Schedule]() + err := scheduleSvc.ReplaceById(schedule.Id, *schedule) + require.NoError(t, err) + + // Set up router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.POST("/schedules/:id/disable", nil, tonic.Handler(controllers.PostScheduleDisable, 200)) + + // Create test request + req, err := http.NewRequest("POST", "/schedules/"+schedule.Id.Hex()+"/disable", nil) + req.Header.Set("Authorization", TestToken) + require.NoError(t, err) + + // Execute request + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + // Verify response + assert.Equal(t, http.StatusOK, resp.Code) + + // Verify schedule is now disabled + updatedSchedule, err := scheduleSvc.GetById(schedule.Id) + require.NoError(t, err) + assert.False(t, updatedSchedule.Enabled) +} diff --git a/core/controllers/spider_test.go b/core/controllers/spider_test.go index d4614222..fe2e3954 100644 --- a/core/controllers/spider_test.go +++ b/core/controllers/spider_test.go @@ -34,7 +34,10 @@ func TestCreateSpider(t *testing.T) { Name: "Test Spider", ColName: "test_spiders", } - jsonValue, _ := json.Marshal(payload) + requestParams := controllers.PostParams[models.Spider]{ + Data: payload, + } + jsonValue, _ := json.Marshal(requestParams) req, _ := http.NewRequest("POST", "/spiders", bytes.NewBuffer(jsonValue)) req.Header.Set("Authorization", TestToken) resp := httptest.NewRecorder() diff --git a/core/controllers/task_test.go b/core/controllers/task_test.go new file mode 100644 index 00000000..363f4d14 --- /dev/null +++ b/core/controllers/task_test.go @@ -0,0 +1,343 @@ +package controllers_test + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/controllers" + "github.com/crawlab-team/crawlab/core/middlewares" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/gin-gonic/gin" + "github.com/loopfz/gadgeto/tonic" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +// Helper function to create a test task +func createTestTask(t *testing.T) (task *models.Task, spiderId primitive.ObjectID) { + // First, create a test spider + spider := models.Spider{ + Name: "Test Spider for Task", + ColName: "test_spider_for_task", + } + spiderSvc := service.NewModelService[models.Spider]() + var err error + spiderId, err = spiderSvc.InsertOne(spider) + require.NoError(t, err) + require.False(t, spiderId.IsZero()) + + // Now create a task associated with the spider + task = &models.Task{ + SpiderId: spiderId, + Status: constants.TaskStatusPending, + Priority: 10, + Mode: constants.RunTypeAllNodes, + Param: "test param", + Cmd: "python main.py", + UserId: TestUserId, + } + + // Set timestamps + now := time.Now() + task.CreatedAt = now + task.UpdatedAt = now + + taskSvc := service.NewModelService[models.Task]() + taskId, err := taskSvc.InsertOne(*task) + require.NoError(t, err) + require.False(t, taskId.IsZero()) + + task.Id = taskId + return task, spiderId +} + +// Test GetTaskById endpoint +func TestGetTaskById(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + // Create a test task + task, _ := createTestTask(t) + + // Set up router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.GET("/tasks/:id", nil, tonic.Handler(controllers.GetTaskById, 200)) + + // Create test request + req, err := http.NewRequest("GET", "/tasks/"+task.Id.Hex(), nil) + req.Header.Set("Authorization", TestToken) + require.NoError(t, err) + + // Execute request + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + // Verify response + assert.Equal(t, http.StatusOK, resp.Code) + + var response controllers.Response[models.Task] + err = json.Unmarshal(resp.Body.Bytes(), &response) + require.NoError(t, err) + + assert.True(t, response.Status == "ok") + assert.Equal(t, task.Id, response.Data.Id) + assert.Equal(t, task.SpiderId, response.Data.SpiderId) + assert.Equal(t, task.Status, response.Data.Status) +} + +// Test GetTaskList endpoint +func TestGetTaskList(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + // Create several test tasks + task1, _ := createTestTask(t) + task2, _ := createTestTask(t) + task2.Status = constants.TaskStatusRunning + + // Use ReplaceById instead of UpdateById with the model + taskSvc := service.NewModelService[models.Task]() + err := taskSvc.ReplaceById(task2.Id, *task2) + require.NoError(t, err) + + // Set up router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.GET("/tasks", nil, tonic.Handler(controllers.GetTaskList, 200)) + + // Create test request + req, err := http.NewRequest("GET", "/tasks?page=1&size=10", nil) + req.Header.Set("Authorization", TestToken) + require.NoError(t, err) + + // Execute request + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + // Verify response + assert.Equal(t, http.StatusOK, resp.Code) + + var response controllers.ListResponse[models.Task] + err = json.Unmarshal(resp.Body.Bytes(), &response) + require.NoError(t, err) + + assert.True(t, response.Status == "ok") + assert.Equal(t, 2, response.Total) // We created 2 tasks + assert.Equal(t, 2, len(response.Data)) + + // Verify both tasks (including task1) are in the response + foundTask1 := false + foundTask2 := false + for _, task := range response.Data { + if task.Id == task1.Id { + foundTask1 = true + assert.Equal(t, constants.TaskStatusPending, task.Status) + } + if task.Id == task2.Id { + foundTask2 = true + assert.Equal(t, constants.TaskStatusRunning, task.Status) + } + } + assert.True(t, foundTask1, "task1 should be in the response") + assert.True(t, foundTask2, "task2 should be in the response") +} + +// Test DeleteTaskById endpoint +func TestDeleteTaskById(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + // Create a test task + task, _ := createTestTask(t) + + // Set up router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.DELETE("/tasks/:id", nil, tonic.Handler(controllers.DeleteTaskById, 200)) + + // Create test request + req, err := http.NewRequest("DELETE", "/tasks/"+task.Id.Hex(), nil) + req.Header.Set("Authorization", TestToken) + require.NoError(t, err) + + // Execute request + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + // Verify response + assert.Equal(t, http.StatusOK, resp.Code) + + // Verify task is deleted from database + taskSvc := service.NewModelService[models.Task]() + _, err = taskSvc.GetById(task.Id) + assert.Error(t, err) // Should return error as the task is deleted +} + +// Test PostTaskRun endpoint +func TestPostTaskRun(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + // Create a test spider + spider := models.Spider{ + Name: "Test Spider for Run", + ColName: "test_spider_for_run", + } + spiderSvc := service.NewModelService[models.Spider]() + spiderId, err := spiderSvc.InsertOne(spider) + require.NoError(t, err) + + // Set up router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.POST("/tasks/run", nil, tonic.Handler(controllers.PostTaskRun, 200)) + + // Create payload + payload := controllers.PostTaskRunParams{ + SpiderId: spiderId.Hex(), + Mode: constants.RunTypeAllNodes, + Cmd: "python main.py", + Param: "test param", + Priority: 1, + } + jsonValue, _ := json.Marshal(payload) + + // Create test request + req, err := http.NewRequest("POST", "/tasks/run", bytes.NewBuffer(jsonValue)) + req.Header.Set("Authorization", TestToken) + req.Header.Set("Content-Type", "application/json") + require.NoError(t, err) + + // Execute request + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + // Verify response - it may fail if the scheduler service is not properly initialized in test environment + // This is more of an integration test, so we'll check the status code but not the exact response + assert.Equal(t, http.StatusOK, resp.Code) +} + +// Test PostTaskCancel endpoint +func TestPostTaskCancel(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + // Create a test task + task, _ := createTestTask(t) + + // Set status to running to make it cancellable + task.Status = constants.TaskStatusRunning + + // Use ReplaceById instead of UpdateById with the model + taskSvc := service.NewModelService[models.Task]() + err := taskSvc.ReplaceById(task.Id, *task) + require.NoError(t, err) + + // Set up router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.POST("/tasks/:id/cancel", nil, tonic.Handler(controllers.PostTaskCancel, 200)) + + // Create payload + payload := controllers.PostTaskCancelParams{ + Force: true, + } + jsonValue, _ := json.Marshal(payload) + + // Create test request + req, err := http.NewRequest("POST", "/tasks/"+task.Id.Hex()+"/cancel", bytes.NewBuffer(jsonValue)) + req.Header.Set("Authorization", TestToken) + req.Header.Set("Content-Type", "application/json") + require.NoError(t, err) + + // Execute request + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + // Verify response - it may fail if the scheduler service is not properly initialized in test environment + // This is more of an integration test, so we'll check the status code but not the exact response + assert.Equal(t, http.StatusOK, resp.Code) +} + +// Test PostTaskRestart endpoint +func TestPostTaskRestart(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + // Create a test task + task, _ := createTestTask(t) + + // Set up router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.POST("/tasks/:id/restart", nil, tonic.Handler(controllers.PostTaskRestart, 200)) + + // Create test request + req, err := http.NewRequest("POST", "/tasks/"+task.Id.Hex()+"/restart", nil) + req.Header.Set("Authorization", TestToken) + require.NoError(t, err) + + // Execute request + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + // Verify response - it may fail if the scheduler service is not properly initialized in test environment + // This is more of an integration test, so we'll check the status code but not the exact response + assert.Equal(t, http.StatusOK, resp.Code) +} + +// Test GetTaskLogs endpoint +func TestGetTaskLogs(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + // Create a test task + task, _ := createTestTask(t) + + // Set up router + router := SetupRouter() + router.Use(middlewares.AuthorizationMiddleware()) + router.GET("/tasks/:id/logs", nil, tonic.Handler(controllers.GetTaskLogs, 200)) + + // Create test request + req, err := http.NewRequest("GET", "/tasks/"+task.Id.Hex()+"/logs?page=1&size=100", nil) + req.Header.Set("Authorization", TestToken) + require.NoError(t, err) + + // Execute request + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + // Verify response + assert.Equal(t, http.StatusOK, resp.Code) + + var response controllers.ListResponse[string] + err = json.Unmarshal(resp.Body.Bytes(), &response) + require.NoError(t, err) + + // Check status is ok - the logs might be empty since we didn't create any, + // but the endpoint should still function correctly + assert.Equal(t, "ok", response.Status) +} diff --git a/core/controllers/utils_test.go b/core/controllers/utils_test.go new file mode 100644 index 00000000..4ace445a --- /dev/null +++ b/core/controllers/utils_test.go @@ -0,0 +1,178 @@ +package controllers_test + +import ( + "testing" + + "github.com/crawlab-team/crawlab/core/controllers" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func TestGetFilterFromConditionString(t *testing.T) { + // Simple condition with string value + condStr := `[{"key":"name","op":"eq","value":"test"}]` + filter, err := controllers.GetFilterFromConditionString(condStr) + require.NoError(t, err) + require.NotNil(t, filter) + require.Len(t, filter.Conditions, 1) + assert.Equal(t, "name", filter.Conditions[0].Key) + assert.Equal(t, "eq", filter.Conditions[0].Op) + assert.Equal(t, "test", filter.Conditions[0].Value) + + // Multiple conditions with different types + condStr = `[{"key":"name","op":"eq","value":"test"},{"key":"priority","op":"gt","value":5}]` + filter, err = controllers.GetFilterFromConditionString(condStr) + require.NoError(t, err) + require.NotNil(t, filter) + require.Len(t, filter.Conditions, 2) + assert.Equal(t, "name", filter.Conditions[0].Key) + assert.Equal(t, "eq", filter.Conditions[0].Op) + assert.Equal(t, "test", filter.Conditions[0].Value) + assert.Equal(t, "priority", filter.Conditions[1].Key) + assert.Equal(t, "gt", filter.Conditions[1].Op) + assert.Equal(t, float64(5), filter.Conditions[1].Value) // JSON parses numbers as float64 + + // Invalid JSON should return error + condStr = `[{"key":"name","op":"eq","value":"test"` + _, err = controllers.GetFilterFromConditionString(condStr) + assert.Error(t, err) +} + +func TestGetFilterQueryFromConditionString(t *testing.T) { + // Simple equality condition + condStr := `[{"key":"name","op":"eq","value":"test"}]` + query, err := controllers.GetFilterQueryFromConditionString(condStr) + require.NoError(t, err) + require.NotNil(t, query) + expected := bson.M{"name": "test"} + assert.Equal(t, expected, query) + + // Greater than condition + condStr = `[{"key":"priority","op":"gt","value":5}]` + query, err = controllers.GetFilterQueryFromConditionString(condStr) + require.NoError(t, err) + require.NotNil(t, query) + expected = bson.M{"priority": bson.M{"$gt": float64(5)}} + assert.Equal(t, expected, query) + + // Multiple conditions + condStr = `[{"key":"name","op":"eq","value":"test"},{"key":"priority","op":"gt","value":5}]` + query, err = controllers.GetFilterQueryFromConditionString(condStr) + require.NoError(t, err) + require.NotNil(t, query) + expected = bson.M{"name": "test", "priority": bson.M{"$gt": float64(5)}} + assert.Equal(t, expected, query) + + // Contains operator + condStr = `[{"key":"name","op":"contains","value":"test"}]` + query, err = controllers.GetFilterQueryFromConditionString(condStr) + require.NoError(t, err) + require.NotNil(t, query) + expectedRegex := bson.M{"name": bson.M{"$regex": "test", "$options": "i"}} + assert.Equal(t, expectedRegex, query) + + // Invalid condition should return error + condStr = `[{"key":"name","op":"invalid_op","value":"test"}]` + _, err = controllers.GetFilterQueryFromConditionString(condStr) + assert.Error(t, err) +} + +func TestGetFilterQueryFromListParams(t *testing.T) { + // No conditions + params := &controllers.GetListParams{} + query, err := controllers.GetFilterQueryFromListParams(params) + require.NoError(t, err) + assert.Nil(t, query) + + // With conditions + params.Conditions = `[{"key":"name","op":"eq","value":"test"}]` + query, err = controllers.GetFilterQueryFromListParams(params) + require.NoError(t, err) + require.NotNil(t, query) + expected := bson.M{"name": "test"} + assert.Equal(t, expected, query) +} + +func TestGetUserFromContext(t *testing.T) { + // Empty context should return nil + c := &gin.Context{} + user := controllers.GetUserFromContext(c) + assert.Nil(t, user) + + // Context with non-user value should return nil + c = &gin.Context{} + c.Set("user", "not a user object") + user = controllers.GetUserFromContext(c) + assert.Nil(t, user) + + // Context with user should return the user + c = &gin.Context{} + expectedUser := &models.User{Username: "test_user"} + expectedUser.Id = primitive.NewObjectID() + c.Set("user", expectedUser) + user = controllers.GetUserFromContext(c) + assert.NotNil(t, user) + assert.Equal(t, expectedUser.Id, user.Id) + assert.Equal(t, expectedUser.Username, user.Username) +} + +func TestGetErrorResponse(t *testing.T) { + // Error response test + err := assert.AnError + resp, _ := controllers.GetErrorResponse[models.Task](err) + assert.Equal(t, err.Error(), resp.Error) + assert.Equal(t, models.Task{}, resp.Data) +} + +func TestGetDataResponse(t *testing.T) { + // Data response test + task := models.Task{ + Status: "running", + Cmd: "python main.py", + Param: "test param", + } + task.Id = primitive.NewObjectID() + + resp, err := controllers.GetDataResponse(task) + require.NoError(t, err) + assert.Equal(t, "ok", resp.Status) + assert.Equal(t, task, resp.Data) + assert.Empty(t, resp.Error) +} + +func TestGetListResponse(t *testing.T) { + // List response test + tasks := []models.Task{ + { + Status: "running", + Cmd: "python main.py", + }, + { + Status: "pending", + Cmd: "python main.py", + }, + } + tasks[0].Id = primitive.NewObjectID() + tasks[1].Id = primitive.NewObjectID() + + total := 2 + resp, err := controllers.GetListResponse(tasks, total) + require.NoError(t, err) + assert.Equal(t, "ok", resp.Status) + assert.Equal(t, tasks, resp.Data) + assert.Equal(t, total, resp.Total) + assert.Empty(t, resp.Error) +} + +func TestGetErrorListResponse(t *testing.T) { + // Error list response test + err := assert.AnError + resp, _ := controllers.GetErrorListResponse[models.Task](err) + assert.Equal(t, err.Error(), resp.Error) + assert.Nil(t, resp.Data) + assert.Equal(t, 0, resp.Total) +} diff --git a/core/task/scheduler/service.go b/core/task/scheduler/service.go index 53c552bd..85b7d90b 100644 --- a/core/task/scheduler/service.go +++ b/core/task/scheduler/service.go @@ -245,6 +245,7 @@ func newTaskSchedulerService() *Service { interval: 5 * time.Second, svr: server.GetGrpcServer(), handlerSvc: handler.GetTaskHandlerService(), + Logger: utils.NewLogger("TaskScheduler"), } } From 700c263cfe062d7fdf8e109c63a664aa9553f975 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 16 Mar 2025 20:30:52 +0800 Subject: [PATCH 06/13] refactor: update filter constants and improve type handling in filter functions - Renamed filter operation constants for clarity and consistency. - Introduced short forms for certain filter operations. - Updated test cases to reflect changes in filter operation handling. - Enhanced type handling in filter functions to support int64 values for conditions. - Improved query generation logic to accommodate new constant names and types. --- core/constants/filter.go | 9 ++++-- core/controllers/utils.go | 53 +++++++++++++++++++++++++--------- core/controllers/utils_test.go | 8 ++--- core/utils/filter.go | 4 +-- 4 files changed, 51 insertions(+), 23 deletions(-) diff --git a/core/constants/filter.go b/core/constants/filter.go index 9ed1d8db..ad22256a 100644 --- a/core/constants/filter.go +++ b/core/constants/filter.go @@ -8,8 +8,10 @@ const ( const ( FilterOpNotSet = "ns" - FilterOpContains = "c" - FilterOpNotContains = "nc" + FilterOpContains = "contains" + FilterOpContainsShort = "c" + FilterOpNotContainsShort = "nc" + FilterOpNotContains = "not-contains" FilterOpRegex = "r" FilterOpEqual = "eq" FilterOpNotEqual = "ne" @@ -19,5 +21,6 @@ const ( FilterOpLessThan = "lt" FilterOpGreaterThanEqual = "gte" FilterOpLessThanEqual = "lte" - FilterOpSearch = "s" + FilterOpSearchShort = "s" + FilterOpSearch = "search" ) diff --git a/core/controllers/utils.go b/core/controllers/utils.go index cca52bac..ece41ddf 100644 --- a/core/controllers/utils.go +++ b/core/controllers/utils.go @@ -2,7 +2,9 @@ package controllers import ( "encoding/json" - "errors" + "net/http" + "reflect" + "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/models/models" @@ -12,8 +14,6 @@ import ( "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" - "reflect" ) var logger = utils.NewLogger("Controllers") @@ -67,30 +67,55 @@ func GetFilterFromConditionString(condStr string) (f *entity.Filter, err error) } else { conditions[i].Value = item } + case reflect.Float64: + // JSON numbers are decoded as float64 by default + switch cond.Value.(type) { + case float64: + num := cond.Value.(float64) + // Check if it's a whole number + if num == float64(int64(num)) { + conditions[i].Value = int64(num) + } else { + conditions[i].Value = num + } + case int: + num := cond.Value.(int) + conditions[i].Value = int64(num) + case int64: + num := cond.Value.(int64) + conditions[i].Value = num + } + case reflect.Bool: + conditions[i].Value = cond.Value.(bool) case reflect.Slice, reflect.Array: var items []interface{} for i := 0; i < v.Len(); i++ { vItem := v.Index(i) item := vItem.Interface() - // string - stringItem, ok := item.(string) - if ok { - id, err := primitive.ObjectIDFromHex(stringItem) - if err == nil { + switch typedItem := item.(type) { + case string: + // Try to convert to ObjectID first + if id, err := primitive.ObjectIDFromHex(typedItem); err == nil { items = append(items, id) } else { - items = append(items, stringItem) + items = append(items, typedItem) } - continue + case float64: + if typedItem == float64(int64(typedItem)) { + items = append(items, int64(typedItem)) + } else { + items = append(items, typedItem) + } + case bool: + items = append(items, typedItem) + default: + items = append(items, item) } - - // default - items = append(items, item) } conditions[i].Value = items default: - return nil, errors.New("invalid type") + conditions[i].Value = cond.Value } } diff --git a/core/controllers/utils_test.go b/core/controllers/utils_test.go index 4ace445a..29e872a3 100644 --- a/core/controllers/utils_test.go +++ b/core/controllers/utils_test.go @@ -34,7 +34,7 @@ func TestGetFilterFromConditionString(t *testing.T) { assert.Equal(t, "test", filter.Conditions[0].Value) assert.Equal(t, "priority", filter.Conditions[1].Key) assert.Equal(t, "gt", filter.Conditions[1].Op) - assert.Equal(t, float64(5), filter.Conditions[1].Value) // JSON parses numbers as float64 + assert.Equal(t, int64(5), filter.Conditions[1].Value) // Invalid JSON should return error condStr = `[{"key":"name","op":"eq","value":"test"` @@ -56,7 +56,7 @@ func TestGetFilterQueryFromConditionString(t *testing.T) { query, err = controllers.GetFilterQueryFromConditionString(condStr) require.NoError(t, err) require.NotNil(t, query) - expected = bson.M{"priority": bson.M{"$gt": float64(5)}} + expected = bson.M{"priority": bson.M{"$gt": int64(5)}} assert.Equal(t, expected, query) // Multiple conditions @@ -64,11 +64,11 @@ func TestGetFilterQueryFromConditionString(t *testing.T) { query, err = controllers.GetFilterQueryFromConditionString(condStr) require.NoError(t, err) require.NotNil(t, query) - expected = bson.M{"name": "test", "priority": bson.M{"$gt": float64(5)}} + expected = bson.M{"name": "test", "priority": bson.M{"$gt": int64(5)}} assert.Equal(t, expected, query) // Contains operator - condStr = `[{"key":"name","op":"contains","value":"test"}]` + condStr = `[{"key":"name","op":"c","value":"test"}]` query, err = controllers.GetFilterQueryFromConditionString(condStr) require.NoError(t, err) require.NotNil(t, query) diff --git a/core/utils/filter.go b/core/utils/filter.go index 49e33160..16fe804a 100644 --- a/core/utils/filter.go +++ b/core/utils/filter.go @@ -25,9 +25,9 @@ func FilterToQuery(f interfaces.Filter) (q bson.M, err error) { q[key] = cond.GetValue() case constants.FilterOpNotEqual: q[key] = bson.M{"$ne": value} - case constants.FilterOpContains, constants.FilterOpRegex, constants.FilterOpSearch: + case constants.FilterOpContains, constants.FilterOpContainsShort, constants.FilterOpRegex, constants.FilterOpSearch, constants.FilterOpSearchShort: q[key] = bson.M{"$regex": value, "$options": "i"} - case constants.FilterOpNotContains: + case constants.FilterOpNotContains, constants.FilterOpNotContainsShort: q[key] = bson.M{"$not": bson.M{"$regex": value}} case constants.FilterOpIn: q[key] = bson.M{"$in": value} From 43d1c7692beb82fdb6ffc3bf3efe7fed8d7f8ac1 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 16 Mar 2025 22:25:13 +0800 Subject: [PATCH 07/13] refactor: standardize response types across controllers - Updated multiple controller methods to return VoidResponse instead of generic Response[any]. - Consolidated error handling to utilize GetErrorVoidResponse for consistent error responses. - Enhanced parameter handling in export and file management functions for improved clarity and maintainability. - Refactored health check and login/logout methods to align with new response structure. - Improved overall consistency in response formatting across various endpoints. --- core/controllers/base_file.go | 58 +++++++++++++++---------------- core/controllers/export.go | 14 +++++--- core/controllers/filter.go | 28 ++++++++++++++- core/controllers/health.go | 11 +++--- core/controllers/login.go | 4 +-- core/controllers/router.go | 16 ++++----- core/controllers/schedule.go | 14 ++++---- core/controllers/spider.go | 26 +++++++------- core/controllers/task.go | 28 +++++++-------- core/controllers/utils.go | 21 +++++++++++ core/entity/export.go | 26 +++++++------- core/export/csv_service.go | 18 +++------- core/export/json_service.go | 17 +++------ core/interfaces/export.go | 7 ++-- core/interfaces/export_service.go | 4 ++- core/openapi/wrapper.go | 18 +++++++--- 16 files changed, 175 insertions(+), 135 deletions(-) diff --git a/core/controllers/base_file.go b/core/controllers/base_file.go index 1384d1f0..c43e6ec4 100644 --- a/core/controllers/base_file.go +++ b/core/controllers/base_file.go @@ -62,46 +62,46 @@ type PostBaseFileSaveOneParams struct { Data string `json:"data"` } -func PostBaseFileSaveOne(rootPath, path, data string) (response *Response[any], err error) { +func PostBaseFileSaveOne(rootPath, path, data string) (response *VoidResponse, err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if err := fsSvc.Save(path, []byte(data)); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } - return GetDataResponse(any(data)) + return GetVoidResponse() } -func PostBaseFileSaveOneForm(rootPath, path string, file *multipart.FileHeader) (response *Response[any], err error) { +func PostBaseFileSaveOneForm(rootPath, path string, file *multipart.FileHeader) (response *VoidResponse, err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } f, err := file.Open() if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } fileData, err := io.ReadAll(f) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if err := fsSvc.Save(path, fileData); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } - return GetDataResponse[any](nil) + return GetVoidResponse() } -func PostBaseFileSaveMany(rootPath string, form *multipart.Form) (response *Response[any], err error) { +func PostBaseFileSaveMany(rootPath string, form *multipart.Form) (response *VoidResponse, err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } wg := sync.WaitGroup{} @@ -140,67 +140,67 @@ func PostBaseFileSaveMany(rootPath string, form *multipart.Form) (response *Resp } wg.Wait() - return GetDataResponse[any](nil) + return GetVoidResponse() } -func PostBaseFileSaveDir(rootPath, path string) (response *Response[any], err error) { +func PostBaseFileSaveDir(rootPath, path string) (response *VoidResponse, err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if err := fsSvc.CreateDir(path); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } - return GetDataResponse[any](nil) + return GetVoidResponse() } -func PostBaseFileRename(rootPath, path, newPath string) (response *Response[any], err error) { +func PostBaseFileRename(rootPath, path, newPath string) (response *VoidResponse, err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if err := fsSvc.Rename(path, newPath); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } - return GetDataResponse[any](nil) + return GetVoidResponse() } -func DeleteBaseFile(rootPath, path string) (response *Response[any], err error) { +func DeleteBaseFile(rootPath, path string) (response *VoidResponse, err error) { if path == "~" { path = "." } fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if err := fsSvc.Delete(path); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } _, err = fsSvc.GetFileInfo(".") if err != nil { _ = fsSvc.CreateDir("/") } - return GetDataResponse[any](nil) + return GetVoidResponse() } -func PostBaseFileCopy(rootPath, path, newPath string) (response *Response[any], err error) { +func PostBaseFileCopy(rootPath, path, newPath string) (response *VoidResponse, err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if err := fsSvc.Copy(path, newPath); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } - return GetDataResponse[any](nil) + return GetVoidResponse() } func PostBaseFileExport(rootPath string, c *gin.Context) (err error) { diff --git a/core/controllers/export.go b/core/controllers/export.go index 3282cacc..fe91e531 100644 --- a/core/controllers/export.go +++ b/core/controllers/export.go @@ -11,18 +11,22 @@ import ( ) type PostExportParams struct { - Type string `path:"type" validate:"required"` - Target string `query:"target" validate:"required"` - Filter interfaces.Filter `query:"filter"` + Type string `path:"type" validate:"required"` + Target string `query:"target" validate:"required"` + Conditions string `query:"conditions" description:"Filter conditions. Format: [{\"key\":\"name\",\"op\":\"eq\",\"value\":\"test\"}]"` } func PostExport(_ *gin.Context, params *PostExportParams) (response *Response[string], err error) { + query, err := GetFilterQueryFromConditionString(params.Conditions) + if err != nil { + return GetErrorResponse[string](err) + } var exportId string switch params.Type { case constants.ExportTypeCsv: - exportId, err = export.GetCsvService().Export(params.Type, params.Target, params.Filter) + exportId, err = export.GetCsvService().Export(params.Type, params.Target, query) case constants.ExportTypeJson: - exportId, err = export.GetJsonService().Export(params.Type, params.Target, params.Filter) + exportId, err = export.GetJsonService().Export(params.Type, params.Target, query) default: return GetErrorResponse[string](errors.BadRequestf("invalid export type: %s", params.Type)) } diff --git a/core/controllers/filter.go b/core/controllers/filter.go index d616f2f6..0afcbe5f 100644 --- a/core/controllers/filter.go +++ b/core/controllers/filter.go @@ -10,13 +10,39 @@ import ( ) type GetFilterColFieldOptionsParams struct { + Col string `path:"col" validate:"required"` + Conditions string `query:"conditions" description:"Filter conditions. Format: [{\"key\":\"name\",\"op\":\"eq\",\"value\":\"test\"}]"` +} + +func GetFilterColFieldOptions(c *gin.Context, params *GetFilterColFieldOptionsParams) (response *Response[[]entity.FilterSelectOption], err error) { + return GetFilterColFieldOptionsWithValueLabel(c, &GetFilterColFieldOptionsWithValueLabelParams{ + Col: params.Col, + Conditions: params.Conditions, + }) +} + +type GetFilterColFieldOptionsWithValueParams struct { + Col string `path:"col" validate:"required"` + Value string `path:"value"` + Conditions string `query:"conditions" description:"Filter conditions. Format: [{\"key\":\"name\",\"op\":\"eq\",\"value\":\"test\"}]"` +} + +func GetFilterColFieldOptionsWithValue(c *gin.Context, params *GetFilterColFieldOptionsWithValueParams) (response *Response[[]entity.FilterSelectOption], err error) { + return GetFilterColFieldOptionsWithValueLabel(c, &GetFilterColFieldOptionsWithValueLabelParams{ + Col: params.Col, + Value: params.Value, + Conditions: params.Conditions, + }) +} + +type GetFilterColFieldOptionsWithValueLabelParams struct { Col string `path:"col" validate:"required"` Value string `path:"value"` Label string `path:"label"` Conditions string `query:"conditions" description:"Filter conditions. Format: [{\"key\":\"name\",\"op\":\"eq\",\"value\":\"test\"}]"` } -func GetFilterColFieldOptions(_ *gin.Context, params *GetFilterColFieldOptionsParams) (response *Response[[]entity.FilterSelectOption], err error) { +func GetFilterColFieldOptionsWithValueLabel(_ *gin.Context, params *GetFilterColFieldOptionsWithValueLabelParams) (response *Response[[]entity.FilterSelectOption], err error) { value := params.Value if value == "" { value = "_id" diff --git a/core/controllers/health.go b/core/controllers/health.go index bf687a5e..37e71db4 100644 --- a/core/controllers/health.go +++ b/core/controllers/health.go @@ -1,20 +1,17 @@ package controllers import ( - "errors" "github.com/gin-gonic/gin" "net/http" ) -func GetHealthFn(healthFn func() bool) func(c *gin.Context) error { - return func(c *gin.Context) (err error) { +func GetHealthFn(healthFn func() bool) func(c *gin.Context) { + return func(c *gin.Context) { if healthFn() { - c.Writer.Write([]byte("ok")) + _, _ = c.Writer.Write([]byte("ok")) c.AbortWithStatus(http.StatusOK) - return } - c.Writer.Write([]byte("not ready")) + _, _ = c.Writer.Write([]byte("not ready")) c.AbortWithStatus(http.StatusServiceUnavailable) - return errors.New("not ready") } } diff --git a/core/controllers/login.go b/core/controllers/login.go index 1a174b12..b95806f4 100644 --- a/core/controllers/login.go +++ b/core/controllers/login.go @@ -27,6 +27,6 @@ func PostLogin(c *gin.Context, params *PostLoginParams) (response *Response[stri return GetDataResponse(token) } -func PostLogout(_ *gin.Context) (response *Response[any], err error) { - return GetDataResponse[any](nil) +func PostLogout(_ *gin.Context) (response *VoidResponse, err error) { + return GetVoidResponse() } diff --git a/core/controllers/router.go b/core/controllers/router.go index 95aa7805..fc6c5450 100644 --- a/core/controllers/router.go +++ b/core/controllers/router.go @@ -29,7 +29,7 @@ func GetGlobalFizzWrapper() *openapi.FizzWrapper { // NewRouterGroups initializes the router groups with their respective middleware func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) { // Create OpenAPI wrapper - globalWrapper = openapi.NewFizzWrapper(app) + globalWrapper = openapi.GetFizzWrapper(app) return &RouterGroups{ AuthGroup: app.Group("/", middlewares.AuthorizationMiddleware()), @@ -485,12 +485,12 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodGet, Path: "/:col/:value", - HandlerFunc: GetFilterColFieldOptions, + HandlerFunc: GetFilterColFieldOptionsWithValue, }, { Method: http.MethodGet, Path: "/:col/:value/:label", - HandlerFunc: GetFilterColFieldOptions, + HandlerFunc: GetFilterColFieldOptionsWithValueLabel, }, }) RegisterActions(groups.AuthGroup, "/settings", []Action{ @@ -543,13 +543,6 @@ func InitRoutes(app *gin.Engine) (err error) { }) // Register public routes that don't require authentication - RegisterActions(groups.AnonymousGroup, "/health", []Action{ - { - Path: "", - Method: http.MethodGet, - HandlerFunc: GetHealthFn(func() bool { return true }), - }, - }) RegisterActions(groups.AnonymousGroup, "/system-info", []Action{ { Path: "", @@ -570,6 +563,9 @@ func InitRoutes(app *gin.Engine) (err error) { }, }) + // Register health check route + groups.AnonymousGroup.GET("/health", GetHealthFn(func() bool { return true })) + // Register OpenAPI documentation route groups.AnonymousGroup.GET("/openapi.json", GetOpenAPI) diff --git a/core/controllers/schedule.go b/core/controllers/schedule.go index 4a5bf363..c98d6f95 100644 --- a/core/controllers/schedule.go +++ b/core/controllers/schedule.go @@ -83,26 +83,26 @@ type PostScheduleEnableDisableParams struct { Id string `path:"id"` } -func PostScheduleEnable(c *gin.Context, params *PostScheduleEnableDisableParams) (response *Response[any], err error) { +func PostScheduleEnable(c *gin.Context, params *PostScheduleEnableDisableParams) (response *VoidResponse, err error) { userId := GetUserFromContext(c).Id return postScheduleEnableDisableFunc(true, userId, params) } -func PostScheduleDisable(c *gin.Context, params *PostScheduleEnableDisableParams) (response *Response[any], err error) { +func PostScheduleDisable(c *gin.Context, params *PostScheduleEnableDisableParams) (response *VoidResponse, err error) { userId := GetUserFromContext(c).Id return postScheduleEnableDisableFunc(false, userId, params) } -func postScheduleEnableDisableFunc(isEnable bool, userId primitive.ObjectID, params *PostScheduleEnableDisableParams) (response *Response[any], err error) { +func postScheduleEnableDisableFunc(isEnable bool, userId primitive.ObjectID, params *PostScheduleEnableDisableParams) (response *VoidResponse, err error) { id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - return GetErrorResponse[any](errors.BadRequestf("invalid schedule id: %v", err)) + return GetErrorVoidResponse(errors.BadRequestf("invalid schedule id: %v", err)) } svc := schedule.GetScheduleService() s, err := service.NewModelService[models.Schedule]().GetById(id) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if isEnable { @@ -111,10 +111,10 @@ func postScheduleEnableDisableFunc(isEnable bool, userId primitive.ObjectID, par err = svc.Disable(*s, userId) } if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } - return GetDataResponse[any](nil) + return GetVoidResponse() } type PostScheduleRunParams struct { diff --git a/core/controllers/spider.go b/core/controllers/spider.go index 8bbbb5c3..722efeb3 100644 --- a/core/controllers/spider.go +++ b/core/controllers/spider.go @@ -575,10 +575,10 @@ type PostSpiderSaveFileParams struct { File *multipart.FileHeader `form:"file"` } -func PostSpiderSaveFile(c *gin.Context, params *PostSpiderSaveFileParams) (response *Response[any], err error) { +func PostSpiderSaveFile(c *gin.Context, params *PostSpiderSaveFileParams) (response *VoidResponse, err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if c.GetHeader("Content-Type") == "application/json" { return PostBaseFileSaveOne(rootPath, params.Path, params.Data) @@ -592,14 +592,14 @@ type PostSpiderSaveFilesParams struct { TargetDirectory string `form:"targetDirectory"` } -func PostSpiderSaveFiles(c *gin.Context, params *PostSpiderSaveFilesParams) (response *Response[any], err error) { +func PostSpiderSaveFiles(c *gin.Context, params *PostSpiderSaveFilesParams) (response *VoidResponse, err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } form, err := c.MultipartForm() if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } return PostBaseFileSaveMany(filepath.Join(rootPath, params.TargetDirectory), form) } @@ -609,10 +609,10 @@ type PostSpiderSaveDirParams struct { Path string `json:"path"` } -func PostSpiderSaveDir(c *gin.Context, params *PostSpiderSaveDirParams) (response *Response[any], err error) { +func PostSpiderSaveDir(c *gin.Context, params *PostSpiderSaveDirParams) (response *VoidResponse, err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } return PostBaseFileSaveDir(rootPath, params.Path) } @@ -623,10 +623,10 @@ type PostSpiderRenameFileParams struct { NewPath string `json:"newPath"` } -func PostSpiderRenameFile(c *gin.Context, params *PostSpiderRenameFileParams) (response *Response[any], err error) { +func PostSpiderRenameFile(c *gin.Context, params *PostSpiderRenameFileParams) (response *VoidResponse, err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } return PostBaseFileRename(rootPath, params.Path, params.NewPath) } @@ -636,10 +636,10 @@ type DeleteSpiderFileParams struct { Path string `json:"path"` } -func DeleteSpiderFile(c *gin.Context, params *DeleteSpiderFileParams) (response *Response[any], err error) { +func DeleteSpiderFile(c *gin.Context, params *DeleteSpiderFileParams) (response *VoidResponse, err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } return DeleteBaseFile(rootPath, params.Path) } @@ -650,10 +650,10 @@ type PostSpiderCopyFileParams struct { NewPath string `json:"new_path"` } -func PostSpiderCopyFile(c *gin.Context, params *PostSpiderCopyFileParams) (response *Response[any], err error) { +func PostSpiderCopyFile(c *gin.Context, params *PostSpiderCopyFileParams) (response *VoidResponse, err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } return PostBaseFileCopy(rootPath, params.Path, params.NewPath) } diff --git a/core/controllers/task.go b/core/controllers/task.go index 23299c7f..25fb80e8 100644 --- a/core/controllers/task.go +++ b/core/controllers/task.go @@ -170,10 +170,10 @@ type DeleteTaskByIdParams struct { Id string `path:"id"` } -func DeleteTaskById(_ *gin.Context, params *DeleteTaskByIdParams) (response *Response[any], err error) { +func DeleteTaskById(_ *gin.Context, params *DeleteTaskByIdParams) (response *VoidResponse, err error) { id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } // delete in db @@ -202,7 +202,7 @@ func DeleteTaskById(_ *gin.Context, params *DeleteTaskByIdParams) (response *Res return nil }); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } // delete task logs @@ -211,19 +211,19 @@ func DeleteTaskById(_ *gin.Context, params *DeleteTaskByIdParams) (response *Res logger.Warnf("failed to remove task log directory: %s", logPath) } - return GetDataResponse[any](nil) + return GetVoidResponse() } type DeleteTaskListParams struct { Ids []string `json:"ids"` } -func DeleteList(_ *gin.Context, params *DeleteTaskListParams) (response *Response[any], err error) { +func DeleteList(_ *gin.Context, params *DeleteTaskListParams) (response *VoidResponse, err error) { var ids []primitive.ObjectID for _, id := range params.Ids { id, err := primitive.ObjectIDFromHex(id) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } ids = append(ids, id) } @@ -250,7 +250,7 @@ func DeleteList(_ *gin.Context, params *DeleteTaskListParams) (response *Respons return nil }); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } // delete tasks logs @@ -268,7 +268,7 @@ func DeleteList(_ *gin.Context, params *DeleteTaskListParams) (response *Respons } wg.Wait() - return GetDataResponse[any](nil) + return GetVoidResponse() } type PostTaskRunParams struct { @@ -379,22 +379,22 @@ type PostTaskCancelParams struct { Force bool `json:"force,omitempty"` } -func PostTaskCancel(c *gin.Context, params *PostTaskCancelParams) (response *Response[any], err error) { +func PostTaskCancel(c *gin.Context, params *PostTaskCancelParams) (response *VoidResponse, err error) { // id id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } // task t, err := service.NewModelService[models.Task]().GetById(id) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } // validate if !utils.IsCancellable(t.Status) { - return GetErrorResponse[any](errors.New("task is not cancellable")) + return GetErrorVoidResponse(errors.New("task is not cancellable")) } u := GetUserFromContext(c) @@ -403,10 +403,10 @@ func PostTaskCancel(c *gin.Context, params *PostTaskCancelParams) (response *Res schedulerSvc := scheduler.GetTaskSchedulerService() err = schedulerSvc.Cancel(id, u.Id, params.Force) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } - return GetDataResponse[any](nil) + return GetVoidResponse() } type GetTaskLogsParams struct { diff --git a/core/controllers/utils.go b/core/controllers/utils.go index ece41ddf..b61bf8b1 100644 --- a/core/controllers/utils.go +++ b/core/controllers/utils.go @@ -263,6 +263,12 @@ type ListResponse[T any] struct { Error string `json:"error"` } +type VoidResponse struct { + Status string `json:"status"` + Message string `json:"message"` + Error string `json:"error"` +} + func GetDataResponse[T any](model T) (res *Response[T], err error) { return &Response[T]{ Status: constants.HttpResponseStatusOk, @@ -280,6 +286,13 @@ func GetListResponse[T any](models []T, total int) (res *ListResponse[T], err er }, nil } +func GetVoidResponse() (res *VoidResponse, err error) { + return &VoidResponse{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageSuccess, + }, nil +} + func GetErrorResponse[T any](err error) (res *Response[T], err2 error) { return &Response[T]{ Status: constants.HttpResponseStatusOk, @@ -288,6 +301,14 @@ func GetErrorResponse[T any](err error) (res *Response[T], err2 error) { }, err } +func GetErrorVoidResponse(err error) (res *VoidResponse, err2 error) { + return &VoidResponse{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageError, + Error: err.Error(), + }, err +} + func GetErrorListResponse[T any](err error) (res *ListResponse[T], err2 error) { return &ListResponse[T]{ Status: constants.HttpResponseStatusOk, diff --git a/core/entity/export.go b/core/entity/export.go index 5bd06a7b..62605659 100644 --- a/core/entity/export.go +++ b/core/entity/export.go @@ -1,21 +1,21 @@ package entity import ( - "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" "time" ) type Export struct { - Id string `json:"id"` - Type string `json:"type"` - Target string `json:"target"` - Filter interfaces.Filter `json:"filter"` - Status string `json:"status"` - StartTs time.Time `json:"start_ts"` - EndTs time.Time `json:"end_ts"` - FileName string `json:"file_name"` - DownloadPath string `json:"-"` - Limit int `json:"-"` + Id string `json:"id"` + Type string `json:"type"` + Target string `json:"target"` + Query bson.M `json:"query"` + Status string `json:"status"` + StartTs time.Time `json:"start_ts"` + EndTs time.Time `json:"end_ts"` + FileName string `json:"file_name"` + DownloadPath string `json:"-"` + Limit int `json:"-"` } func (e *Export) GetId() string { @@ -30,8 +30,8 @@ func (e *Export) GetTarget() string { return e.Target } -func (e *Export) GetFilter() interfaces.Filter { - return e.Filter +func (e *Export) GetQuery() bson.M { + return e.Query } func (e *Export) GetStatus() string { diff --git a/core/export/csv_service.go b/core/export/csv_service.go index aac21876..83bf962d 100644 --- a/core/export/csv_service.go +++ b/core/export/csv_service.go @@ -36,7 +36,7 @@ func (svc *CsvService) GenerateId() (exportId string, err error) { return exportId, nil } -func (svc *CsvService) Export(exportType, target string, filter interfaces.Filter) (exportId string, err error) { +func (svc *CsvService) Export(exportType, target string, query bson.M) (exportId string, err error) { // generate export id exportId, err = svc.GenerateId() if err != nil { @@ -48,7 +48,7 @@ func (svc *CsvService) Export(exportType, target string, filter interfaces.Filte Id: exportId, Type: exportType, Target: target, - Filter: filter, + Query: query, Status: constants.TaskStatusRunning, StartTs: time.Now(), FileName: svc.getFileName(exportId), @@ -90,18 +90,8 @@ func (svc *CsvService) export(export *entity.Export) { // mongo collection col := mongo.GetMongoCol(export.Target) - // mongo query - query, err := utils.FilterToQuery(export.Filter) - if err != nil { - export.Status = constants.TaskStatusError - export.EndTs = time.Now() - svc.Errorf("export error (id: %s): %v", export.Id, err) - svc.cache.Set(export.Id, export) - return - } - // mongo cursor - cur := col.Find(query, nil).GetCursor() + cur := col.Find(export.Query, nil).GetCursor() // csv writer csvWriter, csvFile, err := svc.getCsvWriter(export) @@ -126,7 +116,7 @@ func (svc *CsvService) export(export *entity.Export) { } // write csv header row - columns, err := svc.getColumns(query, export) + columns, err := svc.getColumns(export.Query, export) err = csvWriter.Write(columns) if err != nil { export.Status = constants.TaskStatusError diff --git a/core/export/json_service.go b/core/export/json_service.go index 0929d9e2..66813b47 100644 --- a/core/export/json_service.go +++ b/core/export/json_service.go @@ -11,6 +11,7 @@ import ( "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/utils" "github.com/hashicorp/go-uuid" + "go.mongodb.org/mongo-driver/bson" mongo2 "go.mongodb.org/mongo-driver/mongo" "os" "path" @@ -31,7 +32,7 @@ func (svc *JsonService) GenerateId() (exportId string, err error) { return exportId, nil } -func (svc *JsonService) Export(exportType, target string, filter interfaces.Filter) (exportId string, err error) { +func (svc *JsonService) Export(exportType, target string, query bson.M) (exportId string, err error) { // generate export id exportId, err = svc.GenerateId() if err != nil { @@ -43,7 +44,7 @@ func (svc *JsonService) Export(exportType, target string, filter interfaces.Filt Id: exportId, Type: exportType, Target: target, - Filter: filter, + Query: query, Status: constants.TaskStatusRunning, StartTs: time.Now(), FileName: svc.getFileName(exportId), @@ -85,18 +86,8 @@ func (svc *JsonService) export(export *entity.Export) { // mongo collection col := mongo.GetMongoCol(export.Target) - // mongo query - query, err := utils.FilterToQuery(export.Filter) - if err != nil { - export.Status = constants.TaskStatusError - export.EndTs = time.Now() - svc.Errorf("export error (id: %s): %v", export.Id, err) - svc.cache.Set(export.Id, export) - return - } - // mongo cursor - cur := col.Find(query, nil).GetCursor() + cur := col.Find(export.Query, nil).GetCursor() // data var jsonData []interface{} diff --git a/core/interfaces/export.go b/core/interfaces/export.go index 2205b842..b9951711 100644 --- a/core/interfaces/export.go +++ b/core/interfaces/export.go @@ -1,12 +1,15 @@ package interfaces -import "time" +import ( + "go.mongodb.org/mongo-driver/bson" + "time" +) type Export interface { GetId() string GetType() string GetTarget() string - GetFilter() Filter + GetQuery() bson.M GetStatus() string GetStartTs() time.Time GetEndTs() time.Time diff --git a/core/interfaces/export_service.go b/core/interfaces/export_service.go index d48ae2b8..a21dfa6e 100644 --- a/core/interfaces/export_service.go +++ b/core/interfaces/export_service.go @@ -1,7 +1,9 @@ package interfaces +import "go.mongodb.org/mongo-driver/bson" + type ExportService interface { GenerateId() (exportId string, err error) - Export(exportType, target string, filter Filter) (exportId string, err error) + Export(exportType, target string, query bson.M) (exportId string, err error) GetExport(exportId string) (export Export, err error) } diff --git a/core/openapi/wrapper.go b/core/openapi/wrapper.go index 8f8214a1..86ea281e 100644 --- a/core/openapi/wrapper.go +++ b/core/openapi/wrapper.go @@ -4,10 +4,10 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/utils" - "github.com/loopfz/gadgeto/tonic" - "github.com/gin-gonic/gin" + "github.com/loopfz/gadgeto/tonic" "github.com/wI2L/fizz" + "sync" ) // FizzWrapper wraps an existing Gin Engine to add OpenAPI functionality @@ -17,9 +17,9 @@ type FizzWrapper struct { logger interfaces.Logger } -// NewFizzWrapper creates a new wrapper around an existing Gin Engine +// newFizzWrapper creates a new wrapper around an existing Gin Engine // This approach ensures we don't break existing functionality -func NewFizzWrapper(engine *gin.Engine) *FizzWrapper { +func newFizzWrapper(engine *gin.Engine) *FizzWrapper { // Create a new Fizz instance using the existing Gin engine f := fizz.NewFromEngine(engine) return &FizzWrapper{ @@ -116,3 +116,13 @@ func (w *FizzWrapper) buildOperationOptions(id, summary, description string, res return opts } + +var wrapper *FizzWrapper +var wrapperOnce sync.Once + +func GetFizzWrapper(app *gin.Engine) *FizzWrapper { + wrapperOnce.Do(func() { + wrapper = newFizzWrapper(app) + }) + return wrapper +} From bc5e6273b8fc78947934060be5d870d4169055fc Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 17 Mar 2025 12:08:35 +0800 Subject: [PATCH 08/13] chore: update dependencies and refactor imports - Updated various dependencies in go.mod and go.sum for improved stability and performance. - Replaced references to the old fizz package with the new crawlab-team/fizz package in multiple files. - Adjusted indirect dependencies to their latest versions for better compatibility. - Enhanced overall code clarity by standardizing import paths across the project. --- core/controllers/base_test.go | 2 +- core/controllers/openapi.go | 2 +- core/go.mod | 37 ++++++++-------- core/go.sum | 39 +++++++++++++++++ core/openapi/wrapper.go | 2 +- go.work.sum | 82 ++++++++++++++++++++++++++++++++++- 6 files changed, 141 insertions(+), 23 deletions(-) diff --git a/core/controllers/base_test.go b/core/controllers/base_test.go index e3715938..0217b636 100644 --- a/core/controllers/base_test.go +++ b/core/controllers/base_test.go @@ -20,10 +20,10 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/user" + "github.com/crawlab-team/fizz" "github.com/loopfz/gadgeto/tonic" "github.com/spf13/viper" "github.com/stretchr/testify/require" - "github.com/wI2L/fizz" "github.com/stretchr/testify/assert" "go.mongodb.org/mongo-driver/bson" diff --git a/core/controllers/openapi.go b/core/controllers/openapi.go index d6996b7c..2685b039 100644 --- a/core/controllers/openapi.go +++ b/core/controllers/openapi.go @@ -1,8 +1,8 @@ package controllers import ( + "github.com/crawlab-team/fizz/openapi" "github.com/gin-gonic/gin" - "github.com/wI2L/fizz/openapi" ) func GetOpenAPI(c *gin.Context) { diff --git a/core/go.mod b/core/go.mod index 4f0f91f6..b9e39502 100644 --- a/core/go.mod +++ b/core/go.mod @@ -6,7 +6,6 @@ replace ( github.com/crawlab-team/crawlab/grpc => ../grpc github.com/crawlab-team/crawlab/trace => ../trace github.com/crawlab-team/crawlab/vcs => ../vcs - github.com/wI2L/fizz => github.com/crawlab-team/fizz v0.0.0-20250312082934-630a98f9d791 ) require ( @@ -35,12 +34,11 @@ require ( github.com/spf13/cobra v1.3.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 - github.com/wI2L/fizz v0.22.0 - go.mongodb.org/mongo-driver v1.15.1 - golang.org/x/oauth2 v0.23.0 + go.mongodb.org/mongo-driver v1.17.3 + golang.org/x/oauth2 v0.28.0 golang.org/x/text v0.23.0 - google.golang.org/api v0.189.0 - google.golang.org/grpc v1.69.2 + google.golang.org/api v0.226.0 + google.golang.org/grpc v1.71.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df ) @@ -50,9 +48,9 @@ require ( ) require ( - cloud.google.com/go/auth v0.7.2 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect - cloud.google.com/go/compute/metadata v0.5.2 // indirect + cloud.google.com/go/auth v0.15.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect + cloud.google.com/go/compute/metadata v0.6.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect @@ -61,6 +59,7 @@ require ( github.com/cloudflare/circl v1.3.7 // indirect github.com/cloudwego/base64x v0.1.5 // indirect github.com/cloudwego/iasm v0.2.0 // indirect + github.com/crawlab-team/fizz v0.0.0-20250317032929-767b58e01fa9 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -79,9 +78,9 @@ require ( github.com/gofrs/uuid v3.2.0+incompatible // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/s2a-go v0.1.8 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.13.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.5 // indirect + github.com/googleapis/gax-go/v2 v2.14.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect @@ -96,7 +95,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/montanaflynn/stats v0.7.1 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -117,14 +116,14 @@ require ( github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect github.com/ztrue/tracerr v0.4.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect - go.opentelemetry.io/otel v1.31.0 // indirect - go.opentelemetry.io/otel/metric v1.31.0 // indirect - go.opentelemetry.io/otel/trace v1.31.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect + go.opentelemetry.io/otel v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.34.0 // indirect go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.15.0 // indirect @@ -133,7 +132,7 @@ require ( golang.org/x/net v0.37.0 // indirect golang.org/x/sync v0.12.0 // indirect golang.org/x/sys v0.31.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect google.golang.org/protobuf v1.36.5 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/go-playground/validator.v9 v9.30.0 // indirect diff --git a/core/go.sum b/core/go.sum index 33aa2bc2..e103cef7 100644 --- a/core/go.sum +++ b/core/go.sum @@ -29,8 +29,12 @@ cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0c cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE= cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= +cloud.google.com/go/auth v0.15.0 h1:Ly0u4aA5vG/fsSsxu98qCQBemXtAtJf+95z9HK+cxps= +cloud.google.com/go/auth v0.15.0/go.mod h1:WJDGqZ1o9E9wKIL+IwStfyn/+s59zl4Bi+1KQNVXLZ8= cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= +cloud.google.com/go/auth/oauth2adapt v0.2.7 h1:/Lc7xODdqcEw8IrZ9SvwnlLX6j9FHQM74z6cBk9Rw6M= +cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -39,6 +43,8 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= @@ -144,6 +150,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crawlab-team/fizz v0.0.0-20250312082934-630a98f9d791 h1:K849MMP329dseKSpMqX3XDTccpWxghMwWyrDclKzkdM= github.com/crawlab-team/fizz v0.0.0-20250312082934-630a98f9d791/go.mod h1:CMxMR1amz8id9wr2YUpONf+F/F9hW1cqRXxVNNuWVxE= +github.com/crawlab-team/fizz v0.0.0-20250317032929-767b58e01fa9 h1:Rnc78P4Wib2XBPd0VeGx0XtuF9sufmJepFLjXiYLyrU= +github.com/crawlab-team/fizz v0.0.0-20250317032929-767b58e01fa9/go.mod h1:PotXqgZl5b2duH/101hv5/C6811jptoh1HxWOmObM+4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= @@ -304,6 +312,7 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -326,6 +335,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -333,12 +344,16 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/enterprise-certificate-proxy v0.3.5 h1:VgzTY2jogw3xt39CusEnFJWm7rlsq5yL5q9XdLOuP5g= +github.com/googleapis/enterprise-certificate-proxy v0.3.5/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= +github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -479,6 +494,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -618,6 +634,7 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -633,6 +650,8 @@ go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3 go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.mongodb.org/mongo-driver v1.15.1 h1:l+RvoUOoMXFmADTLfYDm7On9dRm7p4T80/lEQM+r7HU= go.mongodb.org/mongo-driver v1.15.1/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ= +go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -644,16 +663,26 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= @@ -800,6 +829,8 @@ golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= +golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1024,6 +1055,8 @@ google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3h google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI= google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8= +google.golang.org/api v0.226.0 h1:9A29y1XUD+YRXfnHkO66KggxHBZWg9LsTGqm7TkUvtQ= +google.golang.org/api v0.226.0/go.mod h1:WP/0Xm4LVvMOCldfvOISnWquSRWbG2kArDZcg+W2DbY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1094,11 +1127,15 @@ google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCeFn7MoLy+wBDQZ1UQIJD4UNM1Qvkg= google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= +google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 h1:GVIKPyP/kLIyVOgOnTwFOrvQaQUzOzGMCxgFUOEmm24= google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 h1:LWZqQOEjDyONlF1H6afSWpAL/znlREo2tHfLoe+8LMA= google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1128,6 +1165,8 @@ google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= +google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/core/openapi/wrapper.go b/core/openapi/wrapper.go index 86ea281e..4d4e8758 100644 --- a/core/openapi/wrapper.go +++ b/core/openapi/wrapper.go @@ -4,9 +4,9 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/fizz" "github.com/gin-gonic/gin" "github.com/loopfz/gadgeto/tonic" - "github.com/wI2L/fizz" "sync" ) diff --git a/go.work.sum b/go.work.sum index 9c0840c0..15ebccf1 100644 --- a/go.work.sum +++ b/go.work.sum @@ -4,6 +4,8 @@ cel.dev/expr v0.16.1 h1:NR0+oFYzR1CqLFhTAqg3ql59G9VfN8fKq1TCHJ6gq1g= cel.dev/expr v0.16.1/go.mod h1:AsGA5zb3WruAEQeQng1RZdGEXmBj0jvMWh6l5SnNuC8= cel.dev/expr v0.16.2 h1:RwRhoH17VhAu9U5CMvMhH1PDVgf0tuz9FT+24AfMLfU= cel.dev/expr v0.16.2/go.mod h1:gXngZQMkWJoSbE8mOzehJlXQyubn/Vg0vR9/F3W7iw8= +cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= +cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.99.0 h1:y/cM2iqGgGi5D5DQZl6D9STN/3dR/Vx5Mp8s752oJTY= @@ -95,6 +97,7 @@ cloud.google.com/go/auth v0.4.2 h1:sb0eyLkhRtpq5jA+a8KWw0W70YcdVca7KJ8TM0AFYDg= cloud.google.com/go/auth v0.4.2/go.mod h1:Kqvlz1cf1sNA0D+sYJnkPQOP+JMHkuHeIgVmCRtZOLc= cloud.google.com/go/auth v0.5.1/go.mod h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbwDsa6s= cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= +cloud.google.com/go/auth v0.12.1/go.mod h1:BFMu+TNpF3DmvfBO9ClqTR/SiqVIm7LukKF9mbendF4= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/automl v1.13.5 h1:ijiJy9sYWh75WrqImXsfWc1e3HR3iO+ef9fvW03Ig/4= @@ -755,6 +758,8 @@ github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 h1:oVLqHXhnYtUwM89y9T1 github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0/go.mod h1:dppbR7CwXD4pgtV9t3wD1812RaLDcBjtblcDF5f1vI0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.2 h1:cZpsGsWTIFKymTA0je7IIvi1O7Es7apb9CF3EQlOcfE= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.2/go.mod h1:itPGVDKf9cC/ov4MdvJ2QZ0khw4bfoo9jzwTJlaxy2k= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= @@ -806,6 +811,8 @@ github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnTh github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3 h1:boJj011Hh+874zpIySeApCX4GeOjPl9qhRF3QuIZq+Q= +github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= @@ -833,17 +840,27 @@ github.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmd github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8= github.com/envoyproxy/go-control-plane v0.13.1 h1:vPfJZCkob6yTMEgS+0TwfTUfbHjfy/6vOJ8hUWX/uXE= github.com/envoyproxy/go-control-plane v0.13.1/go.mod h1:X45hY0mufo6Fd0KW3rqsGvQMw58jvjymeCzBU3mWyHw= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/fasthttp/websocket v1.4.3-rc.6 h1:omHqsl8j+KXpmzRjF8bmzOSYJ8GnS0E3efi1wYT+niY= github.com/fasthttp/websocket v1.4.3-rc.6/go.mod h1:43W9OM2T8FeXpCWMsBd9Cb7nE2CACNqNvCqQCoty/Lc= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/gin-contrib/cors v1.3.0 h1:PolezCc89peu+NgkIWt9OB01Kbzt6IP0J/JvkG6xxlg= +github.com/gin-contrib/cors v1.7.3 h1:hV+a5xp8hwJoTw7OY+a70FsL8JkVVFTXw9EcfrYUdns= +github.com/gin-contrib/cors v1.7.3/go.mod h1:M3bcKZhxzsvI+rlRSkkxHyljJt1ESd93COUvemZ79j4= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df h1:Bao6dhmbTA1KFVxmJ6nBoMuOJit2yjEgLJpIMYpop0E= @@ -868,6 +885,8 @@ github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= +github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -879,8 +898,11 @@ github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8i github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9 h1:OF1IPgv+F4NmqmJ98KTjdN97Vs1JxDPB3vbmYzV2dpk= github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= +github.com/google/go-pkcs11 v0.3.0 h1:PVRnTgtArZ3QQqTGtbtjtnIkzl2iY2kt24yqbrf7td8= +github.com/google/go-pkcs11 v0.3.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= @@ -892,10 +914,12 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU= github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= @@ -906,6 +930,7 @@ github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/ github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 h1:tlyzajkF3030q6M8SvmJSemC9DTHL/xaMa18b65+JM4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720 h1:zC34cGQu69FG7qzJ3WiKW244WfhDC3xxYMeNOX2gtUQ= @@ -1010,6 +1035,7 @@ github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4 github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= github.com/mmcloughlin/avo v0.5.0 h1:nAco9/aI9Lg2kiuROBY6BhCI/z0t5jEvJfjWbL8qXLU= github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/nats-io/nats.go v1.34.0 h1:fnxnPCNiwIG5w08rlMcEKTUw4AV/nKyGCOJE8TdhSPk= github.com/nats-io/nats.go v1.34.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= @@ -1027,6 +1053,8 @@ github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xl github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= @@ -1045,6 +1073,8 @@ github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cY github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= @@ -1057,6 +1087,8 @@ github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4 h1:BN/Nyn2nWMoqGRA7G7paDNDqTXE30mXGqzzybrfo05w= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -1096,6 +1128,7 @@ github.com/ugorji/go v1.2.6 h1:tGiWC9HENWE2tqYycIqFTNorMmFRVhNwCpDOpWqnk8E= github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= @@ -1117,16 +1150,23 @@ go.etcd.io/etcd/client/v2 v2.305.12/go.mod h1:aQ/yhsxMu+Oht1FOupSr60oBvcS9cKXHrz go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/detectors/gcp v1.31.0 h1:G1JQOreVrfhRkner+l4mrGxmfqYCAuy76asTDAo0xsA= go.opentelemetry.io/contrib/detectors/gcp v1.31.0/go.mod h1:tzQL6E1l+iV44YFTkcAeNQqzXUiekSYP9jjJjXwEd00= +go.opentelemetry.io/contrib/detectors/gcp v1.34.0 h1:JRxssobiPg23otYU5SbWtQC//snGVIM3Tx6QRzlQBao= +go.opentelemetry.io/contrib/detectors/gcp v1.34.0/go.mod h1:cV4BMFcscUR/ckqLkbfQmF0PRsq8w/lMGzdbCSveBHo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.48.0/go.mod h1:tIKj3DbO8N9Y2xo52og3irLsPI4GW02DSMtrVgNMgxg= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0/go.mod h1:rdENBZMT2OE6Ne/KLwpiXudnAsbdrdBaqBvTN8M8BgA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= go.opentelemetry.io/otel v1.23.0/go.mod h1:YCycw9ZeKhcJFrb34iVSkyT0iczq/zYDtZYFufObyB0= @@ -1140,6 +1180,8 @@ go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6 go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.opentelemetry.io/otel/trace v1.23.0/go.mod h1:GSGTbIClEsuZrGIzoEHqsVfxgn5UkggkflQwDScNUsk= @@ -1169,9 +1211,13 @@ golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1m golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= @@ -1197,12 +1243,14 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= @@ -1210,11 +1258,15 @@ golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5H golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1232,6 +1284,7 @@ golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -1249,13 +1302,19 @@ golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= @@ -1287,7 +1346,9 @@ google.golang.org/api v0.177.0/go.mod h1:srbhue4MLjkjbkux5p3dw/ocYOSZTaIEvf7bCOn google.golang.org/api v0.180.0/go.mod h1:51AiyoEg1MJPSZ9zvklA8VnRILPXxn1iVen9v25XHAE= google.golang.org/api v0.182.0 h1:if5fPvudRQ78GeRx3RayIoiuV7modtErPIZC/T2bIvE= google.golang.org/api v0.182.0/go.mod h1:cGhjy4caqA5yXRzEhkHI8Y9mfyC2VLTlER2l08xaqtM= +google.golang.org/api v0.183.0/go.mod h1:q43adC5/pHoSZTx5h2mSmdF7NcyfW9JuDyIOJAgS9ZQ= google.golang.org/api v0.187.0/go.mod h1:KIHlTc4x7N7gKKuVsdmfBXN13yEEWXWFURWY6SBp2gk= +google.golang.org/api v0.211.0/go.mod h1:XOloB4MXFH4UTlQSGuNUxw0UT74qdENK8d6JNsXKLi0= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1301,9 +1362,9 @@ google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= google.golang.org/genproto v0.0.0-20240205150955-31a09d347014/go.mod h1:xEgQu1e4stdSSsxPDK8Azkrk/ECl5HvdPf6nbZrTS5M= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw= +google.golang.org/genproto v0.0.0-20240528184218-531527333157/go.mod h1:ubQlAQnzejB8uZzszhrTCU2Fyp6Vi7ZE5nn0c3W8+qQ= google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:s7iA721uChleev562UJO2OYB0PPT9CMFjV+Ce7VJH5M= google.golang.org/genproto v0.0.0-20240722135656-d784300faade/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= @@ -1325,6 +1386,9 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go. google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422/go.mod h1:b6h1vNKhxaSoEI+5jc3PJUCustfli/mRab7295pY7rw= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240304161311-37d4d3c04a78 h1:YqFWYZXim8bG9v68xU8WjTZmYKb5M5dMeSOWIp6jogI= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240304161311-37d4d3c04a78/go.mod h1:vh/N7795ftP0AkN1w8XKqN4w1OdUKXW5Eummda+ofv8= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240314234333-6e1732d8331c h1:4z0DVWmDWWZ4OeQHLrb6lLBE3uCgSLs9DDA5Zb36DFg= @@ -1333,6 +1397,8 @@ google.golang.org/genproto/googleapis/bytestream v0.0.0-20240521202816-d264139d6 google.golang.org/genproto/googleapis/bytestream v0.0.0-20240521202816-d264139d666e/go.mod h1:0J6mmn3XAEjfNbPvpH63c0RXCjGNFcCzlEfWSN4In+k= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240722135656-d784300faade h1:fc+h2kSr2nW2DHxAdGYeX3bnkr4iFsKHUu9Fi6Rh4Y8= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240722135656-d784300faade/go.mod h1:5/MT647Cn/GGhwTpXC7QqcaR5Cnee4v4MKCU1/nwnIQ= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20250303144028-a0af3efb3deb h1:kw/Q892zrnljh8PXAIHmsCXgpxtSyWL4oV1eRnFtdeg= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:35wIojE/F1ptq1nfNDNjtowabHoMSA2qQs7+smpCO5s= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= @@ -1352,11 +1418,16 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240509183442-62759503f434/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d/go.mod h1:3ENsm/5D1mzDyhpzeRi1NR784I0BcofWBoSc5QqqMK4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= @@ -1370,7 +1441,12 @@ google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFL google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/grpc v1.67.3/go.mod h1:YGaHCc6Oap+FzBJTZLBzkGSYt/cvGPFTPxkn7QfSU8s= +google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= @@ -1378,7 +1454,11 @@ google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= From 06ad8fc7b0fd89f6495f95ad2652c601cd26bc8e Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 17 Mar 2025 12:09:38 +0800 Subject: [PATCH 09/13] chore: update dependencies in go.mod and go.sum - Updated Go version to 1.23.7 in backend/go.mod. - Upgraded various dependencies to their latest versions for improved stability and performance. - Adjusted indirect dependencies for better compatibility. - Enhanced overall clarity by standardizing import paths across the project. --- backend/go.mod | 71 +++++++++++++++++++++++++------------------------- backend/go.sum | 52 ++++++++++++++++++++++++++++++++++++ go.work.sum | 1 + 3 files changed, 88 insertions(+), 36 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index cacc2600..8bb8641e 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,8 +1,6 @@ module crawlab -go 1.23 - -toolchain go1.23.0 +go 1.23.7 replace ( github.com/crawlab-team/crawlab/core => ../core @@ -14,9 +12,9 @@ replace ( require github.com/crawlab-team/crawlab/core v0.0.0 require ( - cloud.google.com/go/auth v0.7.2 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect - cloud.google.com/go/compute/metadata v0.5.2 // indirect + cloud.google.com/go/auth v0.15.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect + cloud.google.com/go/compute/metadata v0.6.0 // indirect dario.cat/mergo v1.0.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect @@ -24,22 +22,23 @@ require ( github.com/ReneKroon/ttlcache v1.7.0 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/apex/log v1.9.0 // indirect - github.com/bytedance/sonic v1.11.6 // indirect - github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/bytedance/sonic v1.13.1 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect - github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect github.com/crawlab-team/crawlab/grpc v0.0.0 // indirect github.com/crawlab-team/crawlab/trace v0.0.0 // indirect github.com/crawlab-team/crawlab/vcs v0.0.0 // indirect + github.com/crawlab-team/fizz v0.0.0-20250317032929-767b58e01fa9 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.3 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.8 // indirect + github.com/gin-contrib/sse v1.0.0 // indirect github.com/gin-gonic/gin v1.10.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect @@ -49,17 +48,17 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.20.0 // indirect - github.com/goccy/go-json v0.10.2 // indirect + github.com/go-playground/validator/v10 v10.25.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/gofrs/uuid v3.2.0+incompatible // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gomarkdown/markdown v0.0.0-20241105142532-d03b89096d81 // indirect - github.com/google/s2a-go v0.1.8 // indirect + github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.13.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.5 // indirect + github.com/googleapis/gax-go/v2 v2.14.1 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -72,7 +71,7 @@ require ( github.com/juju/errors v1.0.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.17.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/loopfz/gadgeto v0.9.0 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -81,8 +80,8 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/robfig/cron/v3 v3.0.0 // indirect @@ -107,28 +106,28 @@ require ( github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect github.com/ztrue/tracerr v0.4.0 // indirect - go.mongodb.org/mongo-driver v1.15.1 // indirect + go.mongodb.org/mongo-driver v1.17.3 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect - go.opentelemetry.io/otel v1.31.0 // indirect - go.opentelemetry.io/otel/metric v1.31.0 // indirect - go.opentelemetry.io/otel/trace v1.31.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect + go.opentelemetry.io/otel v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.34.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/arch v0.8.0 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/arch v0.15.0 // indirect + golang.org/x/crypto v0.36.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect - google.golang.org/api v0.189.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 // indirect - google.golang.org/grpc v1.69.2 // indirect - google.golang.org/protobuf v1.36.1 // indirect + golang.org/x/net v0.37.0 // indirect + golang.org/x/oauth2 v0.28.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/api v0.226.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/grpc v1.71.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/go-playground/validator.v9 v9.30.0 // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect diff --git a/backend/go.sum b/backend/go.sum index ebd397ca..9a7c15c5 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -29,8 +29,10 @@ cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0c cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE= cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= +cloud.google.com/go/auth v0.15.0/go.mod h1:WJDGqZ1o9E9wKIL+IwStfyn/+s59zl4Bi+1KQNVXLZ8= cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= +cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -39,6 +41,7 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= @@ -98,8 +101,12 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic v1.13.1 h1:Jyd5CIvdFnkOWuKXr+wm4Nyk2h0yAFsr8ucJgEasO3g= +github.com/bytedance/sonic v1.13.1/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -118,6 +125,8 @@ github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vc github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -135,6 +144,8 @@ github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crawlab-team/fizz v0.0.0-20250317032929-767b58e01fa9 h1:Rnc78P4Wib2XBPd0VeGx0XtuF9sufmJepFLjXiYLyrU= +github.com/crawlab-team/fizz v0.0.0-20250317032929-767b58e01fa9/go.mod h1:PotXqgZl5b2duH/101hv5/C6811jptoh1HxWOmObM+4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= @@ -170,12 +181,16 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= +github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/cors v1.3.0/go.mod h1:artPvLlhkF7oG06nK8v3U8TNz6IeX+w1uzCSEId5/Vc= github.com/gin-contrib/sse v0.0.0-20190125020943-a7658810eb74/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= +github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= @@ -222,9 +237,13 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.25.0 h1:5Dh7cjvzR7BRZadnsVOzPhWsrwUr0nmsZJxEAnFLNO8= +github.com/go-playground/validator/v10 v10.25.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -309,6 +328,7 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -316,12 +336,14 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/enterprise-certificate-proxy v0.3.5/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= @@ -396,6 +418,8 @@ github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ib github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -462,6 +486,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -472,6 +497,8 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -562,6 +589,7 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= @@ -600,6 +628,7 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -615,6 +644,8 @@ go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3 go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.mongodb.org/mongo-driver v1.15.1 h1:l+RvoUOoMXFmADTLfYDm7On9dRm7p4T80/lEQM+r7HU= go.mongodb.org/mongo-driver v1.15.1/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ= +go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -626,16 +657,20 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= @@ -648,6 +683,8 @@ go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= +golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -667,6 +704,8 @@ golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2Uz golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -757,6 +796,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -776,6 +817,7 @@ golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -791,6 +833,7 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -872,6 +915,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -896,6 +941,8 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -993,6 +1040,7 @@ google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3h google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI= google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8= +google.golang.org/api v0.226.0/go.mod h1:WP/0Xm4LVvMOCldfvOISnWquSRWbG2kArDZcg+W2DbY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1068,6 +1116,7 @@ google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1: google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 h1:LWZqQOEjDyONlF1H6afSWpAL/znlREo2tHfLoe+8LMA= google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1097,6 +1146,7 @@ google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1113,6 +1163,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= diff --git a/go.work.sum b/go.work.sum index 15ebccf1..f4f9a6fb 100644 --- a/go.work.sum +++ b/go.work.sum @@ -13,6 +13,7 @@ cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVqux cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= +cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= cloud.google.com/go v0.112.2/go.mod h1:iEqjp//KquGIJV/m+Pk3xecgKNhV+ry+vVTsy4TbDms= cloud.google.com/go v0.113.0/go.mod h1:glEqlogERKYeePz6ZdkcLJ28Q2I6aERgDDErBg9GzO8= cloud.google.com/go v0.114.0 h1:OIPFAdfrFDFO2ve2U7r/H5SwSbBzEdrBdE7xkgwc+kY= From 4dace3ce8e5ae37f0c36abc41b76ca690b9c2aa0 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 17 Mar 2025 12:50:50 +0800 Subject: [PATCH 10/13] refactor: migrate router and controller methods to use Fizz package - Updated router groups to utilize the new crawlab-team/fizz package for improved routing capabilities. - Refactored controller methods to accept Fizz router groups, enhancing consistency and maintainability. - Simplified route registration by incorporating OpenAPI metadata directly into group definitions. - Improved error handling and response generation in sync controller methods for better clarity. - Enhanced overall code structure by standardizing route definitions and improving parameter handling. --- core/controllers/router.go | 86 ++++++++++++++++++-------------------- core/controllers/sync.go | 25 ++++------- core/openapi/wrapper.go | 19 +++++---- 3 files changed, 57 insertions(+), 73 deletions(-) diff --git a/core/controllers/router.go b/core/controllers/router.go index fc6c5450..f73b6de2 100644 --- a/core/controllers/router.go +++ b/core/controllers/router.go @@ -4,6 +4,8 @@ import ( "net/http" "strings" + "github.com/crawlab-team/fizz" + "github.com/crawlab-team/crawlab/core/middlewares" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/openapi" @@ -12,9 +14,9 @@ import ( // RouterGroups defines the different authentication levels for API routes type RouterGroups struct { - AuthGroup *gin.RouterGroup // Routes requiring full authentication + AuthGroup *fizz.RouterGroup // Routes requiring full authentication + AnonymousGroup *fizz.RouterGroup // Public routes that don't require auth SyncAuthGroup *gin.RouterGroup // Routes for sync operations with special auth - AnonymousGroup *gin.RouterGroup // Public routes that don't require auth Wrapper *openapi.FizzWrapper // OpenAPI wrapper for documentation } @@ -31,32 +33,34 @@ func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) { // Create OpenAPI wrapper globalWrapper = openapi.GetFizzWrapper(app) + f := globalWrapper.GetFizz() + return &RouterGroups{ - AuthGroup: app.Group("/", middlewares.AuthorizationMiddleware()), + AuthGroup: f.Group("/", "AuthGroup", "Router group that requires authentication", middlewares.AuthorizationMiddleware()), + AnonymousGroup: f.Group("/", "AnonymousGroup", "Router group that doesn't require authentication"), SyncAuthGroup: app.Group("/", middlewares.SyncAuthorizationMiddleware()), - AnonymousGroup: app.Group("/"), Wrapper: globalWrapper, } } // RegisterController registers a generic controller with standard CRUD endpoints // and any additional custom actions -func RegisterController[T any](group *gin.RouterGroup, basePath string, ctr *BaseController[T]) { +func RegisterController[T any](group *fizz.RouterGroup, basePath string, ctr *BaseController[T]) { // Track registered paths to avoid duplicates actionPaths := make(map[string]bool) for _, action := range ctr.actions { - path := basePath + action.Path - key := action.Method + " - " + path + fullPath := basePath + action.Path + key := action.Method + " - " + fullPath actionPaths[key] = true // Create appropriate model response based on the action responses := globalWrapper.BuildModelResponse() - id := getIDForAction(action.Method, path) + id := getIDForAction(action.Method, fullPath) summary := getSummaryForAction(action.Method, basePath, action.Path) description := getDescriptionForAction(action.Method, basePath, action.Path) - globalWrapper.RegisterRoute(action.Method, path, action.HandlerFunc, id, summary, description, responses) + globalWrapper.RegisterRoute(action.Method, fullPath, group, action.HandlerFunc, id, summary, description, responses) } // Register built-in handlers if they haven't been overridden @@ -72,24 +76,24 @@ func RegisterController[T any](group *gin.RouterGroup, basePath string, ctr *Bas } // RegisterActions registers a list of custom action handlers to a route group -func RegisterActions(group *gin.RouterGroup, basePath string, actions []Action) { +func RegisterActions(group *fizz.RouterGroup, basePath string, actions []Action) { for _, action := range actions { - path := basePath + action.Path + fullPath := basePath + action.Path // Create generic response responses := globalWrapper.BuildModelResponse() - id := getIDForAction(action.Method, path) + id := getIDForAction(action.Method, fullPath) summary := getSummaryForAction(action.Method, basePath, action.Path) description := getDescriptionForAction(action.Method, basePath, action.Path) - globalWrapper.RegisterRoute(action.Method, path, action.HandlerFunc, id, summary, description, responses) + globalWrapper.RegisterRoute(action.Method, fullPath, group, action.HandlerFunc, id, summary, description, responses) } } // registerBuiltinHandler registers a standard handler if it hasn't been overridden // by a custom action -func registerBuiltinHandler[T any](group *gin.RouterGroup, wrapper *openapi.FizzWrapper, basePath, method, pathSuffix string, handlerFunc interface{}, existingActionPaths map[string]bool, summary, description string, model T) { +func registerBuiltinHandler[T any](group *fizz.RouterGroup, wrapper *openapi.FizzWrapper, basePath, method, pathSuffix string, handlerFunc interface{}, existingActionPaths map[string]bool, summary, description string, model T) { path := basePath + pathSuffix key := method + " - " + path _, ok := existingActionPaths[key] @@ -102,7 +106,7 @@ func registerBuiltinHandler[T any](group *gin.RouterGroup, wrapper *openapi.Fizz // Create appropriate response based on the method responses := wrapper.BuildModelResponse() - wrapper.RegisterRoute(method, path, handlerFunc, id, summary, description, responses) + wrapper.RegisterRoute(method, path, group, handlerFunc, id, summary, description, responses) } // Helper functions to generate OpenAPI documentation @@ -222,17 +226,17 @@ func InitRoutes(app *gin.Engine) (err error) { // Register resource controllers with their respective endpoints // Each RegisterController call sets up standard CRUD operations // Additional custom actions can be specified in the controller initialization - RegisterController(groups.AuthGroup, "/data/collections", NewController[models.DataCollection]()) - RegisterController(groups.AuthGroup, "/environments", NewController[models.Environment]()) - RegisterController(groups.AuthGroup, "/nodes", NewController[models.Node]()) - RegisterController(groups.AuthGroup, "/projects", NewController[models.Project]([]Action{ + RegisterController(groups.AuthGroup.Group("", "Data Collections", "APIs for data collections management"), "/data/collections", NewController[models.DataCollection]()) + RegisterController(groups.AuthGroup.Group("", "Environments", "APIs for environment variables management"), "/environments", NewController[models.Environment]()) + RegisterController(groups.AuthGroup.Group("", "Nodes", "APIs for nodes management"), "/nodes", NewController[models.Node]()) + RegisterController(groups.AuthGroup.Group("", "Projects", "APIs for projects management"), "/projects", NewController[models.Project]([]Action{ { Method: http.MethodGet, Path: "", HandlerFunc: GetProjectList, }, }...)) - RegisterController(groups.AuthGroup, "/spiders", NewController[models.Spider]([]Action{ + RegisterController(groups.AuthGroup.Group("", "Spiders", "APIs for spiders management"), "/spiders", NewController[models.Spider]([]Action{ { Method: http.MethodGet, Path: "/:id", @@ -324,7 +328,7 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetSpiderResults, }, }...)) - RegisterController(groups.AuthGroup, "/schedules", NewController[models.Schedule]([]Action{ + RegisterController(groups.AuthGroup.Group("", "Schedules", "APIs for schedules management"), "/schedules", NewController[models.Schedule]([]Action{ { Method: http.MethodPost, Path: "", @@ -351,7 +355,7 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: PostScheduleRun, }, }...)) - RegisterController(groups.AuthGroup, "/tasks", NewController[models.Task]([]Action{ + RegisterController(groups.AuthGroup.Group("", "Tasks", "APIs for tasks management"), "/tasks", NewController[models.Task]([]Action{ { Method: http.MethodGet, Path: "/:id", @@ -393,7 +397,7 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetTaskLogs, }, }...)) - RegisterController(groups.AuthGroup, "/tokens", NewController[models.Token]([]Action{ + RegisterController(groups.AuthGroup.Group("", "Tokens", "APIs for PAT management"), "/tokens", NewController[models.Token]([]Action{ { Method: http.MethodPost, Path: "", @@ -405,7 +409,7 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetTokenList, }, }...)) - RegisterController(groups.AuthGroup, "/users", NewController[models.User]([]Action{ + RegisterController(groups.AuthGroup.Group("", "Users", "APIs for users management"), "/users", NewController[models.User]([]Action{ { Method: http.MethodGet, Path: "/:id", @@ -459,7 +463,7 @@ func InitRoutes(app *gin.Engine) (err error) { }...)) // Register standalone action routes that don't fit the standard CRUD pattern - RegisterActions(groups.AuthGroup, "/export", []Action{ + RegisterActions(groups.AuthGroup.Group("", "Export", "APIs for exporting data"), "/export", []Action{ { Method: http.MethodPost, Path: "/:type", @@ -476,7 +480,7 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetExportDownload, }, }) - RegisterActions(groups.AuthGroup, "/filters", []Action{ + RegisterActions(groups.AuthGroup.Group("", "Filters", "APIs for data collections filters management"), "/filters", []Action{ { Method: http.MethodGet, Path: "/:col", @@ -493,7 +497,7 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetFilterColFieldOptionsWithValueLabel, }, }) - RegisterActions(groups.AuthGroup, "/settings", []Action{ + RegisterActions(groups.AuthGroup.Group("", "Settings", "APIs for settings management"), "/settings", []Action{ { Method: http.MethodGet, Path: "/:key", @@ -510,7 +514,7 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: PutSetting, }, }) - RegisterActions(groups.AuthGroup, "/stats", []Action{ + RegisterActions(groups.AuthGroup.Group("", "Stats", "APIs for data stats"), "/stats", []Action{ { Method: http.MethodGet, Path: "/overview", @@ -528,29 +532,15 @@ func InitRoutes(app *gin.Engine) (err error) { }, }) - // Register sync routes that require special authentication - RegisterActions(groups.SyncAuthGroup, "/sync", []Action{ - { - Method: http.MethodGet, - Path: "/:id/scan", - HandlerFunc: GetSyncScan, - }, - { - Method: http.MethodGet, - Path: "/:id/download", - HandlerFunc: GetSyncDownload, - }, - }) - // Register public routes that don't require authentication - RegisterActions(groups.AnonymousGroup, "/system-info", []Action{ + RegisterActions(groups.AnonymousGroup.Group("", "System", "APIs for system info"), "/system-info", []Action{ { Path: "", Method: http.MethodGet, HandlerFunc: GetSystemInfo, }, }) - RegisterActions(groups.AnonymousGroup, "/", []Action{ + RegisterActions(groups.AnonymousGroup.Group("", "Auth", "APIs for authentication"), "/", []Action{ { Method: http.MethodPost, Path: "/login", @@ -563,11 +553,15 @@ func InitRoutes(app *gin.Engine) (err error) { }, }) + // Register sync routes that require special authentication + groups.SyncAuthGroup.GET("/sync/:id/scan", GetSyncScan) + groups.SyncAuthGroup.GET("/sync/:id/download", GetSyncDownload) + // Register health check route - groups.AnonymousGroup.GET("/health", GetHealthFn(func() bool { return true })) + groups.AnonymousGroup.GinRouterGroup().GET("/health", GetHealthFn(func() bool { return true })) // Register OpenAPI documentation route - groups.AnonymousGroup.GET("/openapi.json", GetOpenAPI) + groups.AnonymousGroup.GinRouterGroup().GET("/openapi.json", GetOpenAPI) return nil } diff --git a/core/controllers/sync.go b/core/controllers/sync.go index 81555c53..3f3216d1 100644 --- a/core/controllers/sync.go +++ b/core/controllers/sync.go @@ -3,34 +3,23 @@ package controllers import ( "path/filepath" - "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/utils" "github.com/gin-gonic/gin" ) -type GetSyncScanParams struct { - Id string `path:"id" validate:"required"` - Path string `query:"path"` -} - -func GetSyncScan(_ *gin.Context, params *GetSyncScanParams) (response *Response[map[string]entity.FsFileInfo], err error) { +func GetSyncScan(c *gin.Context) { workspacePath := utils.GetWorkspace() - dirPath := filepath.Join(workspacePath, params.Id, params.Path) + dirPath := filepath.Join(workspacePath, c.Param("id"), c.Param("path")) files, err := utils.ScanDirectory(dirPath) if err != nil { - return GetErrorResponse[map[string]entity.FsFileInfo](err) + HandleErrorInternalServerError(c, err) + return } - return GetDataResponse(files) + HandleSuccessWithData(c, files) } -type GetSyncDownloadParams struct { - Id string `path:"id" validate:"required"` - Path string `query:"path" validate:"required"` -} - -func GetSyncDownload(c *gin.Context, params *GetSyncDownloadParams) (err error) { +func GetSyncDownload(c *gin.Context) { workspacePath := utils.GetWorkspace() - filePath := filepath.Join(workspacePath, params.Id, params.Path) + filePath := filepath.Join(workspacePath, c.Param("id"), c.Param("path")) c.File(filePath) - return nil } diff --git a/core/openapi/wrapper.go b/core/openapi/wrapper.go index 4d4e8758..f75b7615 100644 --- a/core/openapi/wrapper.go +++ b/core/openapi/wrapper.go @@ -2,12 +2,13 @@ package openapi import ( "fmt" + "sync" + "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/fizz" "github.com/gin-gonic/gin" "github.com/loopfz/gadgeto/tonic" - "sync" ) // FizzWrapper wraps an existing Gin Engine to add OpenAPI functionality @@ -46,26 +47,26 @@ type Response struct { } // RegisterRoute registers a route with OpenAPI documentation -func (w *FizzWrapper) RegisterRoute(method, path string, handler interface{}, id, summary, description string, responses map[int]Response) { +func (w *FizzWrapper) RegisterRoute(method, path string, group *fizz.RouterGroup, handler interface{}, id, summary, description string, responses map[int]Response) { // Build operation options for OpenAPI documentation opts := w.buildOperationOptions(id, summary, description, responses) // Register the route with OpenAPI documentation switch method { case "GET": - w.fizz.GET(path, opts, tonic.Handler(handler, 200)) + group.GET(path, opts, tonic.Handler(handler, 200)) case "POST": - w.fizz.POST(path, opts, tonic.Handler(handler, 200)) + group.POST(path, opts, tonic.Handler(handler, 200)) case "PUT": - w.fizz.PUT(path, opts, tonic.Handler(handler, 200)) + group.PUT(path, opts, tonic.Handler(handler, 200)) case "DELETE": - w.fizz.DELETE(path, opts, tonic.Handler(handler, 200)) + group.DELETE(path, opts, tonic.Handler(handler, 200)) case "PATCH": - w.fizz.PATCH(path, opts, tonic.Handler(handler, 200)) + group.PATCH(path, opts, tonic.Handler(handler, 200)) case "HEAD": - w.fizz.HEAD(path, opts, tonic.Handler(handler, 200)) + group.HEAD(path, opts, tonic.Handler(handler, 200)) case "OPTIONS": - w.fizz.OPTIONS(path, opts, tonic.Handler(handler, 200)) + group.OPTIONS(path, opts, tonic.Handler(handler, 200)) } } From 657c73ad2f3078b320e7e4f5a65e498a5b2380cb Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 17 Mar 2025 14:20:19 +0800 Subject: [PATCH 11/13] refactor: improve filter query handling in utility functions - Added nil return for empty condition strings in GetFilterQueryFromConditionString and GetFilterFromConditionString functions to enhance error handling. - Improved clarity and maintainability by ensuring consistent behavior when handling empty input conditions. - Updated comments for better understanding of the changes made. --- core/controllers/utils.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/controllers/utils.go b/core/controllers/utils.go index b61bf8b1..2cae437d 100644 --- a/core/controllers/utils.go +++ b/core/controllers/utils.go @@ -42,6 +42,9 @@ func GetFilterQueryFromListParams(params *GetListParams) (q bson.M, err error) { } func GetFilterQueryFromConditionString(condStr string) (q bson.M, err error) { + if condStr == "" { + return nil, nil + } conditions, err := GetFilterFromConditionString(condStr) if err != nil { return nil, err @@ -50,6 +53,9 @@ func GetFilterQueryFromConditionString(condStr string) (q bson.M, err error) { } func GetFilterFromConditionString(condStr string) (f *entity.Filter, err error) { + if condStr == "" { + return nil, nil + } var conditions []*entity.Condition if err := json.Unmarshal([]byte(condStr), &conditions); err != nil { return nil, err From 95a6eb4e7ba0417464753cfc27041fd235b559a2 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 17 Mar 2025 14:24:07 +0800 Subject: [PATCH 12/13] chore: add OpenTelemetry auto SDK dependency - Introduced go.opentelemetry.io/auto/sdk v1.1.0 to go.mod and updated go.sum accordingly. - Enhanced observability capabilities by including the OpenTelemetry SDK for automatic instrumentation. - Ensured consistency in dependency management across the project. --- core/controllers/router_test.go | 7 ++++--- core/go.mod | 1 + core/go.sum | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/core/controllers/router_test.go b/core/controllers/router_test.go index 94a40bb0..d15cc2c1 100644 --- a/core/controllers/router_test.go +++ b/core/controllers/router_test.go @@ -1,11 +1,12 @@ package controllers_test import ( + "testing" + "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/models/models" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" - "testing" ) func TestRouterGroups(t *testing.T) { @@ -16,8 +17,8 @@ func TestRouterGroups(t *testing.T) { group *gin.RouterGroup name string }{ - {groups.AuthGroup, "AuthGroup"}, - {groups.AnonymousGroup, "AnonymousGroup"}, + {groups.AuthGroup.GinRouterGroup(), "AuthGroup"}, + {groups.AnonymousGroup.GinRouterGroup(), "AnonymousGroup"}, } for _, a := range assertions { diff --git a/core/go.mod b/core/go.mod index b9e39502..2e9aad2f 100644 --- a/core/go.mod +++ b/core/go.mod @@ -120,6 +120,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.3 // indirect github.com/ztrue/tracerr v0.4.0 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect go.opentelemetry.io/otel v1.34.0 // indirect go.opentelemetry.io/otel/metric v1.34.0 // indirect diff --git a/core/go.sum b/core/go.sum index e103cef7..e6562d76 100644 --- a/core/go.sum +++ b/core/go.sum @@ -661,6 +661,8 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= From 98843fdbaccfdcad2e771f887dd26cba7e763bd5 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 17 Mar 2025 17:14:08 +0800 Subject: [PATCH 13/13] refactor: standardize sorting logic across controllers - Introduced a consistent approach for sorting in multiple controller methods by utilizing GetSortOptionFromString for improved clarity and maintainability. - Updated the GetListParams structure to ensure sorting is handled uniformly across different entities. - Enhanced error handling for sorting parameters to provide clearer feedback on invalid formats. - Improved comments for better understanding of sorting logic and its integration within the controller methods. --- core/controllers/base.go | 58 ++++++++++++++++++++++++++++++++----- core/controllers/project.go | 7 ++++- core/controllers/spider.go | 24 ++++++++------- core/controllers/task.go | 7 ++++- core/controllers/token.go | 8 ++++- core/controllers/user.go | 8 ++++- core/controllers/utils.go | 21 ++++++++++++++ 7 files changed, 111 insertions(+), 22 deletions(-) diff --git a/core/controllers/base.go b/core/controllers/base.go index 52cc3bcf..ade6602c 100644 --- a/core/controllers/base.go +++ b/core/controllers/base.go @@ -1,6 +1,8 @@ package controllers import ( + "github.com/loopfz/gadgeto/tonic" + "net/http" "time" "github.com/crawlab-team/crawlab/core/interfaces" @@ -13,6 +15,33 @@ import ( mongo2 "go.mongodb.org/mongo-driver/mongo" ) +func init() { + tonic.SetErrorHook(func(context *gin.Context, err error) (int, interface{}) { + response := gin.H{ + "error": errors.Unwrap(err).Error(), + } + status := http.StatusInternalServerError + constErr, ok := errors.AsType[errors.ConstError](err) + if ok { + switch { + case errors.Is(constErr, errors.NotFound): + status = http.StatusNotFound + case errors.Is(constErr, errors.BadRequest): + status = http.StatusBadRequest + case errors.Is(constErr, errors.Unauthorized): + status = http.StatusUnauthorized + case errors.Is(constErr, errors.Forbidden): + status = http.StatusForbidden + default: + status = http.StatusInternalServerError + } + } else { + status = http.StatusInternalServerError + } + return status, response + }) +} + type Action struct { Method string Path string @@ -27,7 +56,7 @@ type BaseController[T any] struct { // GetListParams represents parameters for GetList with pagination type GetListParams struct { Conditions string `query:"conditions" description:"Filter conditions. Format: [{\"key\":\"name\",\"op\":\"eq\",\"value\":\"test\"}]"` - Sort bson.D `query:"sort" description:"Sort options"` + Sort string `query:"sort" description:"Sort options"` Page int `query:"page" default:"1" description:"Page number"` Size int `query:"size" default:"10" description:"Page size"` All bool `query:"all" default:"false" description:"Whether to get all items"` @@ -201,15 +230,19 @@ func (ctr *BaseController[T]) DeleteList(_ *gin.Context, params *DeleteListParam // GetAll retrieves all items based on filter and sort func (ctr *BaseController[T]) GetAll(params *GetListParams) (response *ListResponse[T], err error) { + // Get filter query query, err := GetFilterQueryFromListParams(params) if err != nil { return GetErrorListResponse[T](errors.BadRequestf("invalid request parameters: %v", err)) } - sort := params.Sort - if sort == nil { - sort = bson.D{{"_id", -1}} + + // Get sort options + sort, err := GetSortOptionFromString(params.Sort) + if err != nil { + return GetErrorListResponse[T](errors.BadRequestf("invalid sort format: %v", err)) } + // Get models models, err := ctr.modelSvc.GetMany(query, &mongo.FindOptions{ Sort: sort, }) @@ -217,22 +250,33 @@ func (ctr *BaseController[T]) GetAll(params *GetListParams) (response *ListRespo return nil, err } + // Total count total, err := ctr.modelSvc.Count(query) if err != nil { return nil, err } + // Response return GetListResponse(models, total) } // GetWithPagination retrieves items with pagination func (ctr *BaseController[T]) GetWithPagination(params *GetListParams) (response *ListResponse[T], err error) { + // Get filter query query, err := GetFilterQueryFromListParams(params) if err != nil { return GetErrorListResponse[T](errors.BadRequestf("invalid request parameters: %v", err)) } + + // Get sort options + sort, err := GetSortOptionFromString(params.Sort) + if err != nil { + return GetErrorListResponse[T](errors.BadRequestf("invalid sort format: %v", err)) + } + + // Get models models, err := ctr.modelSvc.GetMany(query, &mongo.FindOptions{ - Sort: params.Sort, + Sort: sort, Skip: params.Size * (params.Page - 1), Limit: params.Size, }) @@ -244,13 +288,13 @@ func (ctr *BaseController[T]) GetWithPagination(params *GetListParams) (response } } - // total count + // Total count total, err := ctr.modelSvc.Count(query) if err != nil { return nil, err } - // response + // Response return GetListResponse(models, total) } diff --git a/core/controllers/project.go b/core/controllers/project.go index 8ec44378..ef5ebfc4 100644 --- a/core/controllers/project.go +++ b/core/controllers/project.go @@ -21,9 +21,14 @@ func GetProjectList(c *gin.Context, params *GetListParams) (response *ListRespon return GetErrorListResponse[models.Project](errors.BadRequestf("invalid request parameters: %v", err)) } + sort, err := GetSortOptionFromString(params.Sort) + if err != nil { + return GetErrorListResponse[models.Project](errors.BadRequestf("invalid request parameters: %v", err)) + } + // get list projects, err := service.NewModelService[models.Project]().GetMany(query, &mongo.FindOptions{ - Sort: params.Sort, + Sort: sort, Skip: params.Size * (params.Page - 1), Limit: params.Size, }) diff --git a/core/controllers/spider.go b/core/controllers/spider.go index 722efeb3..a47e5ca0 100644 --- a/core/controllers/spider.go +++ b/core/controllers/spider.go @@ -93,9 +93,14 @@ func getSpiderListWithStats(params *GetListParams) (response *ListResponse[model if err != nil { return GetErrorListResponse[models.Spider](errors.BadRequestf("invalid request parameters: %v", err)) } - // get list + + sort, err := GetSortOptionFromString(params.Sort) + if err != nil { + return GetErrorListResponse[models.Spider](errors.BadRequestf("invalid request parameters: %v", err)) + } + spiders, err := service.NewModelService[models.Spider]().GetMany(query, &mongo2.FindOptions{ - Sort: params.Sort, + Sort: sort, Skip: params.Size * (params.Page - 1), Limit: params.Size, }) @@ -662,7 +667,7 @@ type PostSpiderExportParams struct { Id string `path:"id"` } -func PostSpiderExport(c *gin.Context, params *PostSpiderExportParams) (err error) { +func PostSpiderExport(c *gin.Context, _ *PostSpiderExportParams) (err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { return err @@ -712,9 +717,6 @@ func PostSpiderRun(c *gin.Context, params *PostSpiderRunParams) (response *Respo ScheduleId: scheduleId, Priority: params.Priority, } - if err := c.ShouldBindJSON(&opts); err != nil { - return GetErrorResponse[[]primitive.ObjectID](err) - } // user if u := GetUserFromContext(c); u != nil { @@ -731,7 +733,9 @@ func PostSpiderRun(c *gin.Context, params *PostSpiderRunParams) (response *Respo } type GetSpiderResultsParams struct { - Id string `path:"id"` + Id string `path:"id"` + Page int `query:"page"` + Size int `query:"size"` } func GetSpiderResults(c *gin.Context, params *GetSpiderResultsParams) (response *ListResponse[bson.M], err error) { @@ -745,8 +749,6 @@ func GetSpiderResults(c *gin.Context, params *GetSpiderResultsParams) (response return GetErrorListResponse[bson.M](err) } - // params - pagination := MustGetPagination(c) query := getResultListQuery(c) col := mongo2.GetMongoCol(s.ColName) @@ -754,8 +756,8 @@ func GetSpiderResults(c *gin.Context, params *GetSpiderResultsParams) (response var results []bson.M err = col.Find(mongo2.GetMongoQuery(query), mongo2.GetMongoOpts(&mongo2.ListOptions{ Sort: []mongo2.ListSort{{"_id", mongo2.SortDirectionDesc}}, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, + Skip: params.Size * (params.Page - 1), + Limit: params.Size, })).All(&results) if err != nil { return GetErrorListResponse[bson.M](err) diff --git a/core/controllers/task.go b/core/controllers/task.go index 25fb80e8..d772b457 100644 --- a/core/controllers/task.go +++ b/core/controllers/task.go @@ -84,9 +84,14 @@ func GetTaskList(c *gin.Context, params *GetTaskListParams) (response *ListRespo return GetErrorListResponse[models.Task](err) } + sort, err := GetSortOptionFromString(params.GetListParams.Sort) + if err != nil { + return GetErrorListResponse[models.Task](err) + } + // get tasks tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo3.FindOptions{ - Sort: params.Sort, + Sort: sort, Skip: params.Size * (params.Page - 1), Limit: params.Size, }) diff --git a/core/controllers/token.go b/core/controllers/token.go index 1fab41c2..4c052e02 100644 --- a/core/controllers/token.go +++ b/core/controllers/token.go @@ -51,9 +51,15 @@ func GetTokenList(c *gin.Context, params *GetListParams) (response *ListResponse // Add filter for tokens created by the current user query["created_by"] = u.Id + // Get sort options + sort, err := GetSortOptionFromString(params.Sort) + if err != nil { + return GetErrorListResponse[models.Token](errors.BadRequestf("invalid request parameters: %v", err)) + } + // Get tokens with pagination tokens, err := service.NewModelService[models.Token]().GetMany(query, &mongo.FindOptions{ - Sort: params.Sort, + Sort: sort, Skip: params.Size * (params.Page - 1), Limit: params.Size, }) diff --git a/core/controllers/user.go b/core/controllers/user.go index 435b534f..9dd55664 100644 --- a/core/controllers/user.go +++ b/core/controllers/user.go @@ -28,8 +28,14 @@ func GetUserList(_ *gin.Context, params *GetListParams) (response *ListResponse[ if err != nil { return GetErrorListResponse[models.User](err) } + + sort, err := GetSortOptionFromString(params.Sort) + if err != nil { + return GetErrorListResponse[models.User](err) + } + users, err := service.NewModelService[models.User]().GetMany(query, &mongo.FindOptions{ - Sort: params.Sort, + Sort: sort, Skip: params.Size * (params.Page - 1), Limit: params.Size, }) diff --git a/core/controllers/utils.go b/core/controllers/utils.go index 2cae437d..0313bc44 100644 --- a/core/controllers/utils.go +++ b/core/controllers/utils.go @@ -205,6 +205,27 @@ func MustGetPagination(c *gin.Context) (p *entity.Pagination) { return p } +func GetSortsFromString(sortStr string) (sorts []entity.Sort, err error) { + if sortStr == "" { + return nil, nil + } + if err := json.Unmarshal([]byte(sortStr), &sorts); err != nil { + return nil, err + } + return sorts, nil +} + +func GetSortOptionFromString(sortStr string) (sort bson.D, err error) { + sorts, err := GetSortsFromString(sortStr) + if err != nil { + return nil, err + } + if sorts == nil || len(sorts) == 0 { + return bson.D{{"_id", -1}}, nil + } + return SortsToOption(sorts) +} + // GetSorts Get entity.Sort from gin.Context func GetSorts(c *gin.Context) (sorts []entity.Sort, err error) { // bind