diff --git a/core/controllers/base.go b/core/controllers/base.go index ade6602c..2ec01277 100644 --- a/core/controllers/base.go +++ b/core/controllers/base.go @@ -1,10 +1,11 @@ package controllers import ( - "github.com/loopfz/gadgeto/tonic" "net/http" "time" + "github.com/loopfz/gadgeto/tonic" + "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/mongo" @@ -45,6 +46,8 @@ func init() { type Action struct { Method string Path string + Name string + Description string HandlerFunc interface{} } @@ -57,8 +60,8 @@ type BaseController[T any] struct { type GetListParams struct { Conditions string `query:"conditions" description:"Filter conditions. Format: [{\"key\":\"name\",\"op\":\"eq\",\"value\":\"test\"}]"` Sort string `query:"sort" description:"Sort options"` - Page int `query:"page" default:"1" description:"Page number"` - Size int `query:"size" default:"10" description:"Page size"` + Page int `query:"page" default:"1" description:"Page number" minimum:"1"` + Size int `query:"size" default:"10" description:"Page size" minimum:"1"` All bool `query:"all" default:"false" description:"Whether to get all items"` } @@ -72,7 +75,7 @@ func (ctr *BaseController[T]) GetList(_ *gin.Context, params *GetListParams) (re } type GetByIdParams struct { - Id string `path:"id" description:"The ID of the item to get"` + Id string `path:"id" description:"The ID of the item to get" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` } func (ctr *BaseController[T]) GetById(_ *gin.Context, params *GetByIdParams) (response *Response[T], err error) { @@ -90,7 +93,7 @@ func (ctr *BaseController[T]) GetById(_ *gin.Context, params *GetByIdParams) (re } type PostParams[T any] struct { - Data T `json:"data"` + Data T `json:"data" description:"The data to create" validate:"required"` } func (ctr *BaseController[T]) Post(c *gin.Context, params *PostParams[T]) (response *Response[T], err error) { @@ -114,8 +117,8 @@ func (ctr *BaseController[T]) Post(c *gin.Context, params *PostParams[T]) (respo } type PutByIdParams[T any] struct { - Id string `path:"id" description:"The ID of the item to update"` - Data T `json:"data"` + Id string `path:"id" description:"The ID of the item to update" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Data T `json:"data" description:"The data to update" validate:"required"` } func (ctr *BaseController[T]) PutById(c *gin.Context, params *PutByIdParams[T]) (response *Response[T], err error) { @@ -144,7 +147,7 @@ func (ctr *BaseController[T]) PutById(c *gin.Context, params *PutByIdParams[T]) } type PatchParams struct { - Ids []string `json:"ids" description:"The IDs of the items to update" validate:"required"` + Ids []string `json:"ids" description:"The IDs of the items to update" validate:"required" items.type:"string" items.format:"objectid" items.pattern:"^[0-9a-fA-F]{24}$"` Update bson.M `json:"update" description:"The update object" validate:"required"` } @@ -184,7 +187,7 @@ func (ctr *BaseController[T]) PatchList(c *gin.Context, params *PatchParams) (re } type DeleteByIdParams struct { - Id string `path:"id" description:"The ID of the item to get"` + Id string `path:"id" description:"The ID of the item to delete" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` } func (ctr *BaseController[T]) DeleteById(c *gin.Context, params *DeleteByIdParams) (res *Response[T], err error) { @@ -203,7 +206,7 @@ func (ctr *BaseController[T]) DeleteById(c *gin.Context, params *DeleteByIdParam } type DeleteListParams struct { - Ids []string `json:"ids" description:"The IDs of the items to delete"` + Ids []string `json:"ids" description:"The IDs of the items to delete" items.type:"string" items.format:"objectid" items.pattern:"^[0-9a-fA-F]{24}$"` } func (ctr *BaseController[T]) DeleteList(_ *gin.Context, params *DeleteListParams) (res *Response[T], err error) { diff --git a/core/controllers/login.go b/core/controllers/login.go index b95806f4..9ab48890 100644 --- a/core/controllers/login.go +++ b/core/controllers/login.go @@ -8,8 +8,8 @@ import ( ) type PostLoginParams struct { - Username string `json:"username" validate:"required"` - Password string `json:"password" validate:"required"` + Username string `json:"username" description:"Username" validate:"required"` + Password string `json:"password" description:"Password" validate:"required"` } func PostLogin(c *gin.Context, params *PostLoginParams) (response *Response[string], err error) { diff --git a/core/controllers/router.go b/core/controllers/router.go index f73b6de2..c0f9aa0b 100644 --- a/core/controllers/router.go +++ b/core/controllers/router.go @@ -1,9 +1,12 @@ package controllers import ( + "fmt" "net/http" "strings" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/fizz" "github.com/crawlab-team/crawlab/core/middlewares" @@ -56,23 +59,22 @@ func RegisterController[T any](group *fizz.RouterGroup, basePath string, ctr *Ba // Create appropriate model response based on the action responses := globalWrapper.BuildModelResponse() - id := getIDForAction(action.Method, fullPath) - summary := getSummaryForAction(action.Method, basePath, action.Path) - description := getDescriptionForAction(action.Method, basePath, action.Path) + id := getIDForAction(action.Method, fullPath, action.Name) + summary := getSummaryForAction(action.Method, basePath, action.Path, action.Name) + description := getDescriptionForAction(action.Method, basePath, action.Path, action.Description) globalWrapper.RegisterRoute(action.Method, fullPath, group, action.HandlerFunc, id, summary, description, responses) } // Register built-in handlers if they haven't been overridden - // Create a zero value of T to use as the model - var model T - registerBuiltinHandler(group, globalWrapper, basePath, http.MethodGet, "", ctr.GetList, actionPaths, "Get list", "Get a list of items", model) - registerBuiltinHandler(group, globalWrapper, basePath, http.MethodGet, "/:id", ctr.GetById, actionPaths, "Get by ID", "Get a single item by ID", model) - registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPost, "", ctr.Post, actionPaths, "Create", "Create a new item", model) - registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPut, "/:id", ctr.PutById, actionPaths, "Update by ID", "Update an item by ID", model) - registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPatch, "", ctr.PatchList, actionPaths, "Patch list", "Patch multiple items", model) - registerBuiltinHandler(group, globalWrapper, basePath, http.MethodDelete, "/:id", ctr.DeleteById, actionPaths, "Delete by ID", "Delete an item by ID", model) - registerBuiltinHandler(group, globalWrapper, basePath, http.MethodDelete, "", ctr.DeleteList, actionPaths, "Delete list", "Delete multiple items", model) + resource := getResourceName(basePath) + registerBuiltinHandler(group, globalWrapper, basePath, http.MethodGet, "", ctr.GetList, actionPaths, fmt.Sprintf("Get %s List", resource), "Get a list of items") + registerBuiltinHandler(group, globalWrapper, basePath, http.MethodGet, "/:id", ctr.GetById, actionPaths, fmt.Sprintf("Get %s by ID", resource), "Get a single item by ID") + registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPost, "", ctr.Post, actionPaths, fmt.Sprintf("Create %s", resource), "Create a new item") + registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPut, "/:id", ctr.PutById, actionPaths, fmt.Sprintf("Update %s by ID", resource), "Update an item by ID") + registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPatch, "", ctr.PatchList, actionPaths, fmt.Sprintf("Patch %s List", resource), "Patch multiple items") + registerBuiltinHandler(group, globalWrapper, basePath, http.MethodDelete, "/:id", ctr.DeleteById, actionPaths, fmt.Sprintf("Delete %s by ID", resource), "Delete an item by ID") + registerBuiltinHandler(group, globalWrapper, basePath, http.MethodDelete, "", ctr.DeleteList, actionPaths, fmt.Sprintf("Delete %s List", resource), "Delete multiple items") } // RegisterActions registers a list of custom action handlers to a route group @@ -83,9 +85,9 @@ func RegisterActions(group *fizz.RouterGroup, basePath string, actions []Action) // Create generic response responses := globalWrapper.BuildModelResponse() - id := getIDForAction(action.Method, fullPath) - summary := getSummaryForAction(action.Method, basePath, action.Path) - description := getDescriptionForAction(action.Method, basePath, action.Path) + id := getIDForAction(action.Method, fullPath, action.Name) + summary := getSummaryForAction(action.Method, basePath, action.Path, action.Name) + description := getDescriptionForAction(action.Method, basePath, action.Path, action.Description) globalWrapper.RegisterRoute(action.Method, fullPath, group, action.HandlerFunc, id, summary, description, responses) } @@ -93,7 +95,7 @@ func RegisterActions(group *fizz.RouterGroup, basePath string, actions []Action) // registerBuiltinHandler registers a standard handler if it hasn't been overridden // by a custom action -func registerBuiltinHandler[T any](group *fizz.RouterGroup, wrapper *openapi.FizzWrapper, basePath, method, pathSuffix string, handlerFunc interface{}, existingActionPaths map[string]bool, summary, description string, model T) { +func registerBuiltinHandler(group *fizz.RouterGroup, wrapper *openapi.FizzWrapper, basePath, method, pathSuffix string, handlerFunc interface{}, existingActionPaths map[string]bool, summary, description string) { path := basePath + pathSuffix key := method + " - " + path _, ok := existingActionPaths[key] @@ -101,7 +103,7 @@ func registerBuiltinHandler[T any](group *fizz.RouterGroup, wrapper *openapi.Fiz return } - id := getIDForAction(method, path) + id := getIDForAction(method, path, summary) // Create appropriate response based on the method responses := wrapper.BuildModelResponse() @@ -110,7 +112,11 @@ func registerBuiltinHandler[T any](group *fizz.RouterGroup, wrapper *openapi.Fiz } // Helper functions to generate OpenAPI documentation -func getIDForAction(method, path string) string { +func getIDForAction(method, path, summary string) string { + if summary != "" { + return utils.ToPascalCase(summary) + } + // Remove leading slash and convert remaining slashes to underscores cleanPath := strings.TrimPrefix(path, "/") cleanPath = strings.ReplaceAll(cleanPath, "/", "_") @@ -120,7 +126,11 @@ func getIDForAction(method, path string) string { return method + "_" + cleanPath } -func getSummaryForAction(method, basePath, path string) string { +func getSummaryForAction(method, basePath, path, summary string) string { + if summary != "" { + return summary + } + resource := getResourceName(basePath) switch method { @@ -140,13 +150,13 @@ func getSummaryForAction(method, basePath, path string) string { } case http.MethodPatch: if path == "" { - return "Patch " + resource + " list" + return "Patch " + resource + " List" } case http.MethodDelete: if path == "/:id" { return "Delete " + resource + " by ID" } else if path == "" { - return "Delete " + resource + " list" + return "Delete " + resource + " List" } } @@ -158,7 +168,11 @@ func getSummaryForAction(method, basePath, path string) string { return method + " " + resource } -func getDescriptionForAction(method, basePath, path string) string { +func getDescriptionForAction(method, basePath, path, description string) string { + if description != "" { + return description + } + resource := getResourceName(basePath) switch method { @@ -197,22 +211,25 @@ func getDescriptionForAction(method, basePath, path string) string { } func getResourceName(basePath string) string { + resource := basePath + // Remove leading slash and get the last part of the path - if len(basePath) > 0 && basePath[0] == '/' { - basePath = basePath[1:] + if len(resource) > 0 && resource[0] == '/' { + resource = resource[1:] } // Remove trailing slash if present - if len(basePath) > 0 && basePath[len(basePath)-1] == '/' { - basePath = basePath[:len(basePath)-1] + if len(resource) > 0 && resource[len(resource)-1] == '/' { + resource = resource[:len(resource)-1] } - // If path is empty, return "resource" - if basePath == "" { - return "resource" - } + // Convert to capitalized form + resource = utils.Capitalize(resource) - return basePath + // Convert to non-plural form + resource = strings.TrimSuffix(resource, "s") + + return resource } // InitRoutes configures all API routes for the application @@ -233,6 +250,8 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodGet, Path: "", + Name: "Get Project List", + Description: "Get a list of projects", HandlerFunc: GetProjectList, }, }...)) @@ -240,91 +259,127 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodGet, Path: "/:id", + Name: "Get Spider by ID", + Description: "Get a single spider by ID", HandlerFunc: GetSpiderById, }, { Method: http.MethodGet, Path: "", + Name: "Get Spider List", + Description: "Get a list of spiders", HandlerFunc: GetSpiderList, }, { Method: http.MethodPost, Path: "", + Name: "Create Spider", + Description: "Create a new spider", HandlerFunc: PostSpider, }, { Method: http.MethodPut, Path: "/:id", + Name: "Update Spider by ID", + Description: "Update a spider by ID", HandlerFunc: PutSpiderById, }, { Method: http.MethodDelete, Path: "/:id", + Name: "Delete Spider by ID", + Description: "Delete a spider by ID", HandlerFunc: DeleteSpiderById, }, { Method: http.MethodDelete, Path: "", + Name: "Delete Spider List", + Description: "Delete a list of spiders", HandlerFunc: DeleteSpiderList, }, { Method: http.MethodGet, Path: "/:id/files/list", + Name: "Get Spider List Dir", + Description: "Get a list of files in a spider directory", HandlerFunc: GetSpiderListDir, }, { Method: http.MethodGet, Path: "/:id/files/get", + Name: "Get Spider File Content", + Description: "Get the content of a spider file", HandlerFunc: GetSpiderFileContent, }, { Method: http.MethodGet, Path: "/:id/files/info", + Name: "Get Spider File Info", + Description: "Get the info of a spider file", HandlerFunc: GetSpiderFileInfo, }, { Method: http.MethodPost, Path: "/:id/files/save", + Name: "Save Spider File", + Description: "Save a spider file", HandlerFunc: PostSpiderSaveFile, }, { Method: http.MethodPost, Path: "/:id/files/save/batch", + Name: "Save Spider Files", + Description: "Save multiple spider files", HandlerFunc: PostSpiderSaveFiles, }, { Method: http.MethodPost, Path: "/:id/files/save/dir", + Name: "Save Spider Dir", + Description: "Save a spider directory", HandlerFunc: PostSpiderSaveDir, }, { Method: http.MethodPost, Path: "/:id/files/rename", + Name: "Rename Spider File", + Description: "Rename a spider file", HandlerFunc: PostSpiderRenameFile, }, { Method: http.MethodDelete, Path: "/:id/files", + Name: "Delete Spider File", + Description: "Delete a spider file", HandlerFunc: DeleteSpiderFile, }, { Method: http.MethodPost, Path: "/:id/files/copy", + Name: "Copy Spider File", + Description: "Copy a spider file", HandlerFunc: PostSpiderCopyFile, }, { Method: http.MethodPost, Path: "/:id/files/export", + Name: "Export Spider Files", + Description: "Export spider files to a zip file", HandlerFunc: PostSpiderExport, }, { Method: http.MethodPost, Path: "/:id/run", + Name: "Run Spider", + Description: "Run a task for the given spider", HandlerFunc: PostSpiderRun, }, { Method: http.MethodGet, Path: "/:id/results", + Name: "Get Spider Results", + Description: "Get the results of a spider", HandlerFunc: GetSpiderResults, }, }...)) @@ -332,26 +387,36 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodPost, Path: "", + Name: "Create Schedule", + Description: "Create a new schedule", HandlerFunc: PostSchedule, }, { Method: http.MethodPut, Path: "/:id", + Name: "Update Schedule by ID", + Description: "Update a schedule by ID", HandlerFunc: PutScheduleById, }, { Method: http.MethodPost, Path: "/:id/enable", + Name: "Enable Schedule", + Description: "Enable a schedule", HandlerFunc: PostScheduleEnable, }, { Method: http.MethodPost, Path: "/:id/disable", + Name: "Disable Schedule", + Description: "Disable a schedule", HandlerFunc: PostScheduleDisable, }, { Method: http.MethodPost, Path: "/:id/run", + Name: "Run Schedule", + Description: "Run a schedule", HandlerFunc: PostScheduleRun, }, }...)) @@ -359,41 +424,57 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodGet, Path: "/:id", + Name: "Get Task by ID", + Description: "Get a single task by ID", HandlerFunc: GetTaskById, }, { Method: http.MethodGet, Path: "", + Name: "Get Task List", + Description: "Get a list of tasks", HandlerFunc: GetTaskList, }, { Method: http.MethodDelete, Path: "/:id", + Name: "Delete Task by ID", + Description: "Delete a task by ID", HandlerFunc: DeleteTaskById, }, { Method: http.MethodDelete, Path: "", - HandlerFunc: DeleteList, + Name: "Delete Task List", + Description: "Delete a list of tasks", + HandlerFunc: DeleteTaskList, }, { Method: http.MethodPost, Path: "/run", + Name: "Run Task", + Description: "Run a task", HandlerFunc: PostTaskRun, }, { Method: http.MethodPost, Path: "/:id/restart", + Name: "Restart Task", + Description: "Restart a task", HandlerFunc: PostTaskRestart, }, { Method: http.MethodPost, Path: "/:id/cancel", + Name: "Cancel Task", + Description: "Cancel a task", HandlerFunc: PostTaskCancel, }, { Method: http.MethodGet, Path: "/:id/logs", + Name: "Get Task Logs", + Description: "Get the logs of a task", HandlerFunc: GetTaskLogs, }, }...)) @@ -401,11 +482,15 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodPost, Path: "", + Name: "Create Token", + Description: "Create a new token", HandlerFunc: PostToken, }, { Method: http.MethodGet, Path: "", + Name: "Get Token List", + Description: "Get a list of tokens", HandlerFunc: GetTokenList, }, }...)) @@ -413,51 +498,71 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodGet, Path: "/:id", + Name: "Get User by ID", + Description: "Get a single user by ID", HandlerFunc: GetUserById, }, { Method: http.MethodGet, Path: "", + Name: "Get User List", + Description: "Get a list of users", HandlerFunc: GetUserList, }, { Method: http.MethodPost, Path: "", + Name: "Create User", + Description: "Create a new user", HandlerFunc: PostUser, }, { Method: http.MethodPut, Path: "/:id", + Name: "Update User by ID", + Description: "Update a user by ID", HandlerFunc: PutUserById, }, { Method: http.MethodPost, Path: "/:id/change-password", + Name: "Change User Password", + Description: "Change a user's password", HandlerFunc: PostUserChangePassword, }, { Method: http.MethodDelete, Path: "/:id", + Name: "Delete User by ID", + Description: "Delete a user by ID", HandlerFunc: DeleteUserById, }, { Method: http.MethodDelete, Path: "", + Name: "Delete User List", + Description: "Delete a list of users", HandlerFunc: DeleteUserList, }, { Method: http.MethodGet, Path: "/me", + Name: "Get Me", + Description: "Get the current user", HandlerFunc: GetUserMe, }, { Method: http.MethodPut, Path: "/me", + Name: "Update Me", + Description: "Update the current user", HandlerFunc: PutUserMe, }, { Method: http.MethodPost, Path: "/me/change-password", + Name: "Change My Password", + Description: "Change the current user's password", HandlerFunc: PostUserMeChangePassword, }, }...)) @@ -467,16 +572,22 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodPost, Path: "/:type", + Name: "Export Data", + Description: "Export data", HandlerFunc: PostExport, }, { Method: http.MethodGet, Path: "/:type/:id", + Name: "Get Export", + Description: "Get an export", HandlerFunc: GetExport, }, { Method: http.MethodGet, Path: "/:type/:id/download", + Name: "Get Export Download", + Description: "Get an export download", HandlerFunc: GetExportDownload, }, }) @@ -484,16 +595,22 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodGet, Path: "/:col", + Name: "Get Filter Column Field Options", + Description: "Get the field options of a collection", HandlerFunc: GetFilterColFieldOptions, }, { Method: http.MethodGet, Path: "/:col/:value", + Name: "Get Filter Col Field Options With Value", + Description: "Get the field options of a collection with a value", HandlerFunc: GetFilterColFieldOptionsWithValue, }, { Method: http.MethodGet, Path: "/:col/:value/:label", + Name: "Get Filter Col Field Options With Value And Label", + Description: "Get the field options of a collection with a value and label", HandlerFunc: GetFilterColFieldOptionsWithValueLabel, }, }) @@ -501,16 +618,22 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodGet, Path: "/:key", + Name: "Get Setting", + Description: "Get a setting", HandlerFunc: GetSetting, }, { Method: http.MethodPost, Path: "/:key", + Name: "Create Setting", + Description: "Create a new setting", HandlerFunc: PostSetting, }, { Method: http.MethodPut, Path: "/:key", + Name: "Update Setting", + Description: "Update a setting", HandlerFunc: PutSetting, }, }) @@ -518,16 +641,22 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodGet, Path: "/overview", + Name: "Get Stats Overview", + Description: "Get the overview of the stats", HandlerFunc: GetStatsOverview, }, { Method: http.MethodGet, Path: "/daily", + Name: "Get Stats Daily", + Description: "Get the daily stats", HandlerFunc: GetStatsDaily, }, { Method: http.MethodGet, Path: "/tasks", + Name: "Get Stats Tasks", + Description: "Get the tasks stats", HandlerFunc: GetStatsTasks, }, }) @@ -537,6 +666,8 @@ func InitRoutes(app *gin.Engine) (err error) { { Path: "", Method: http.MethodGet, + Name: "Get System Info", + Description: "Get the system info", HandlerFunc: GetSystemInfo, }, }) @@ -544,11 +675,15 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodPost, Path: "/login", + Name: "Login", + Description: "Login", HandlerFunc: PostLogin, }, { Method: http.MethodPost, Path: "/logout", + Name: "Logout", + Description: "Logout", HandlerFunc: PostLogout, }, }) diff --git a/core/controllers/schedule.go b/core/controllers/schedule.go index c98d6f95..bb29f79b 100644 --- a/core/controllers/schedule.go +++ b/core/controllers/schedule.go @@ -14,7 +14,7 @@ import ( ) type PostScheduleParams struct { - Data models.Schedule `json:"data"` + Data models.Schedule `json:"data" description:"The data to create" validate:"required"` } func PostSchedule(c *gin.Context, params *PostScheduleParams) (response *Response[models.Schedule], err error) { @@ -42,8 +42,8 @@ func PostSchedule(c *gin.Context, params *PostScheduleParams) (response *Respons } type PutScheduleByIdParams struct { - Id string `path:"id"` - Data models.Schedule `json:"data"` + Id string `path:"id" description:"Schedule ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Data models.Schedule `json:"data" description:"The data to update" validate:"required"` } func PutScheduleById(c *gin.Context, params *PutScheduleByIdParams) (response *Response[models.Schedule], err error) { @@ -80,7 +80,7 @@ func PutScheduleById(c *gin.Context, params *PutScheduleByIdParams) (response *R } type PostScheduleEnableDisableParams struct { - Id string `path:"id"` + Id string `path:"id" description:"Schedule ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` } func PostScheduleEnable(c *gin.Context, params *PostScheduleEnableDisableParams) (response *VoidResponse, err error) { @@ -118,12 +118,12 @@ func postScheduleEnableDisableFunc(isEnable bool, userId primitive.ObjectID, par } type PostScheduleRunParams struct { - Id string `path:"id"` - Mode string `json:"mode"` - NodeIds []string `json:"node_ids"` - Cmd string `json:"cmd"` - Param string `json:"param"` - Priority int `json:"priority"` + Id string `path:"id" description:"Schedule ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Mode string `json:"mode" description:"Run mode" enum:"random,all,selected-nodes"` + NodeIds []string `json:"node_ids" description:"Node IDs" items.type:"string" items.format:"objectid" items.pattern:"^[0-9a-fA-F]{24}$"` + Cmd string `json:"cmd" description:"Command"` + Param string `json:"param" description:"Parameters"` + Priority int `json:"priority" description:"Priority" default:"5" minimum:"1" maximum:"10"` } func PostScheduleRun(c *gin.Context, params *PostScheduleRunParams) (response *Response[[]primitive.ObjectID], err error) { diff --git a/core/controllers/setting.go b/core/controllers/setting.go index f04bb56b..a29ce468 100644 --- a/core/controllers/setting.go +++ b/core/controllers/setting.go @@ -9,7 +9,7 @@ import ( ) type GetSettingParams struct { - Key string `path:"key" validate:"required"` + Key string `path:"key" description:"Setting key" validate:"required"` } func GetSetting(_ *gin.Context, params *GetSettingParams) (response *Response[models.Setting], err error) { @@ -26,8 +26,8 @@ func GetSetting(_ *gin.Context, params *GetSettingParams) (response *Response[mo } type PostSettingParams struct { - Key string `path:"key" validate:"required"` - Data models.Setting `json:"data"` + Key string `path:"key" description:"Setting key" validate:"required"` + Data models.Setting `json:"data" description:"The data to create" validate:"required"` } func PostSetting(c *gin.Context, params *PostSettingParams) (response *Response[models.Setting], err error) { @@ -50,8 +50,8 @@ func PostSetting(c *gin.Context, params *PostSettingParams) (response *Response[ } type PutSettingParams struct { - Key string `path:"key" validate:"required"` - Data models.Setting `json:"data"` + Key string `path:"key" description:"Setting key" validate:"required"` + Data models.Setting `json:"data" description:"The data to update" validate:"required"` } func PutSetting(c *gin.Context, params *PutSettingParams) (response *Response[models.Setting], err error) { diff --git a/core/controllers/spider.go b/core/controllers/spider.go index a47e5ca0..b41e7eb5 100644 --- a/core/controllers/spider.go +++ b/core/controllers/spider.go @@ -535,8 +535,8 @@ func DeleteSpiderList(_ *gin.Context, params *DeleteSpiderListParams) (response } type GetSpiderListDirParams struct { - Id string `path:"id"` - Path string `query:"path"` + Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Path string `query:"path" description:"Directory path"` } func GetSpiderListDir(c *gin.Context, params *GetSpiderListDirParams) (response *Response[[]interfaces.FsFileInfo], err error) { @@ -548,8 +548,8 @@ func GetSpiderListDir(c *gin.Context, params *GetSpiderListDirParams) (response } type GetSpiderFileContentParams struct { - Id string `path:"id"` - Path string `query:"path"` + Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Path string `query:"path" description:"File path"` } func GetSpiderFileContent(c *gin.Context, params *GetSpiderFileContentParams) (response *Response[string], err error) { @@ -561,8 +561,8 @@ func GetSpiderFileContent(c *gin.Context, params *GetSpiderFileContentParams) (r } type GetSpiderFileInfoParams struct { - Id string `path:"id"` - Path string `query:"path"` + Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Path string `query:"path" description:"File path"` } func GetSpiderFileInfo(c *gin.Context, params *GetSpiderFileInfoParams) (response *Response[interfaces.FsFileInfo], err error) { @@ -574,9 +574,9 @@ func GetSpiderFileInfo(c *gin.Context, params *GetSpiderFileInfoParams) (respons } type PostSpiderSaveFileParams struct { - Id string `path:"id"` - Path string `json:"path"` - Data string `json:"data"` + Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Path string `json:"path" description:"File path to save"` + Data string `json:"data" description:"File content"` File *multipart.FileHeader `form:"file"` } @@ -593,8 +593,8 @@ func PostSpiderSaveFile(c *gin.Context, params *PostSpiderSaveFileParams) (respo } type PostSpiderSaveFilesParams struct { - Id string `path:"id"` - TargetDirectory string `form:"targetDirectory"` + Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + TargetDirectory string `form:"targetDirectory" description:"Target directory path"` } func PostSpiderSaveFiles(c *gin.Context, params *PostSpiderSaveFilesParams) (response *VoidResponse, err error) { @@ -610,8 +610,8 @@ func PostSpiderSaveFiles(c *gin.Context, params *PostSpiderSaveFilesParams) (res } type PostSpiderSaveDirParams struct { - Id string `path:"id"` - Path string `json:"path"` + Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Path string `json:"path" description:"File path to save"` } func PostSpiderSaveDir(c *gin.Context, params *PostSpiderSaveDirParams) (response *VoidResponse, err error) { @@ -623,9 +623,9 @@ func PostSpiderSaveDir(c *gin.Context, params *PostSpiderSaveDirParams) (respons } type PostSpiderRenameFileParams struct { - Id string `path:"id"` - Path string `json:"path"` - NewPath string `json:"newPath"` + Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Path string `json:"path" description:"File path to rename"` + NewPath string `json:"newPath" description:"New file path"` } func PostSpiderRenameFile(c *gin.Context, params *PostSpiderRenameFileParams) (response *VoidResponse, err error) { @@ -637,8 +637,8 @@ func PostSpiderRenameFile(c *gin.Context, params *PostSpiderRenameFileParams) (r } type DeleteSpiderFileParams struct { - Id string `path:"id"` - Path string `json:"path"` + Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Path string `json:"path" description:"File path to delete"` } func DeleteSpiderFile(c *gin.Context, params *DeleteSpiderFileParams) (response *VoidResponse, err error) { @@ -650,9 +650,9 @@ func DeleteSpiderFile(c *gin.Context, params *DeleteSpiderFileParams) (response } type PostSpiderCopyFileParams struct { - Id string `path:"id"` - Path string `json:"path"` - NewPath string `json:"new_path"` + Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Path string `json:"path" description:"File path to copy"` + NewPath string `json:"new_path" description:"New file path"` } func PostSpiderCopyFile(c *gin.Context, params *PostSpiderCopyFileParams) (response *VoidResponse, err error) { @@ -664,7 +664,7 @@ func PostSpiderCopyFile(c *gin.Context, params *PostSpiderCopyFileParams) (respo } type PostSpiderExportParams struct { - Id string `path:"id"` + Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` } func PostSpiderExport(c *gin.Context, _ *PostSpiderExportParams) (err error) { @@ -676,13 +676,13 @@ func PostSpiderExport(c *gin.Context, _ *PostSpiderExportParams) (err error) { } type PostSpiderRunParams struct { - Id string `path:"id"` - Mode string `json:"mode"` - NodeIds []string `json:"node_ids"` - Cmd string `json:"cmd"` - Param string `json:"param"` - ScheduleId string `json:"schedule_id"` - Priority int `json:"priority"` + Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Mode string `json:"mode" description:"Run mode" enum:"random,all,selected-nodes"` + NodeIds []string `json:"node_ids" description:"Node IDs, used in selected-nodes mode"` + Cmd string `json:"cmd" description:"Command"` + Param string `json:"param" description:"Parameters"` + ScheduleId string `json:"schedule_id" description:"Schedule ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Priority int `json:"priority" description:"Priority" default:"5" minimum:"1" maximum:"10"` } func PostSpiderRun(c *gin.Context, params *PostSpiderRunParams) (response *Response[[]primitive.ObjectID], err error) { @@ -733,9 +733,9 @@ func PostSpiderRun(c *gin.Context, params *PostSpiderRunParams) (response *Respo } type GetSpiderResultsParams struct { - Id string `path:"id"` - Page int `query:"page"` - Size int `query:"size"` + Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Page int `query:"page" description:"Page" default:"1" minimum:"1"` + Size int `query:"size" description:"Size" default:"10" minimum:"1"` } func GetSpiderResults(c *gin.Context, params *GetSpiderResultsParams) (response *ListResponse[bson.M], err error) { diff --git a/core/controllers/stats.go b/core/controllers/stats.go index 14a64f4e..019b6667 100644 --- a/core/controllers/stats.go +++ b/core/controllers/stats.go @@ -15,7 +15,7 @@ var statsDefaultQuery = bson.M{ } type GetStatsOverviewParams struct { - Query bson.M `json:"query"` + Query bson.M `json:"query" description:"Query"` } func GetStatsOverview(_ *gin.Context, params *GetStatsOverviewParams) (response *Response[bson.M], err error) { @@ -32,7 +32,7 @@ func GetStatsOverview(_ *gin.Context, params *GetStatsOverviewParams) (response } type GetStatsDailyParams struct { - Query bson.M `json:"query"` + Query bson.M `json:"query" description:"Query"` } func GetStatsDaily(_ *gin.Context, params *GetStatsDailyParams) (response *Response[bson.M], err error) { @@ -49,7 +49,7 @@ func GetStatsDaily(_ *gin.Context, params *GetStatsDailyParams) (response *Respo } type GetStatsTasksParams struct { - Query bson.M `json:"query"` + Query bson.M `json:"query" description:"Query"` } func GetStatsTasks(_ *gin.Context, params *GetStatsTasksParams) (response *Response[bson.M], err error) { diff --git a/core/controllers/task.go b/core/controllers/task.go index d772b457..ea9cfe80 100644 --- a/core/controllers/task.go +++ b/core/controllers/task.go @@ -23,7 +23,7 @@ import ( ) type GetTaskByIdParams struct { - Id string `path:"id"` + Id string `path:"id" description:"Task ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` } func GetTaskById(_ *gin.Context, params *GetTaskByIdParams) (response *Response[models.Task], err error) { @@ -172,7 +172,7 @@ func GetTaskList(c *gin.Context, params *GetTaskListParams) (response *ListRespo } type DeleteTaskByIdParams struct { - Id string `path:"id"` + Id string `path:"id" description:"Task ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` } func DeleteTaskById(_ *gin.Context, params *DeleteTaskByIdParams) (response *VoidResponse, err error) { @@ -223,7 +223,7 @@ type DeleteTaskListParams struct { Ids []string `json:"ids"` } -func DeleteList(_ *gin.Context, params *DeleteTaskListParams) (response *VoidResponse, err error) { +func DeleteTaskList(_ *gin.Context, params *DeleteTaskListParams) (response *VoidResponse, err error) { var ids []primitive.ObjectID for _, id := range params.Ids { id, err := primitive.ObjectIDFromHex(id) @@ -333,7 +333,7 @@ func PostTaskRun(c *gin.Context, params *PostTaskRunParams) (response *Response[ } type PostTaskRestartParams struct { - Id string `path:"id"` + Id string `path:"id" description:"Task ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` } func PostTaskRestart(c *gin.Context, params *PostTaskRestartParams) (response *Response[[]primitive.ObjectID], err error) { @@ -380,8 +380,8 @@ func PostTaskRestart(c *gin.Context, params *PostTaskRestartParams) (response *R } type PostTaskCancelParams struct { - Id string `path:"id"` - Force bool `json:"force,omitempty"` + Id string `path:"id" description:"Task ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Force bool `json:"force,omitempty" description:"Force cancel" default:"false"` } func PostTaskCancel(c *gin.Context, params *PostTaskCancelParams) (response *VoidResponse, err error) { @@ -415,9 +415,9 @@ func PostTaskCancel(c *gin.Context, params *PostTaskCancelParams) (response *Voi } type GetTaskLogsParams struct { - Id string `path:"id"` - Page int `query:"page"` - Size int `query:"size"` + Id string `path:"id" description:"Task ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Page int `query:"page" description:"Page" default:"1" minimum:"1"` + Size int `query:"size" description:"Size" default:"10" minimum:"1"` } func GetTaskLogs(_ *gin.Context, params *GetTaskLogsParams) (response *ListResponse[string], err error) { diff --git a/core/controllers/token.go b/core/controllers/token.go index 4c052e02..44a05913 100644 --- a/core/controllers/token.go +++ b/core/controllers/token.go @@ -11,7 +11,7 @@ import ( ) type PostTokenParams struct { - Data models.Token `json:"data"` + Data models.Token `json:"data" description:"The data to create" validate:"required"` } func PostToken(c *gin.Context, params *PostTokenParams) (response *Response[models.Token], err error) { diff --git a/core/controllers/user.go b/core/controllers/user.go index 9dd55664..4c40ad2d 100644 --- a/core/controllers/user.go +++ b/core/controllers/user.go @@ -89,11 +89,11 @@ func GetUserList(_ *gin.Context, params *GetListParams) (response *ListResponse[ } type PostUserParams struct { - Username string `json:"username" validate:"required"` - Password string `json:"password" validate:"required"` - Role string `json:"role"` - RoleId primitive.ObjectID `json:"role_id"` - Email string `json:"email"` + Username string `json:"username" description:"Username" validate:"required"` + Password string `json:"password" description:"Password" validate:"required"` + Role string `json:"role" description:"Role"` + RoleId string `json:"role_id" description:"Role ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Email string `json:"email" description:"Email"` } func PostUser(c *gin.Context, params *PostUserParams) (response *Response[models.User], err error) { @@ -105,8 +105,13 @@ func PostUser(c *gin.Context, params *PostUserParams) (response *Response[models } } - if !params.RoleId.IsZero() { - _, err := service.NewModelService[models.Role]().GetById(params.RoleId) + var roleId primitive.ObjectID + if params.RoleId != "" { + roleId, err = primitive.ObjectIDFromHex(params.RoleId) + if err != nil { + return GetErrorResponse[models.User](errors.BadRequestf("invalid role id: %v", err)) + } + _, err = service.NewModelService[models.Role]().GetById(roleId) if err != nil { return GetErrorResponse[models.User](errors.BadRequestf("role not found: %v", err)) } @@ -116,7 +121,7 @@ func PostUser(c *gin.Context, params *PostUserParams) (response *Response[models Username: params.Username, Password: utils.EncryptMd5(params.Password), Role: params.Role, - RoleId: params.RoleId, + RoleId: roleId, Email: params.Email, } model.SetCreated(u.Id) @@ -143,12 +148,12 @@ func PutUserById(c *gin.Context, params *PutByIdParams[models.User]) (response * } type PostUserChangePasswordParams struct { - Id string `path:"id"` - Password string `json:"password" validate:"required"` + Id string `path:"id" description:"User ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Password string `json:"password" description:"Password" validate:"required"` } func PostUserChangePassword(c *gin.Context, params *PostUserChangePasswordParams) (response *Response[models.User], err error) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { return GetErrorResponse[models.User](errors.BadRequestf("invalid user id: %v", err)) } @@ -228,7 +233,7 @@ func PutUserMe(c *gin.Context, params *PutUserMeParams) (response *Response[mode } type PostUserMeChangePasswordParams struct { - Password string `json:"password" validate:"required"` + Password string `json:"password" description:"Password" validate:"required"` } func PostUserMeChangePassword(c *gin.Context, params *PostUserMeChangePasswordParams) (response *Response[models.User], err error) { diff --git a/core/models/models/base.go b/core/models/models/base.go index 96f278ed..ce13a62b 100644 --- a/core/models/models/base.go +++ b/core/models/models/base.go @@ -8,11 +8,11 @@ import ( ) type BaseModel struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - CreatedAt time.Time `json:"created_ts,omitempty" bson:"created_ts,omitempty"` - CreatedBy primitive.ObjectID `json:"created_by,omitempty" bson:"created_by,omitempty"` - UpdatedAt time.Time `json:"updated_ts,omitempty" bson:"updated_ts,omitempty"` - UpdatedBy primitive.ObjectID `json:"updated_by,omitempty" bson:"updated_by,omitempty"` + Id primitive.ObjectID `json:"_id" bson:"_id" description:"ID"` + CreatedAt time.Time `json:"created_ts,omitempty" bson:"created_ts,omitempty" description:"Created timestamp"` + CreatedBy primitive.ObjectID `json:"created_by,omitempty" bson:"created_by,omitempty" description:"Created by"` + UpdatedAt time.Time `json:"updated_ts,omitempty" bson:"updated_ts,omitempty" description:"Updated timestamp"` + UpdatedBy primitive.ObjectID `json:"updated_by,omitempty" bson:"updated_by,omitempty" description:"Updated by"` } func (m *BaseModel) GetId() primitive.ObjectID { diff --git a/core/models/models/database.go b/core/models/models/database.go index a4faa23e..a8b9c3a7 100644 --- a/core/models/models/database.go +++ b/core/models/models/database.go @@ -7,42 +7,42 @@ import ( type Database struct { any `collection:"databases"` BaseModel `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - DataSource string `json:"data_source" bson:"data_source"` - Host string `json:"host" bson:"host"` - Port int `json:"port" bson:"port"` - URI string `json:"uri,omitempty" bson:"uri,omitempty"` - Database string `json:"database,omitempty" bson:"database,omitempty"` - Username string `json:"username,omitempty" bson:"username,omitempty"` - Password string `json:"password,omitempty" bson:"-"` - EncryptedPassword string `json:"-,omitempty" bson:"encrypted_password,omitempty"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Active bool `json:"active" bson:"active"` - ActiveAt time.Time `json:"active_ts" bson:"active_ts"` - IsDefault bool `json:"is_default" bson:"-"` + Name string `json:"name" bson:"name" description:"Name"` + Description string `json:"description" bson:"description" description:"Description"` + DataSource string `json:"data_source" bson:"data_source" description:"Data source"` + Host string `json:"host" bson:"host" description:"Host"` + Port int `json:"port" bson:"port" description:"Port"` + URI string `json:"uri,omitempty" bson:"uri,omitempty" description:"URI"` + Database string `json:"database,omitempty" bson:"database,omitempty" description:"Database"` + Username string `json:"username,omitempty" bson:"username,omitempty" description:"Username"` + Password string `json:"password,omitempty" bson:"-" binding:"-"` + EncryptedPassword string `json:"-,omitempty" bson:"encrypted_password,omitempty" description:"Encrypted password"` + Status string `json:"status" bson:"status" description:"Status"` + Error string `json:"error" bson:"error" description:"Error"` + Active bool `json:"active" bson:"active" description:"Active"` + ActiveAt time.Time `json:"active_ts" bson:"active_ts" description:"Active at"` + IsDefault bool `json:"is_default" bson:"-" binding:"-"` MongoParams *struct { - AuthSource string `json:"auth_source,omitempty" bson:"auth_source,omitempty"` - AuthMechanism string `json:"auth_mechanism,omitempty" bson:"auth_mechanism,omitempty"` - } `json:"mongo_params,omitempty" bson:"mongo_params,omitempty"` + AuthSource string `json:"auth_source,omitempty" bson:"auth_source,omitempty" description:"Auth source"` + AuthMechanism string `json:"auth_mechanism,omitempty" bson:"auth_mechanism,omitempty" description:"Auth mechanism"` + } `json:"mongo_params,omitempty" bson:"mongo_params,omitempty" description:"Mongo params"` PostgresParams *struct { - SSLMode string `json:"ssl_mode,omitempty" bson:"ssl_mode,omitempty"` - } `json:"postgres_params,omitempty" bson:"postgres_params,omitempty"` + SSLMode string `json:"ssl_mode,omitempty" bson:"ssl_mode,omitempty" description:"SSL mode"` + } `json:"postgres_params,omitempty" bson:"postgres_params,omitempty" description:"Postgres params"` SnowflakeParams *struct { - Account string `json:"account,omitempty" bson:"account,omitempty"` - Schema string `json:"schema,omitempty" bson:"schema,omitempty"` - Warehouse string `json:"warehouse,omitempty" bson:"warehouse,omitempty"` - Role string `json:"role,omitempty" bson:"role,omitempty"` - } `json:"snowflake_params,omitempty" bson:"snowflake_params,omitempty"` + Account string `json:"account,omitempty" bson:"account,omitempty" description:"Account"` + Schema string `json:"schema,omitempty" bson:"schema,omitempty" description:"Schema"` + Warehouse string `json:"warehouse,omitempty" bson:"warehouse,omitempty" description:"Warehouse"` + Role string `json:"role,omitempty" bson:"role,omitempty" description:"Role"` + } `json:"snowflake_params,omitempty" bson:"snowflake_params,omitempty" description:"Snowflake params"` CassandraParams *struct { - Keyspace string `json:"keyspace,omitempty" bson:"keyspace,omitempty"` - } `json:"cassandra_params,omitempty" bson:"cassandra_params,omitempty"` + Keyspace string `json:"keyspace,omitempty" bson:"keyspace,omitempty" description:"Keyspace"` + } `json:"cassandra_params,omitempty" bson:"cassandra_params,omitempty" description:"Cassandra params"` HiveParams *struct { - Auth string `json:"auth,omitempty" bson:"auth,omitempty"` - } `json:"hive_params,omitempty" bson:"hive_params,omitempty"` + Auth string `json:"auth,omitempty" bson:"auth,omitempty" description:"Auth"` + } `json:"hive_params,omitempty" bson:"hive_params,omitempty" description:"Hive params"` RedisParams *struct { - DB int `json:"db,omitempty" bson:"db,omitempty"` - } `json:"redis_params,omitempty" bson:"redis_params,omitempty"` + DB int `json:"db,omitempty" bson:"db,omitempty" description:"DB"` + } `json:"redis_params,omitempty" bson:"redis_params,omitempty" description:"Redis params"` } diff --git a/core/models/models/database_metric.go b/core/models/models/database_metric.go index 689d0365..004e4995 100644 --- a/core/models/models/database_metric.go +++ b/core/models/models/database_metric.go @@ -5,20 +5,20 @@ import "go.mongodb.org/mongo-driver/bson/primitive" type DatabaseMetric struct { any `collection:"database_metrics"` BaseModel `bson:",inline"` - DatabaseId primitive.ObjectID `json:"database_id" bson:"database_id"` - CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` - TotalMemory uint64 `json:"total_memory" bson:"total_memory"` - AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` - UsedMemory uint64 `json:"used_memory" bson:"used_memory"` - UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` - TotalDisk uint64 `json:"total_disk" bson:"total_disk"` - AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` - UsedDisk uint64 `json:"used_disk" bson:"used_disk"` - UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` - Connections int `json:"connections" bson:"connections"` - QueryPerSecond float64 `json:"query_per_second" bson:"query_per_second"` - TotalQuery uint64 `json:"total_query,omitempty" bson:"total_query,omitempty"` - CacheHitRatio float64 `json:"cache_hit_ratio" bson:"cache_hit_ratio"` - ReplicationLag float64 `json:"replication_lag" bson:"replication_lag"` - LockWaitTime float64 `json:"lock_wait_time" bson:"lock_wait_time"` + DatabaseId primitive.ObjectID `json:"database_id" bson:"database_id" description:"Database ID"` + CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent" description:"CPU usage percentage"` + TotalMemory uint64 `json:"total_memory" bson:"total_memory" description:"Total memory"` + AvailableMemory uint64 `json:"available_memory" bson:"available_memory" description:"Available memory"` + UsedMemory uint64 `json:"used_memory" bson:"used_memory" description:"Used memory"` + UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent" description:"Used memory percentage"` + TotalDisk uint64 `json:"total_disk" bson:"total_disk" description:"Total disk"` + AvailableDisk uint64 `json:"available_disk" bson:"available_disk" description:"Available disk"` + UsedDisk uint64 `json:"used_disk" bson:"used_disk" description:"Used disk"` + UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent" description:"Used disk percentage"` + Connections int `json:"connections" bson:"connections" description:"Connections"` + QueryPerSecond float64 `json:"query_per_second" bson:"query_per_second" description:"Query per second"` + TotalQuery uint64 `json:"total_query,omitempty" bson:"total_query,omitempty" description:"Total query"` + CacheHitRatio float64 `json:"cache_hit_ratio" bson:"cache_hit_ratio" description:"Cache hit ratio"` + ReplicationLag float64 `json:"replication_lag" bson:"replication_lag" description:"Replication lag"` + LockWaitTime float64 `json:"lock_wait_time" bson:"lock_wait_time" description:"Lock wait time"` } diff --git a/core/models/models/dependency.go b/core/models/models/dependency.go index 6514bc20..908264f6 100644 --- a/core/models/models/dependency.go +++ b/core/models/models/dependency.go @@ -7,13 +7,13 @@ import ( type Dependency struct { any `collection:"dependencies"` BaseModel `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - Type string `json:"type" bson:"type"` - Version string `json:"version" bson:"version"` - Status string `json:"status" bson:"status"` - Error string `json:"error,omitempty" bson:"error,omitempty"` - NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-"` - Versions []string `json:"versions,omitempty" bson:"-"` + Name string `json:"name" bson:"name" description:"Name"` + Description string `json:"description" bson:"description" description:"Description"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id" description:"Node ID"` + Type string `json:"type" bson:"type" description:"Type"` + Version string `json:"version" bson:"version" description:"Version"` + Status string `json:"status" bson:"status" description:"Status"` + Error string `json:"error,omitempty" bson:"error,omitempty" description:"Error"` + NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-" binding:"-"` + Versions []string `json:"versions,omitempty" bson:"-" binding:"-"` } diff --git a/core/models/models/dependency_config.go b/core/models/models/dependency_config.go index 29c4c8b6..20701361 100644 --- a/core/models/models/dependency_config.go +++ b/core/models/models/dependency_config.go @@ -3,12 +3,12 @@ package models type DependencyConfig struct { any `collection:"dependency_configs"` BaseModel `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - ExecCmd string `json:"exec_cmd" bson:"exec_cmd"` - PkgCmd string `json:"pkg_cmd" bson:"pkg_cmd"` - PkgSrcURL string `json:"pkg_src_url" bson:"pkg_src_url"` - Setup bool `json:"setup" bson:"-"` - TotalDependencies int `json:"total_dependencies" bson:"-"` - SearchReady bool `json:"search_ready" bson:"-"` + Key string `json:"key" bson:"key" description:"Key"` + Name string `json:"name" bson:"name" description:"Name"` + ExecCmd string `json:"exec_cmd" bson:"exec_cmd" description:"Exec cmd"` + PkgCmd string `json:"pkg_cmd" bson:"pkg_cmd" description:"Pkg cmd"` + PkgSrcURL string `json:"pkg_src_url" bson:"pkg_src_url" description:"Pkg src url"` + Setup bool `json:"setup" bson:"-" binding:"-"` + TotalDependencies int `json:"total_dependencies" bson:"-" binding:"-"` + SearchReady bool `json:"search_ready" bson:"-" binding:"-"` } diff --git a/core/models/models/dependency_config_setup.go b/core/models/models/dependency_config_setup.go index 386141ae..8d74cdc2 100644 --- a/core/models/models/dependency_config_setup.go +++ b/core/models/models/dependency_config_setup.go @@ -5,15 +5,15 @@ import "go.mongodb.org/mongo-driver/bson/primitive" type DependencyConfigSetup struct { any `collection:"dependency_config_setups"` BaseModel `bson:",inline"` - DependencyConfigId primitive.ObjectID `json:"dependency_config_id" bson:"dependency_config_id"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - Version string `json:"version" bson:"version"` - Drivers []DependencyDriver `json:"versions,omitempty" bson:"versions,omitempty"` - Status string `json:"status" bson:"status"` - Error string `json:"error,omitempty" bson:"error,omitempty"` - Node *Node `json:"node,omitempty" bson:"-"` + DependencyConfigId primitive.ObjectID `json:"dependency_config_id" bson:"dependency_config_id" description:"Dependency config ID"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id" description:"Node ID"` + Version string `json:"version" bson:"version" description:"Version"` + Drivers []DependencyDriver `json:"versions,omitempty" bson:"versions,omitempty" description:"Versions"` + Status string `json:"status" bson:"status" description:"Status"` + Error string `json:"error,omitempty" bson:"error,omitempty" description:"Error"` + Node *Node `json:"node,omitempty" bson:"-" binding:"-"` } type DependencyDriver struct { - Name string `json:"name" bson:"name"` - Version string `json:"version" bson:"version"` + Name string `json:"name" bson:"name" description:"Name"` + Version string `json:"version" bson:"version" description:"Version"` } diff --git a/core/models/models/dependency_log.go b/core/models/models/dependency_log.go index e4e13976..96a2d1cc 100644 --- a/core/models/models/dependency_log.go +++ b/core/models/models/dependency_log.go @@ -5,6 +5,6 @@ import "go.mongodb.org/mongo-driver/bson/primitive" type DependencyLog struct { any `collection:"dependency_logs"` BaseModel `bson:",inline"` - TargetId primitive.ObjectID `json:"target_id" bson:"target_id"` - Content string `json:"content" bson:"content"` + TargetId primitive.ObjectID `json:"target_id" bson:"target_id" description:"Target ID"` + Content string `json:"content" bson:"content" description:"Content"` } diff --git a/core/models/models/dependency_pypi_project.go b/core/models/models/dependency_pypi_project.go index ec3947a3..9957a10c 100644 --- a/core/models/models/dependency_pypi_project.go +++ b/core/models/models/dependency_pypi_project.go @@ -3,7 +3,7 @@ package models type DependencyPypiProject struct { any `collection:"dependency_pypi_projects"` BaseModel `bson:",inline"` - Name string `json:"name" bson:"name"` - Version string `json:"version" bson:"version"` - LastSerial int `json:"_last-serial" bson:"last_serial"` + Name string `json:"name" bson:"name" description:"Name"` + Version string `json:"version" bson:"version" description:"Version"` + LastSerial int `json:"_last-serial" bson:"last_serial" description:"Last serial"` } diff --git a/core/models/models/dependency_repo.go b/core/models/models/dependency_repo.go index d3ca1263..78d5a0d9 100644 --- a/core/models/models/dependency_repo.go +++ b/core/models/models/dependency_repo.go @@ -3,8 +3,8 @@ package models type DependencyRepo struct { any `collection:"dependency_repos"` BaseModel `bson:",inline"` - Name string `json:"name" bson:"name"` - Type string `json:"type" bson:"type"` - LatestVersion string `json:"latest_version" bson:"latest_version"` - AllVersions []string `json:"all_versions" bson:"all_versions"` + Name string `json:"name" bson:"name" description:"Name"` + Type string `json:"type" bson:"type" description:"Type"` + LatestVersion string `json:"latest_version" bson:"latest_version" description:"Latest version"` + AllVersions []string `json:"all_versions" bson:"all_versions" description:"All versions"` } diff --git a/core/models/models/environment.go b/core/models/models/environment.go index 3faf8547..3db558a4 100644 --- a/core/models/models/environment.go +++ b/core/models/models/environment.go @@ -3,6 +3,6 @@ package models type Environment struct { any `collection:"environments"` BaseModel `bson:",inline"` - Key string `json:"key" bson:"key"` - Value string `json:"value" bson:"value"` + Key string `json:"key" bson:"key" description:"Key"` + Value string `json:"value" bson:"value" description:"Value"` } diff --git a/core/models/models/git.go b/core/models/models/git.go index c5babffd..b12c2636 100644 --- a/core/models/models/git.go +++ b/core/models/models/git.go @@ -1,26 +1,27 @@ package models import ( - "github.com/crawlab-team/crawlab/vcs" "time" + + "github.com/crawlab-team/crawlab/vcs" ) type Git struct { any `collection:"gits"` BaseModel `bson:",inline"` - Url string `json:"url" bson:"url"` - Name string `json:"name" bson:"name"` - AuthType string `json:"auth_type" bson:"auth_type"` - Username string `json:"username" bson:"username"` - Password string `json:"password" bson:"password"` - CurrentBranch string `json:"current_branch" bson:"current_branch"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Spiders []Spider `json:"spiders,omitempty" bson:"-"` - Refs []vcs.GitRef `json:"refs" bson:"refs"` - RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"` - CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"` + Url string `json:"url" bson:"url" description:"URL"` + Name string `json:"name" bson:"name" description:"Name"` + AuthType string `json:"auth_type" bson:"auth_type" description:"Auth type"` + Username string `json:"username" bson:"username" description:"Username"` + Password string `json:"password" bson:"password" description:"Password"` + CurrentBranch string `json:"current_branch" bson:"current_branch" description:"Current branch"` + Status string `json:"status" bson:"status" description:"Status"` + Error string `json:"error" bson:"error" description:"Error"` + Spiders []Spider `json:"spiders,omitempty" bson:"-" binding:"-"` + Refs []vcs.GitRef `json:"refs" bson:"refs" description:"Refs"` + RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at" description:"Refs updated at"` + CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs" description:"Clone logs"` // settings - AutoPull bool `json:"auto_pull" bson:"auto_pull"` + AutoPull bool `json:"auto_pull" bson:"auto_pull" description:"Auto pull"` } diff --git a/core/models/models/llm_provider.go b/core/models/models/llm_provider.go index d723eb91..e27ad8d3 100644 --- a/core/models/models/llm_provider.go +++ b/core/models/models/llm_provider.go @@ -4,15 +4,15 @@ package models type LLMProvider struct { any `collection:"llm_providers"` BaseModel `bson:",inline"` - Key string `json:"key" bson:"key"` // Provider key (e.g., "openai", "anthropic", "gemini") - Name string `json:"name" bson:"name"` // Display name for UI - Enabled bool `json:"enabled" bson:"enabled"` // Whether this provider is enabled - ApiKey string `json:"api_key" bson:"api_key"` // API key for the provider - ApiBaseUrl string `json:"api_base_url" bson:"api_base_url"` // API base URL for the provider - DeploymentName string `json:"deployment_name" bson:"deployment_name"` // Deployment name for the provider - ApiVersion string `json:"api_version" bson:"api_version"` // API version for the provider - Models []string `json:"models" bson:"models"` // Models supported by this provider - Unset bool `json:"unset" bson:"-"` // Whether the provider is unset + Key string `json:"key" bson:"key" description:"Provider key (e.g., 'openai', 'anthropic', 'gemini')"` + Name string `json:"name" bson:"name" description:"Display name for UI"` + Enabled bool `json:"enabled" bson:"enabled" description:"Whether this provider is enabled"` + ApiKey string `json:"api_key" bson:"api_key" description:"API key for the provider"` + ApiBaseUrl string `json:"api_base_url" bson:"api_base_url" description:"API base URL for the provider"` + DeploymentName string `json:"deployment_name" bson:"deployment_name" description:"Deployment name for the provider"` + ApiVersion string `json:"api_version" bson:"api_version" description:"API version for the provider"` + Models []string `json:"models" bson:"models" description:"Models supported by this provider"` + Unset bool `json:"unset" bson:"-"` // Whether the provider is unset } func (p *LLMProvider) IsUnset() bool { diff --git a/core/models/models/metric.go b/core/models/models/metric.go index a9494754..cc1f4603 100644 --- a/core/models/models/metric.go +++ b/core/models/models/metric.go @@ -7,19 +7,19 @@ import ( type Metric struct { any `collection:"metrics"` BaseModel `bson:",inline"` - Type string `json:"type" bson:"type"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` - TotalMemory uint64 `json:"total_memory" bson:"total_memory"` - AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` - UsedMemory uint64 `json:"used_memory" bson:"used_memory"` - UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` - TotalDisk uint64 `json:"total_disk" bson:"total_disk"` - AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` - UsedDisk uint64 `json:"used_disk" bson:"used_disk"` - UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` - DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"` - DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"` - NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"` - NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"` + Type string `json:"type" bson:"type" description:"Type"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id" description:"Node ID"` + CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent" description:"CPU usage percentage"` + TotalMemory uint64 `json:"total_memory" bson:"total_memory" description:"Total memory"` + AvailableMemory uint64 `json:"available_memory" bson:"available_memory" description:"Available memory"` + UsedMemory uint64 `json:"used_memory" bson:"used_memory" description:"Used memory"` + UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent" description:"Used memory percentage"` + TotalDisk uint64 `json:"total_disk" bson:"total_disk" description:"Total disk"` + AvailableDisk uint64 `json:"available_disk" bson:"available_disk" description:"Available disk"` + UsedDisk uint64 `json:"used_disk" bson:"used_disk" description:"Used disk"` + UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent" description:"Used disk percentage"` + DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate" description:"Disk read bytes rate"` + DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate" description:"Disk write bytes rate"` + NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate" description:"Network bytes sent rate"` + NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate" description:"Network bytes recv rate"` } diff --git a/core/models/models/node.go b/core/models/models/node.go index 2adcdb23..37fe1fa2 100644 --- a/core/models/models/node.go +++ b/core/models/models/node.go @@ -7,17 +7,17 @@ import ( type Node struct { any `collection:"nodes"` BaseModel `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Ip string `json:"ip" bson:"ip"` - Mac string `json:"mac" bson:"mac"` - Hostname string `json:"hostname" bson:"hostname"` - Description string `json:"description" bson:"description"` - IsMaster bool `json:"is_master" bson:"is_master"` - Status string `json:"status" bson:"status"` - Enabled bool `json:"enabled" bson:"enabled"` - Active bool `json:"active" bson:"active"` - ActiveAt time.Time `json:"active_at" bson:"active_ts"` - CurrentRunners int `json:"current_runners" bson:"current_runners"` - MaxRunners int `json:"max_runners" bson:"max_runners"` + Key string `json:"key" bson:"key" description:"Key"` + Name string `json:"name" bson:"name" description:"Name"` + Ip string `json:"ip" bson:"ip" description:"IP"` + Mac string `json:"mac" bson:"mac" description:"MAC"` + Hostname string `json:"hostname" bson:"hostname" description:"Hostname"` + Description string `json:"description" bson:"description" description:"Description"` + IsMaster bool `json:"is_master" bson:"is_master" description:"Is master"` + Status string `json:"status" bson:"status" description:"Status"` + Enabled bool `json:"enabled" bson:"enabled" description:"Enabled"` + Active bool `json:"active" bson:"active" description:"Active"` + ActiveAt time.Time `json:"active_at" bson:"active_ts" description:"Active at"` + CurrentRunners int `json:"current_runners" bson:"current_runners" description:"Current runners"` + MaxRunners int `json:"max_runners" bson:"max_runners" description:"Max runners"` } diff --git a/core/models/models/notification_alert.go b/core/models/models/notification_alert.go index 0fb4c145..97def373 100644 --- a/core/models/models/notification_alert.go +++ b/core/models/models/notification_alert.go @@ -5,15 +5,15 @@ import "go.mongodb.org/mongo-driver/bson/primitive" type NotificationAlert struct { any `collection:"notification_alerts"` BaseModel `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - HasMetricTarget bool `json:"has_metric_target" bson:"has_metric_target"` - MetricTargetId primitive.ObjectID `json:"metric_target_id,omitempty" bson:"metric_target_id,omitempty"` - MetricName string `json:"metric_name" bson:"metric_name"` - Operator string `json:"operator" bson:"operator"` - LastingSeconds int `json:"lasting_seconds" bson:"lasting_seconds"` - TargetValue float32 `json:"target_value" bson:"target_value"` - Level string `json:"level" bson:"level"` - TemplateKey string `json:"template_key,omitempty" bson:"template_key,omitempty"` + Name string `json:"name" bson:"name" description:"Name"` + Description string `json:"description" bson:"description" description:"Description"` + Enabled bool `json:"enabled" bson:"enabled" description:"Enabled"` + HasMetricTarget bool `json:"has_metric_target" bson:"has_metric_target" description:"Has metric target"` + MetricTargetId primitive.ObjectID `json:"metric_target_id,omitempty" bson:"metric_target_id,omitempty" description:"Metric target ID"` + MetricName string `json:"metric_name" bson:"metric_name" description:"Metric name"` + Operator string `json:"operator" bson:"operator" description:"Operator"` + LastingSeconds int `json:"lasting_seconds" bson:"lasting_seconds" description:"Lasting seconds"` + TargetValue float32 `json:"target_value" bson:"target_value" description:"Target value"` + Level string `json:"level" bson:"level" description:"Level"` + TemplateKey string `json:"template_key,omitempty" bson:"template_key,omitempty" description:"Template key"` } diff --git a/core/models/models/notification_channel.go b/core/models/models/notification_channel.go index 972aca09..d73be50e 100644 --- a/core/models/models/notification_channel.go +++ b/core/models/models/notification_channel.go @@ -3,16 +3,16 @@ package models type NotificationChannel struct { any `collection:"notification_channels"` BaseModel `bson:",inline"` - Type string `json:"type" bson:"type"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Provider string `json:"provider" bson:"provider"` - SMTPServer string `json:"smtp_server,omitempty" bson:"smtp_server,omitempty"` - SMTPPort int `json:"smtp_port,omitempty" bson:"smtp_port,omitempty"` - SMTPUsername string `json:"smtp_username,omitempty" bson:"smtp_username,omitempty"` - SMTPPassword string `json:"smtp_password,omitempty" bson:"smtp_password,omitempty"` - WebhookUrl string `json:"webhook_url,omitempty" bson:"webhook_url,omitempty"` - TelegramBotToken string `json:"telegram_bot_token,omitempty" bson:"telegram_bot_token,omitempty"` - TelegramChatId string `json:"telegram_chat_id,omitempty" bson:"telegram_chat_id,omitempty"` - GoogleOAuth2Json string `json:"google_oauth2_json,omitempty" bson:"google_oauth2_json,omitempty"` + Type string `json:"type" bson:"type" description:"Type" enum:"im,mail"` + Name string `json:"name" bson:"name" description:"Name"` + Description string `json:"description" bson:"description" description:"Description"` + Provider string `json:"provider" bson:"provider" description:"Provider"` + SMTPServer string `json:"smtp_server,omitempty" bson:"smtp_server,omitempty" description:"SMTP server"` + SMTPPort int `json:"smtp_port,omitempty" bson:"smtp_port,omitempty" description:"SMTP port"` + SMTPUsername string `json:"smtp_username,omitempty" bson:"smtp_username,omitempty" description:"SMTP username"` + SMTPPassword string `json:"smtp_password,omitempty" bson:"smtp_password,omitempty" description:"SMTP password"` + WebhookUrl string `json:"webhook_url,omitempty" bson:"webhook_url,omitempty" description:"Webhook URL"` + TelegramBotToken string `json:"telegram_bot_token,omitempty" bson:"telegram_bot_token,omitempty" description:"Telegram bot token"` + TelegramChatId string `json:"telegram_chat_id,omitempty" bson:"telegram_chat_id,omitempty" description:"Telegram chat ID"` + GoogleOAuth2Json string `json:"google_oauth2_json,omitempty" bson:"google_oauth2_json,omitempty" description:"Google OAuth2 JSON"` } diff --git a/core/models/models/notification_request.go b/core/models/models/notification_request.go index 355f05aa..f466b235 100644 --- a/core/models/models/notification_request.go +++ b/core/models/models/notification_request.go @@ -5,18 +5,18 @@ import "go.mongodb.org/mongo-driver/bson/primitive" type NotificationRequest struct { any `collection:"notification_requests"` BaseModel `bson:",inline"` - Status string `json:"status" bson:"status"` - Error string `json:"error,omitempty" bson:"error,omitempty"` - Title string `json:"title" bson:"title"` - Content string `json:"content" bson:"content"` - SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` - SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"` - MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty"` - MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` - MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` - SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` - ChannelId primitive.ObjectID `json:"channel_id" bson:"channel_id"` - Setting *NotificationSetting `json:"setting,omitempty" bson:"-"` - Channel *NotificationChannel `json:"channel,omitempty" bson:"-"` - Test bool `json:"test,omitempty" bson:"test,omitempty"` + Status string `json:"status" bson:"status" description:"Status"` + Error string `json:"error,omitempty" bson:"error,omitempty" description:"Error"` + Title string `json:"title" bson:"title" description:"Title"` + Content string `json:"content" bson:"content" description:"Content"` + SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty" description:"Sender email"` + SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty" description:"Sender name"` + MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty" description:"Mail to"` + MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty" description:"Mail CC"` + MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty" description:"Mail BCC"` + SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id" description:"Setting ID"` + ChannelId primitive.ObjectID `json:"channel_id" bson:"channel_id" description:"Channel ID"` + Setting *NotificationSetting `json:"setting,omitempty" bson:"-" binding:"-"` + Channel *NotificationChannel `json:"channel,omitempty" bson:"-" binding:"-"` + Test bool `json:"test,omitempty" bson:"test,omitempty" description:"Test"` } diff --git a/core/models/models/notification_setting.go b/core/models/models/notification_setting.go index b937c446..d6c7bf6d 100644 --- a/core/models/models/notification_setting.go +++ b/core/models/models/notification_setting.go @@ -5,30 +5,30 @@ import "go.mongodb.org/mongo-driver/bson/primitive" type NotificationSetting struct { any `collection:"notification_settings"` BaseModel `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` + Name string `json:"name" bson:"name" description:"Name"` + Description string `json:"description" bson:"description" description:"Description"` + Enabled bool `json:"enabled" bson:"enabled" description:"Enabled"` - Title string `json:"title,omitempty" bson:"title,omitempty"` - Template string `json:"template" bson:"template"` - TemplateMode string `json:"template_mode" bson:"template_mode"` - TemplateMarkdown string `json:"template_markdown,omitempty" bson:"template_markdown,omitempty"` - TemplateRichText string `json:"template_rich_text,omitempty" bson:"template_rich_text,omitempty"` - TemplateRichTextJson string `json:"template_rich_text_json,omitempty" bson:"template_rich_text_json,omitempty"` - TemplateTheme string `json:"template_theme,omitempty" bson:"template_theme,omitempty"` + Title string `json:"title,omitempty" bson:"title,omitempty" description:"Title"` + Template string `json:"template" bson:"template" description:"Template"` + TemplateMode string `json:"template_mode" bson:"template_mode" description:"Template mode"` + TemplateMarkdown string `json:"template_markdown,omitempty" bson:"template_markdown,omitempty" description:"Template markdown"` + TemplateRichText string `json:"template_rich_text,omitempty" bson:"template_rich_text,omitempty" description:"Template rich text"` + TemplateRichTextJson string `json:"template_rich_text_json,omitempty" bson:"template_rich_text_json,omitempty" description:"Template rich text JSON"` + TemplateTheme string `json:"template_theme,omitempty" bson:"template_theme,omitempty" description:"Template theme"` - TaskTrigger string `json:"task_trigger" bson:"task_trigger"` - Trigger string `json:"trigger" bson:"trigger"` + TaskTrigger string `json:"task_trigger" bson:"task_trigger" description:"Task trigger"` + Trigger string `json:"trigger" bson:"trigger" description:"Trigger"` - SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` - UseCustomSenderEmail bool `json:"use_custom_sender_email,omitempty" bson:"use_custom_sender_email,omitempty"` - SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"` - MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty"` - MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` - MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` + SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty" description:"Sender email"` + UseCustomSenderEmail bool `json:"use_custom_sender_email,omitempty" bson:"use_custom_sender_email,omitempty" description:"Use custom sender email"` + SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty" description:"Sender name"` + MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty" description:"Mail to"` + MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty" description:"Mail CC"` + MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty" description:"Mail BCC"` - ChannelIds []primitive.ObjectID `json:"channel_ids,omitempty" bson:"channel_ids,omitempty"` + ChannelIds []primitive.ObjectID `json:"channel_ids,omitempty" bson:"channel_ids,omitempty" description:"Channel IDs"` Channels []NotificationChannel `json:"channels,omitempty" bson:"-"` - AlertId primitive.ObjectID `json:"alert_id,omitempty" bson:"alert_id,omitempty"` + AlertId primitive.ObjectID `json:"alert_id,omitempty" bson:"alert_id,omitempty" description:"Alert ID"` } diff --git a/core/models/models/permission.go b/core/models/models/permission.go index e59007d2..a48ba4f8 100644 --- a/core/models/models/permission.go +++ b/core/models/models/permission.go @@ -5,10 +5,10 @@ import "go.mongodb.org/mongo-driver/bson/primitive" type Permission struct { any `collection:"permissions"` BaseModel `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - Type string `json:"type" bson:"type"` - Routes []string `json:"routes" bson:"routes"` + Key string `json:"key" bson:"key" description:"Key"` + Name string `json:"name" bson:"name" description:"Name"` + Description string `json:"description" bson:"description" description:"Description"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id" description:"Role ID"` + Type string `json:"type" bson:"type" description:"Type"` + Routes []string `json:"routes" bson:"routes" description:"Routes"` } diff --git a/core/models/models/project.go b/core/models/models/project.go index e3f6a1ca..f5f844b5 100644 --- a/core/models/models/project.go +++ b/core/models/models/project.go @@ -3,7 +3,7 @@ package models type Project struct { any `collection:"projects"` BaseModel `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` + Name string `json:"name" bson:"name" description:"Name"` + Description string `json:"description" bson:"description" description:"Description"` Spiders int `json:"spiders" bson:"-"` } diff --git a/core/models/models/role.go b/core/models/models/role.go index ea6b39e9..793ed2ea 100644 --- a/core/models/models/role.go +++ b/core/models/models/role.go @@ -3,10 +3,10 @@ package models type Role struct { any `collection:"roles"` BaseModel `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Routes []string `json:"routes" bson:"routes"` - RootAdmin bool `json:"-" bson:"root_admin,omitempty"` + Name string `json:"name" bson:"name" description:"Name"` + Description string `json:"description" bson:"description" description:"Description"` + Routes []string `json:"routes" bson:"routes" description:"Routes"` + RootAdmin bool `json:"-" bson:"root_admin,omitempty" description:"Root admin"` IsRootAdmin bool `json:"root_admin" bson:"-"` Users int `json:"users" bson:"-"` } diff --git a/core/models/models/schedule.go b/core/models/models/schedule.go index b4bc232b..ad650220 100644 --- a/core/models/models/schedule.go +++ b/core/models/models/schedule.go @@ -8,15 +8,15 @@ import ( type Schedule struct { any `collection:"schedules"` BaseModel `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` - Cron string `json:"cron" bson:"cron"` - EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` - Cmd string `json:"cmd" bson:"cmd"` - Param string `json:"param" bson:"param"` - Mode string `json:"mode" bson:"mode"` - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` - Priority int `json:"priority" bson:"priority"` - Enabled bool `json:"enabled" bson:"enabled"` + Name string `json:"name" bson:"name" description:"Name"` + Description string `json:"description" bson:"description" description:"Description"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id" description:"Spider ID"` + Cron string `json:"cron" bson:"cron" description:"Cron"` + EntryId cron.EntryID `json:"entry_id" bson:"entry_id" description:"Entry ID"` + Cmd string `json:"cmd" bson:"cmd" description:"Cmd"` + Param string `json:"param" bson:"param" description:"Param"` + Mode string `json:"mode" bson:"mode" description:"Mode"` + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids" description:"Node IDs"` + Priority int `json:"priority" bson:"priority" description:"Priority"` + Enabled bool `json:"enabled" bson:"enabled" description:"Enabled"` } diff --git a/core/models/models/setting.go b/core/models/models/setting.go index c66bdbcd..67867de9 100644 --- a/core/models/models/setting.go +++ b/core/models/models/setting.go @@ -7,6 +7,6 @@ import ( type Setting struct { any `collection:"settings"` BaseModel `bson:",inline"` - Key string `json:"key" bson:"key"` - Value bson.M `json:"value" bson:"value"` + Key string `json:"key" bson:"key" description:"Key"` + Value bson.M `json:"value" bson:"value" description:"Value"` } diff --git a/core/models/models/spider.go b/core/models/models/spider.go index 14b008cd..50186592 100644 --- a/core/models/models/spider.go +++ b/core/models/models/spider.go @@ -7,34 +7,34 @@ import ( type Spider struct { any `collection:"spiders"` BaseModel `bson:",inline"` - Name string `json:"name" bson:"name"` // spider name - ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id (deprecated) # TODO: remove this field in the future - ColName string `json:"col_name,omitempty" bson:"col_name"` // data collection name - DbName string `json:"db_name,omitempty" bson:"db_name"` // database name - DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id - DataSource *Database `json:"data_source,omitempty" bson:"-"` // data source - Description string `json:"description" bson:"description"` // description - ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id - Mode string `json:"mode" bson:"mode"` // default Task.Mode - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds - GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id - GitRootPath string `json:"git_root_path" bson:"git_root_path"` + Name string `json:"name" bson:"name" description:"Spider name"` + ColId primitive.ObjectID `json:"col_id" bson:"col_id" description:"Data collection id" deprecated:"true"` + ColName string `json:"col_name,omitempty" bson:"col_name" description:"Data collection name"` + DbName string `json:"db_name,omitempty" bson:"db_name" description:"Database name"` + DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id" description:"Data source id"` + DataSource *Database `json:"data_source,omitempty" bson:"-"` + Description string `json:"description" bson:"description" description:"Description"` + ProjectId primitive.ObjectID `json:"project_id" bson:"project_id" description:"Project ID"` + Mode string `json:"mode" bson:"mode" description:"Default task mode" enum:"random,all,selected-nodes"` + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids" description:"Default node ids, used in selected-nodes mode"` + GitId primitive.ObjectID `json:"git_id" bson:"git_id" description:"Related Git ID"` + GitRootPath string `json:"git_root_path" bson:"git_root_path" description:"Git root path"` Git *Git `json:"git,omitempty" bson:"-"` - Template string `json:"template,omitempty" bson:"template,omitempty"` // spider template - TemplateParams *SpiderTemplateParams `json:"template_params,omitempty" bson:"template_params,omitempty"` + Template string `json:"template,omitempty" bson:"template,omitempty" description:"Spider template"` + TemplateParams *SpiderTemplateParams `json:"template_params,omitempty" bson:"template_params,omitempty" description:"Spider template params"` // stats Stat *SpiderStat `json:"stat,omitempty" bson:"-"` // execution - Cmd string `json:"cmd" bson:"cmd"` // execute command - Param string `json:"param" bson:"param"` // default task param - Priority int `json:"priority" bson:"priority"` + Cmd string `json:"cmd" bson:"cmd" description:"Execute command"` + Param string `json:"param" bson:"param" description:"Default task param"` + Priority int `json:"priority" bson:"priority" description:"Priority" default:"5" minimum:"1" maximum:"10"` } type SpiderTemplateParams struct { - ProjectName string `json:"project_name,omitempty" bson:"project_name,omitempty"` - SpiderName string `json:"spider_name,omitempty" bson:"spider_name,omitempty"` - StartUrls string `json:"start_urls,omitempty" bson:"start_urls,omitempty"` - AllowedDomains string `json:"allowed_domains,omitempty" bson:"allowed_domains,omitempty"` + ProjectName string `json:"project_name,omitempty" bson:"project_name,omitempty" description:"Project name"` + SpiderName string `json:"spider_name,omitempty" bson:"spider_name,omitempty" description:"Spider name"` + StartUrls string `json:"start_urls,omitempty" bson:"start_urls,omitempty" description:"Start urls"` + AllowedDomains string `json:"allowed_domains,omitempty" bson:"allowed_domains,omitempty" description:"Allowed domains"` } diff --git a/core/models/models/spider_stat.go b/core/models/models/spider_stat.go index c122a540..9fd71bbe 100644 --- a/core/models/models/spider_stat.go +++ b/core/models/models/spider_stat.go @@ -7,14 +7,14 @@ import ( type SpiderStat struct { any `collection:"spider_stats"` BaseModel `bson:",inline"` - LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"` + LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty" description:"Last task ID"` LastTask *Task `json:"last_task,omitempty" bson:"-"` - Tasks int `json:"tasks" bson:"tasks"` - Results int `json:"results" bson:"results"` - WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second - RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second - TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second - AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second - AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second - AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second + Tasks int `json:"tasks" bson:"tasks" description:"Task count"` + Results int `json:"results" bson:"results" description:"Result count"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty" description:"Wait duration (in second)"` + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty" description:"Runtime duration (in second)"` + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty" description:"Total duration (in second)"` + AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second + AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second + AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second } diff --git a/core/models/models/task.go b/core/models/models/task.go index 00e723c2..c5c3c29e 100644 --- a/core/models/models/task.go +++ b/core/models/models/task.go @@ -7,17 +7,17 @@ import ( type Task struct { any `collection:"tasks"` BaseModel `bson:",inline"` - SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` - Status string `json:"status" bson:"status"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - Cmd string `json:"cmd" bson:"cmd"` - Param string `json:"param" bson:"param"` - Error string `json:"error" bson:"error"` - Pid int `json:"pid" bson:"pid"` - ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` - Type string `json:"type" bson:"type"` - Mode string `json:"mode" bson:"mode"` - Priority int `json:"priority" bson:"priority"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id" description:"Spider ID"` + Status string `json:"status" bson:"status" description:"Status"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id" description:"Node ID"` + Cmd string `json:"cmd" bson:"cmd" description:"Command"` + Param string `json:"param" bson:"param" description:"Parameter"` + Error string `json:"error" bson:"error" description:"Error"` + Pid int `json:"pid" bson:"pid" description:"Process ID"` + ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id" description:"Schedule ID"` + Type string `json:"type" bson:"type" description:"Type"` + Mode string `json:"mode" bson:"mode" description:"Mode"` + Priority int `json:"priority" bson:"priority" description:"Priority"` NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-"` Stat *TaskStat `json:"stat,omitempty" bson:"-"` Spider *Spider `json:"spider,omitempty" bson:"-"` diff --git a/core/models/models/task_stat.go b/core/models/models/task_stat.go index 4be7b233..5d34e2a0 100644 --- a/core/models/models/task_stat.go +++ b/core/models/models/task_stat.go @@ -7,10 +7,10 @@ import ( type TaskStat struct { any `collection:"task_stats"` BaseModel `bson:",inline"` - StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` - EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` - WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond - RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond - TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond - ResultCount int64 `json:"result_count" bson:"result_count"` + StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty" description:"Start time"` + EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty" description:"End time"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty" description:"Wait duration (in millisecond)"` + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty" description:"Runtime duration (in millisecond)"` + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty" description:"Total duration (in millisecond)"` + ResultCount int64 `json:"result_count" bson:"result_count" description:"Result count"` } diff --git a/core/models/models/token.go b/core/models/models/token.go index f421aa2d..6da62941 100644 --- a/core/models/models/token.go +++ b/core/models/models/token.go @@ -3,6 +3,6 @@ package models type Token struct { any `collection:"tokens"` BaseModel `bson:",inline"` - Name string `json:"name" bson:"name"` - Token string `json:"token" bson:"token"` + Name string `json:"name" bson:"name" description:"Name"` + Token string `json:"token" bson:"token" description:"Token"` } diff --git a/core/models/models/user.go b/core/models/models/user.go index f9681cfc..1166682f 100644 --- a/core/models/models/user.go +++ b/core/models/models/user.go @@ -5,14 +5,14 @@ import "go.mongodb.org/mongo-driver/bson/primitive" type User struct { any `collection:"users"` BaseModel `bson:",inline"` - Username string `json:"username" bson:"username"` - Password string `json:"-" bson:"password"` - Role string `json:"role" bson:"role"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - FirstName string `json:"first_name" bson:"first_name"` - LastName string `json:"last_name" bson:"last_name"` - Email string `json:"email" bson:"email"` - RootAdmin bool `json:"root_admin,omitempty" bson:"root_admin"` + Username string `json:"username" bson:"username" description:"Username"` + Password string `json:"-" bson:"password" description:"Password"` + Role string `json:"role" bson:"role" description:"Role"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id" description:"Role ID"` + FirstName string `json:"first_name" bson:"first_name" description:"First name"` + LastName string `json:"last_name" bson:"last_name" description:"Last name"` + Email string `json:"email" bson:"email" description:"Email"` + RootAdmin bool `json:"root_admin,omitempty" bson:"root_admin" description:"Root admin"` RootAdminRole bool `json:"root_admin_role,omitempty" bson:"-"` Routes []string `json:"routes,omitempty" bson:"-"` } diff --git a/core/models/models/user_role.go b/core/models/models/user_role.go index 976d5a57..ed7ae8a2 100644 --- a/core/models/models/user_role.go +++ b/core/models/models/user_role.go @@ -7,6 +7,6 @@ import ( type UserRole struct { any `collection:"user_roles"` BaseModel `bson:",inline"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - UserId primitive.ObjectID `json:"user_id" bson:"user_id"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id" description:"Role ID"` + UserId primitive.ObjectID `json:"user_id" bson:"user_id" description:"User ID"` } diff --git a/core/utils/string.go b/core/utils/string.go index ed726045..c57ac259 100644 --- a/core/utils/string.go +++ b/core/utils/string.go @@ -32,9 +32,7 @@ func ToSnakeCase(s string) string { } func ToPascalCase(s string) string { - s = strings.TrimSpace(s) - s = strings.ReplaceAll(s, "_", " ") - s = cases.Title(language.English).String(s) + s = Capitalize(s) s = strings.ReplaceAll(s, " ", "") return s } @@ -45,6 +43,15 @@ func ToKebabCase(s string) string { return replaceChars(s, []string{" ", "_", "."}, "-") } +func Capitalize(s string) string { + s = strings.TrimSpace(s) + for _, char := range []string{" ", "_", "-", "/"} { + s = strings.ReplaceAll(s, char, " ") + } + s = cases.Title(language.English).String(s) + return s +} + // splitStringWithQuotes splits a string with quotes // Parameters: // - s: the string to split