git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-26 17:49:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: enhance parameter descriptions and standardize model fields

Browse Source 
- Added descriptive annotations to various parameters across controllers to improve API documentation clarity.
- Standardized field definitions in models by including descriptions for better understanding of their purpose.
- Updated validation patterns for ID fields to ensure consistency and improve data integrity.
- Enhanced overall code readability and maintainability by aligning naming conventions and adding necessary comments.
This commit is contained in:
Marvin Zhang
2025-03-19 13:39:02 +08:00
parent 13ce1d551b
commit 931a36c8bf
41 changed files with 551 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
core/controllers/base.go
View File

@@ -1,10 +1,11 @@
package controllers
import (
"github.com/loopfz/gadgeto/tonic"
"net/http"
"time"
"github.com/loopfz/gadgeto/tonic"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/mongo"
@@ -45,6 +46,8 @@ func init() {
type Action struct {
Method string
Path string
Name string
Description string
HandlerFunc interface{}
}
@@ -57,8 +60,8 @@ type BaseController[T any] struct {
type GetListParams struct {
Conditions string `query:"conditions" description:"Filter conditions. Format: [{\"key\":\"name\",\"op\":\"eq\",\"value\":\"test\"}]"`
Sort string `query:"sort" description:"Sort options"`
Page int `query:"page" default:"1" description:"Page number"`
Size int `query:"size" default:"10" description:"Page size"`
Page int `query:"page" default:"1" description:"Page number" minimum:"1"`
Size int `query:"size" default:"10" description:"Page size" minimum:"1"`
All bool `query:"all" default:"false" description:"Whether to get all items"`
}
@@ -72,7 +75,7 @@ func (ctr *BaseController[T]) GetList(_ *gin.Context, params *GetListParams) (re
}
type GetByIdParams struct {
Id string `path:"id" description:"The ID of the item to get"`
Id string `path:"id" description:"The ID of the item to get" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
}
func (ctr *BaseController[T]) GetById(_ *gin.Context, params *GetByIdParams) (response *Response[T], err error) {
@@ -90,7 +93,7 @@ func (ctr *BaseController[T]) GetById(_ *gin.Context, params *GetByIdParams) (re
}
type PostParams[T any] struct {
Data T `json:"data"`
Data T `json:"data" description:"The data to create" validate:"required"`
}
func (ctr *BaseController[T]) Post(c *gin.Context, params *PostParams[T]) (response *Response[T], err error) {
@@ -114,8 +117,8 @@ func (ctr *BaseController[T]) Post(c *gin.Context, params *PostParams[T]) (respo
}
type PutByIdParams[T any] struct {
Id string `path:"id" description:"The ID of the item to update"`
Data T `json:"data"`
Id string `path:"id" description:"The ID of the item to update" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Data T `json:"data" description:"The data to update" validate:"required"`
}
func (ctr *BaseController[T]) PutById(c *gin.Context, params *PutByIdParams[T]) (response *Response[T], err error) {
@@ -144,7 +147,7 @@ func (ctr *BaseController[T]) PutById(c *gin.Context, params *PutByIdParams[T])
}
type PatchParams struct {
Ids []string `json:"ids" description:"The IDs of the items to update" validate:"required"`
Ids []string `json:"ids" description:"The IDs of the items to update" validate:"required" items.type:"string" items.format:"objectid" items.pattern:"^[0-9a-fA-F]{24}$"`
Update bson.M `json:"update" description:"The update object" validate:"required"`
}
@@ -184,7 +187,7 @@ func (ctr *BaseController[T]) PatchList(c *gin.Context, params *PatchParams) (re
}
type DeleteByIdParams struct {
Id string `path:"id" description:"The ID of the item to get"`
Id string `path:"id" description:"The ID of the item to delete" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
}
func (ctr *BaseController[T]) DeleteById(c *gin.Context, params *DeleteByIdParams) (res *Response[T], err error) {
@@ -203,7 +206,7 @@ func (ctr *BaseController[T]) DeleteById(c *gin.Context, params *DeleteByIdParam
}
type DeleteListParams struct {
Ids []string `json:"ids" description:"The IDs of the items to delete"`
Ids []string `json:"ids" description:"The IDs of the items to delete" items.type:"string" items.format:"objectid" items.pattern:"^[0-9a-fA-F]{24}$"`
}
func (ctr *BaseController[T]) DeleteList(_ *gin.Context, params *DeleteListParams) (res *Response[T], err error) {

4
core/controllers/login.go
View File

@@ -8,8 +8,8 @@ import (
)
type PostLoginParams struct {
Username string `json:"username" validate:"required"`
Password string `json:"password" validate:"required"`
Username string `json:"username" description:"Username" validate:"required"`
Password string `json:"password" description:"Password" validate:"required"`
}
func PostLogin(c *gin.Context, params *PostLoginParams) (response *Response[string], err error) {

199
core/controllers/router.go
View File

@@ -1,9 +1,12 @@
package controllers
import (
"fmt"
"net/http"
"strings"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/fizz"
"github.com/crawlab-team/crawlab/core/middlewares"
@@ -56,23 +59,22 @@ func RegisterController[T any](group *fizz.RouterGroup, basePath string, ctr *Ba
// Create appropriate model response based on the action
responses := globalWrapper.BuildModelResponse()
id := getIDForAction(action.Method, fullPath)
summary := getSummaryForAction(action.Method, basePath, action.Path)
description := getDescriptionForAction(action.Method, basePath, action.Path)
id := getIDForAction(action.Method, fullPath, action.Name)
summary := getSummaryForAction(action.Method, basePath, action.Path, action.Name)
description := getDescriptionForAction(action.Method, basePath, action.Path, action.Description)
globalWrapper.RegisterRoute(action.Method, fullPath, group, action.HandlerFunc, id, summary, description, responses)
}
// Register built-in handlers if they haven't been overridden
// Create a zero value of T to use as the model
var model T
registerBuiltinHandler(group, globalWrapper, basePath, http.MethodGet, "", ctr.GetList, actionPaths, "Get list", "Get a list of items", model)
registerBuiltinHandler(group, globalWrapper, basePath, http.MethodGet, "/:id", ctr.GetById, actionPaths, "Get by ID", "Get a single item by ID", model)
registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPost, "", ctr.Post, actionPaths, "Create", "Create a new item", model)
registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPut, "/:id", ctr.PutById, actionPaths, "Update by ID", "Update an item by ID", model)
registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPatch, "", ctr.PatchList, actionPaths, "Patch list", "Patch multiple items", model)
registerBuiltinHandler(group, globalWrapper, basePath, http.MethodDelete, "/:id", ctr.DeleteById, actionPaths, "Delete by ID", "Delete an item by ID", model)
registerBuiltinHandler(group, globalWrapper, basePath, http.MethodDelete, "", ctr.DeleteList, actionPaths, "Delete list", "Delete multiple items", model)
resource := getResourceName(basePath)
registerBuiltinHandler(group, globalWrapper, basePath, http.MethodGet, "", ctr.GetList, actionPaths, fmt.Sprintf("Get %s List", resource), "Get a list of items")
registerBuiltinHandler(group, globalWrapper, basePath, http.MethodGet, "/:id", ctr.GetById, actionPaths, fmt.Sprintf("Get %s by ID", resource), "Get a single item by ID")
registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPost, "", ctr.Post, actionPaths, fmt.Sprintf("Create %s", resource), "Create a new item")
registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPut, "/:id", ctr.PutById, actionPaths, fmt.Sprintf("Update %s by ID", resource), "Update an item by ID")
registerBuiltinHandler(group, globalWrapper, basePath, http.MethodPatch, "", ctr.PatchList, actionPaths, fmt.Sprintf("Patch %s List", resource), "Patch multiple items")
registerBuiltinHandler(group, globalWrapper, basePath, http.MethodDelete, "/:id", ctr.DeleteById, actionPaths, fmt.Sprintf("Delete %s by ID", resource), "Delete an item by ID")
registerBuiltinHandler(group, globalWrapper, basePath, http.MethodDelete, "", ctr.DeleteList, actionPaths, fmt.Sprintf("Delete %s List", resource), "Delete multiple items")
}
// RegisterActions registers a list of custom action handlers to a route group
@@ -83,9 +85,9 @@ func RegisterActions(group *fizz.RouterGroup, basePath string, actions []Action)
// Create generic response
responses := globalWrapper.BuildModelResponse()
id := getIDForAction(action.Method, fullPath)
summary := getSummaryForAction(action.Method, basePath, action.Path)
description := getDescriptionForAction(action.Method, basePath, action.Path)
id := getIDForAction(action.Method, fullPath, action.Name)
summary := getSummaryForAction(action.Method, basePath, action.Path, action.Name)
description := getDescriptionForAction(action.Method, basePath, action.Path, action.Description)
globalWrapper.RegisterRoute(action.Method, fullPath, group, action.HandlerFunc, id, summary, description, responses)
}
@@ -93,7 +95,7 @@ func RegisterActions(group *fizz.RouterGroup, basePath string, actions []Action)
// registerBuiltinHandler registers a standard handler if it hasn't been overridden
// by a custom action
func registerBuiltinHandler[T any](group *fizz.RouterGroup, wrapper *openapi.FizzWrapper, basePath, method, pathSuffix string, handlerFunc interface{}, existingActionPaths map[string]bool, summary, description string, model T) {
func registerBuiltinHandler(group *fizz.RouterGroup, wrapper *openapi.FizzWrapper, basePath, method, pathSuffix string, handlerFunc interface{}, existingActionPaths map[string]bool, summary, description string) {
path := basePath + pathSuffix
key := method + " - " + path
_, ok := existingActionPaths[key]
@@ -101,7 +103,7 @@ func registerBuiltinHandler[T any](group *fizz.RouterGroup, wrapper *openapi.Fiz
return
}
id := getIDForAction(method, path)
id := getIDForAction(method, path, summary)
// Create appropriate response based on the method
responses := wrapper.BuildModelResponse()
@@ -110,7 +112,11 @@ func registerBuiltinHandler[T any](group *fizz.RouterGroup, wrapper *openapi.Fiz
}
// Helper functions to generate OpenAPI documentation
func getIDForAction(method, path string) string {
func getIDForAction(method, path, summary string) string {
if summary != "" {
return utils.ToPascalCase(summary)
}
// Remove leading slash and convert remaining slashes to underscores
cleanPath := strings.TrimPrefix(path, "/")
cleanPath = strings.ReplaceAll(cleanPath, "/", "_")
@@ -120,7 +126,11 @@ func getIDForAction(method, path string) string {
return method + "_" + cleanPath
}
func getSummaryForAction(method, basePath, path string) string {
func getSummaryForAction(method, basePath, path, summary string) string {
if summary != "" {
return summary
}
resource := getResourceName(basePath)
switch method {
@@ -140,13 +150,13 @@ func getSummaryForAction(method, basePath, path string) string {
}
case http.MethodPatch:
if path == "" {
return "Patch " + resource + " list"
return "Patch " + resource + " List"
}
case http.MethodDelete:
if path == "/:id" {
return "Delete " + resource + " by ID"
} else if path == "" {
return "Delete " + resource + " list"
return "Delete " + resource + " List"
}
}
@@ -158,7 +168,11 @@ func getSummaryForAction(method, basePath, path string) string {
return method + " " + resource
}
func getDescriptionForAction(method, basePath, path string) string {
func getDescriptionForAction(method, basePath, path, description string) string {
if description != "" {
return description
}
resource := getResourceName(basePath)
switch method {
@@ -197,22 +211,25 @@ func getDescriptionForAction(method, basePath, path string) string {
}
func getResourceName(basePath string) string {
resource := basePath
// Remove leading slash and get the last part of the path
if len(basePath) > 0 && basePath[0] == '/' {
basePath = basePath[1:]
if len(resource) > 0 && resource[0] == '/' {
resource = resource[1:]
}
// Remove trailing slash if present
if len(basePath) > 0 && basePath[len(basePath)-1] == '/' {
basePath = basePath[:len(basePath)-1]
if len(resource) > 0 && resource[len(resource)-1] == '/' {
resource = resource[:len(resource)-1]
}
// If path is empty, return "resource"
if basePath == "" {
return "resource"
}
// Convert to capitalized form
resource = utils.Capitalize(resource)
return basePath
// Convert to non-plural form
resource = strings.TrimSuffix(resource, "s")
return resource
}
// InitRoutes configures all API routes for the application
@@ -233,6 +250,8 @@ func InitRoutes(app *gin.Engine) (err error) {
{
Method: http.MethodGet,
Path: "",
Name: "Get Project List",
Description: "Get a list of projects",
HandlerFunc: GetProjectList,
},
}...))
@@ -240,91 +259,127 @@ func InitRoutes(app *gin.Engine) (err error) {
{
Method: http.MethodGet,
Path: "/:id",
Name: "Get Spider by ID",
Description: "Get a single spider by ID",
HandlerFunc: GetSpiderById,
},
{
Method: http.MethodGet,
Path: "",
Name: "Get Spider List",
Description: "Get a list of spiders",
HandlerFunc: GetSpiderList,
},
{
Method: http.MethodPost,
Path: "",
Name: "Create Spider",
Description: "Create a new spider",
HandlerFunc: PostSpider,
},
{
Method: http.MethodPut,
Path: "/:id",
Name: "Update Spider by ID",
Description: "Update a spider by ID",
HandlerFunc: PutSpiderById,
},
{
Method: http.MethodDelete,
Path: "/:id",
Name: "Delete Spider by ID",
Description: "Delete a spider by ID",
HandlerFunc: DeleteSpiderById,
},
{
Method: http.MethodDelete,
Path: "",
Name: "Delete Spider List",
Description: "Delete a list of spiders",
HandlerFunc: DeleteSpiderList,
},
{
Method: http.MethodGet,
Path: "/:id/files/list",
Name: "Get Spider List Dir",
Description: "Get a list of files in a spider directory",
HandlerFunc: GetSpiderListDir,
},
{
Method: http.MethodGet,
Path: "/:id/files/get",
Name: "Get Spider File Content",
Description: "Get the content of a spider file",
HandlerFunc: GetSpiderFileContent,
},
{
Method: http.MethodGet,
Path: "/:id/files/info",
Name: "Get Spider File Info",
Description: "Get the info of a spider file",
HandlerFunc: GetSpiderFileInfo,
},
{
Method: http.MethodPost,
Path: "/:id/files/save",
Name: "Save Spider File",
Description: "Save a spider file",
HandlerFunc: PostSpiderSaveFile,
},
{
Method: http.MethodPost,
Path: "/:id/files/save/batch",
Name: "Save Spider Files",
Description: "Save multiple spider files",
HandlerFunc: PostSpiderSaveFiles,
},
{
Method: http.MethodPost,
Path: "/:id/files/save/dir",
Name: "Save Spider Dir",
Description: "Save a spider directory",
HandlerFunc: PostSpiderSaveDir,
},
{
Method: http.MethodPost,
Path: "/:id/files/rename",
Name: "Rename Spider File",
Description: "Rename a spider file",
HandlerFunc: PostSpiderRenameFile,
},
{
Method: http.MethodDelete,
Path: "/:id/files",
Name: "Delete Spider File",
Description: "Delete a spider file",
HandlerFunc: DeleteSpiderFile,
},
{
Method: http.MethodPost,
Path: "/:id/files/copy",
Name: "Copy Spider File",
Description: "Copy a spider file",
HandlerFunc: PostSpiderCopyFile,
},
{
Method: http.MethodPost,
Path: "/:id/files/export",
Name: "Export Spider Files",
Description: "Export spider files to a zip file",
HandlerFunc: PostSpiderExport,
},
{
Method: http.MethodPost,
Path: "/:id/run",
Name: "Run Spider",
Description: "Run a task for the given spider",
HandlerFunc: PostSpiderRun,
},
{
Method: http.MethodGet,
Path: "/:id/results",
Name: "Get Spider Results",
Description: "Get the results of a spider",
HandlerFunc: GetSpiderResults,
},
}...))
@@ -332,26 +387,36 @@ func InitRoutes(app *gin.Engine) (err error) {
{
Method: http.MethodPost,
Path: "",
Name: "Create Schedule",
Description: "Create a new schedule",
HandlerFunc: PostSchedule,
},
{
Method: http.MethodPut,
Path: "/:id",
Name: "Update Schedule by ID",
Description: "Update a schedule by ID",
HandlerFunc: PutScheduleById,
},
{
Method: http.MethodPost,
Path: "/:id/enable",
Name: "Enable Schedule",
Description: "Enable a schedule",
HandlerFunc: PostScheduleEnable,
},
{
Method: http.MethodPost,
Path: "/:id/disable",
Name: "Disable Schedule",
Description: "Disable a schedule",
HandlerFunc: PostScheduleDisable,
},
{
Method: http.MethodPost,
Path: "/:id/run",
Name: "Run Schedule",
Description: "Run a schedule",
HandlerFunc: PostScheduleRun,
},
}...))
@@ -359,41 +424,57 @@ func InitRoutes(app *gin.Engine) (err error) {
{
Method: http.MethodGet,
Path: "/:id",
Name: "Get Task by ID",
Description: "Get a single task by ID",
HandlerFunc: GetTaskById,
},
{
Method: http.MethodGet,
Path: "",
Name: "Get Task List",
Description: "Get a list of tasks",
HandlerFunc: GetTaskList,
},
{
Method: http.MethodDelete,
Path: "/:id",
Name: "Delete Task by ID",
Description: "Delete a task by ID",
HandlerFunc: DeleteTaskById,
},
{
Method: http.MethodDelete,
Path: "",
HandlerFunc: DeleteList,
Name: "Delete Task List",
Description: "Delete a list of tasks",
HandlerFunc: DeleteTaskList,
},
{
Method: http.MethodPost,
Path: "/run",
Name: "Run Task",
Description: "Run a task",
HandlerFunc: PostTaskRun,
},
{
Method: http.MethodPost,
Path: "/:id/restart",
Name: "Restart Task",
Description: "Restart a task",
HandlerFunc: PostTaskRestart,
},
{
Method: http.MethodPost,
Path: "/:id/cancel",
Name: "Cancel Task",
Description: "Cancel a task",
HandlerFunc: PostTaskCancel,
},
{
Method: http.MethodGet,
Path: "/:id/logs",
Name: "Get Task Logs",
Description: "Get the logs of a task",
HandlerFunc: GetTaskLogs,
},
}...))
@@ -401,11 +482,15 @@ func InitRoutes(app *gin.Engine) (err error) {
{
Method: http.MethodPost,
Path: "",
Name: "Create Token",
Description: "Create a new token",
HandlerFunc: PostToken,
},
{
Method: http.MethodGet,
Path: "",
Name: "Get Token List",
Description: "Get a list of tokens",
HandlerFunc: GetTokenList,
},
}...))
@@ -413,51 +498,71 @@ func InitRoutes(app *gin.Engine) (err error) {
{
Method: http.MethodGet,
Path: "/:id",
Name: "Get User by ID",
Description: "Get a single user by ID",
HandlerFunc: GetUserById,
},
{
Method: http.MethodGet,
Path: "",
Name: "Get User List",
Description: "Get a list of users",
HandlerFunc: GetUserList,
},
{
Method: http.MethodPost,
Path: "",
Name: "Create User",
Description: "Create a new user",
HandlerFunc: PostUser,
},
{
Method: http.MethodPut,
Path: "/:id",
Name: "Update User by ID",
Description: "Update a user by ID",
HandlerFunc: PutUserById,
},
{
Method: http.MethodPost,
Path: "/:id/change-password",
Name: "Change User Password",
Description: "Change a user's password",
HandlerFunc: PostUserChangePassword,
},
{
Method: http.MethodDelete,
Path: "/:id",
Name: "Delete User by ID",
Description: "Delete a user by ID",
HandlerFunc: DeleteUserById,
},
{
Method: http.MethodDelete,
Path: "",
Name: "Delete User List",
Description: "Delete a list of users",
HandlerFunc: DeleteUserList,
},
{
Method: http.MethodGet,
Path: "/me",
Name: "Get Me",
Description: "Get the current user",
HandlerFunc: GetUserMe,
},
{
Method: http.MethodPut,
Path: "/me",
Name: "Update Me",
Description: "Update the current user",
HandlerFunc: PutUserMe,
},
{
Method: http.MethodPost,
Path: "/me/change-password",
Name: "Change My Password",
Description: "Change the current user's password",
HandlerFunc: PostUserMeChangePassword,
},
}...))
@@ -467,16 +572,22 @@ func InitRoutes(app *gin.Engine) (err error) {
{
Method: http.MethodPost,
Path: "/:type",
Name: "Export Data",
Description: "Export data",
HandlerFunc: PostExport,
},
{
Method: http.MethodGet,
Path: "/:type/:id",
Name: "Get Export",
Description: "Get an export",
HandlerFunc: GetExport,
},
{
Method: http.MethodGet,
Path: "/:type/:id/download",
Name: "Get Export Download",
Description: "Get an export download",
HandlerFunc: GetExportDownload,
},
})
@@ -484,16 +595,22 @@ func InitRoutes(app *gin.Engine) (err error) {
{
Method: http.MethodGet,
Path: "/:col",
Name: "Get Filter Column Field Options",
Description: "Get the field options of a collection",
HandlerFunc: GetFilterColFieldOptions,
},
{
Method: http.MethodGet,
Path: "/:col/:value",
Name: "Get Filter Col Field Options With Value",
Description: "Get the field options of a collection with a value",
HandlerFunc: GetFilterColFieldOptionsWithValue,
},
{
Method: http.MethodGet,
Path: "/:col/:value/:label",
Name: "Get Filter Col Field Options With Value And Label",
Description: "Get the field options of a collection with a value and label",
HandlerFunc: GetFilterColFieldOptionsWithValueLabel,
},
})
@@ -501,16 +618,22 @@ func InitRoutes(app *gin.Engine) (err error) {
{
Method: http.MethodGet,
Path: "/:key",
Name: "Get Setting",
Description: "Get a setting",
HandlerFunc: GetSetting,
},
{
Method: http.MethodPost,
Path: "/:key",
Name: "Create Setting",
Description: "Create a new setting",
HandlerFunc: PostSetting,
},
{
Method: http.MethodPut,
Path: "/:key",
Name: "Update Setting",
Description: "Update a setting",
HandlerFunc: PutSetting,
},
})
@@ -518,16 +641,22 @@ func InitRoutes(app *gin.Engine) (err error) {
{
Method: http.MethodGet,
Path: "/overview",
Name: "Get Stats Overview",
Description: "Get the overview of the stats",
HandlerFunc: GetStatsOverview,
},
{
Method: http.MethodGet,
Path: "/daily",
Name: "Get Stats Daily",
Description: "Get the daily stats",
HandlerFunc: GetStatsDaily,
},
{
Method: http.MethodGet,
Path: "/tasks",
Name: "Get Stats Tasks",
Description: "Get the tasks stats",
HandlerFunc: GetStatsTasks,
},
})
@@ -537,6 +666,8 @@ func InitRoutes(app *gin.Engine) (err error) {
{
Path: "",
Method: http.MethodGet,
Name: "Get System Info",
Description: "Get the system info",
HandlerFunc: GetSystemInfo,
},
})
@@ -544,11 +675,15 @@ func InitRoutes(app *gin.Engine) (err error) {
{
Method: http.MethodPost,
Path: "/login",
Name: "Login",
Description: "Login",
HandlerFunc: PostLogin,
},
{
Method: http.MethodPost,
Path: "/logout",
Name: "Logout",
Description: "Logout",
HandlerFunc: PostLogout,
},
})

20
core/controllers/schedule.go
View File

@@ -14,7 +14,7 @@ import (
)
type PostScheduleParams struct {
Data models.Schedule `json:"data"`
Data models.Schedule `json:"data" description:"The data to create" validate:"required"`
}
func PostSchedule(c *gin.Context, params *PostScheduleParams) (response *Response[models.Schedule], err error) {
@@ -42,8 +42,8 @@ func PostSchedule(c *gin.Context, params *PostScheduleParams) (response *Respons
}
type PutScheduleByIdParams struct {
Id string `path:"id"`
Data models.Schedule `json:"data"`
Id string `path:"id" description:"Schedule ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Data models.Schedule `json:"data" description:"The data to update" validate:"required"`
}
func PutScheduleById(c *gin.Context, params *PutScheduleByIdParams) (response *Response[models.Schedule], err error) {
@@ -80,7 +80,7 @@ func PutScheduleById(c *gin.Context, params *PutScheduleByIdParams) (response *R
}
type PostScheduleEnableDisableParams struct {
Id string `path:"id"`
Id string `path:"id" description:"Schedule ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
}
func PostScheduleEnable(c *gin.Context, params *PostScheduleEnableDisableParams) (response *VoidResponse, err error) {
@@ -118,12 +118,12 @@ func postScheduleEnableDisableFunc(isEnable bool, userId primitive.ObjectID, par
}
type PostScheduleRunParams struct {
Id string `path:"id"`
Mode string `json:"mode"`
NodeIds []string `json:"node_ids"`
Cmd string `json:"cmd"`
Param string `json:"param"`
Priority int `json:"priority"`
Id string `path:"id" description:"Schedule ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Mode string `json:"mode" description:"Run mode" enum:"random,all,selected-nodes"`
NodeIds []string `json:"node_ids" description:"Node IDs" items.type:"string" items.format:"objectid" items.pattern:"^[0-9a-fA-F]{24}$"`
Cmd string `json:"cmd" description:"Command"`
Param string `json:"param" description:"Parameters"`
Priority int `json:"priority" description:"Priority" default:"5" minimum:"1" maximum:"10"`
}
func PostScheduleRun(c *gin.Context, params *PostScheduleRunParams) (response *Response[[]primitive.ObjectID], err error) {

10
core/controllers/setting.go
View File

@@ -9,7 +9,7 @@ import (
)
type GetSettingParams struct {
Key string `path:"key" validate:"required"`
Key string `path:"key" description:"Setting key" validate:"required"`
}
func GetSetting(_ *gin.Context, params *GetSettingParams) (response *Response[models.Setting], err error) {
@@ -26,8 +26,8 @@ func GetSetting(_ *gin.Context, params *GetSettingParams) (response *Response[mo
}
type PostSettingParams struct {
Key string `path:"key" validate:"required"`
Data models.Setting `json:"data"`
Key string `path:"key" description:"Setting key" validate:"required"`
Data models.Setting `json:"data" description:"The data to create" validate:"required"`
}
func PostSetting(c *gin.Context, params *PostSettingParams) (response *Response[models.Setting], err error) {
@@ -50,8 +50,8 @@ func PostSetting(c *gin.Context, params *PostSettingParams) (response *Response[
}
type PutSettingParams struct {
Key string `path:"key" validate:"required"`
Data models.Setting `json:"data"`
Key string `path:"key" description:"Setting key" validate:"required"`
Data models.Setting `json:"data" description:"The data to update" validate:"required"`
}
func PutSetting(c *gin.Context, params *PutSettingParams) (response *Response[models.Setting], err error) {

64
core/controllers/spider.go
View File

@@ -535,8 +535,8 @@ func DeleteSpiderList(_ *gin.Context, params *DeleteSpiderListParams) (response
}
type GetSpiderListDirParams struct {
Id string `path:"id"`
Path string `query:"path"`
Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Path string `query:"path" description:"Directory path"`
}
func GetSpiderListDir(c *gin.Context, params *GetSpiderListDirParams) (response *Response[[]interfaces.FsFileInfo], err error) {
@@ -548,8 +548,8 @@ func GetSpiderListDir(c *gin.Context, params *GetSpiderListDirParams) (response
}
type GetSpiderFileContentParams struct {
Id string `path:"id"`
Path string `query:"path"`
Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Path string `query:"path" description:"File path"`
}
func GetSpiderFileContent(c *gin.Context, params *GetSpiderFileContentParams) (response *Response[string], err error) {
@@ -561,8 +561,8 @@ func GetSpiderFileContent(c *gin.Context, params *GetSpiderFileContentParams) (r
}
type GetSpiderFileInfoParams struct {
Id string `path:"id"`
Path string `query:"path"`
Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Path string `query:"path" description:"File path"`
}
func GetSpiderFileInfo(c *gin.Context, params *GetSpiderFileInfoParams) (response *Response[interfaces.FsFileInfo], err error) {
@@ -574,9 +574,9 @@ func GetSpiderFileInfo(c *gin.Context, params *GetSpiderFileInfoParams) (respons
}
type PostSpiderSaveFileParams struct {
Id string `path:"id"`
Path string `json:"path"`
Data string `json:"data"`
Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Path string `json:"path" description:"File path to save"`
Data string `json:"data" description:"File content"`
File *multipart.FileHeader `form:"file"`
}
@@ -593,8 +593,8 @@ func PostSpiderSaveFile(c *gin.Context, params *PostSpiderSaveFileParams) (respo
}
type PostSpiderSaveFilesParams struct {
Id string `path:"id"`
TargetDirectory string `form:"targetDirectory"`
Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
TargetDirectory string `form:"targetDirectory" description:"Target directory path"`
}
func PostSpiderSaveFiles(c *gin.Context, params *PostSpiderSaveFilesParams) (response *VoidResponse, err error) {
@@ -610,8 +610,8 @@ func PostSpiderSaveFiles(c *gin.Context, params *PostSpiderSaveFilesParams) (res
}
type PostSpiderSaveDirParams struct {
Id string `path:"id"`
Path string `json:"path"`
Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Path string `json:"path" description:"File path to save"`
}
func PostSpiderSaveDir(c *gin.Context, params *PostSpiderSaveDirParams) (response *VoidResponse, err error) {
@@ -623,9 +623,9 @@ func PostSpiderSaveDir(c *gin.Context, params *PostSpiderSaveDirParams) (respons
}
type PostSpiderRenameFileParams struct {
Id string `path:"id"`
Path string `json:"path"`
NewPath string `json:"newPath"`
Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Path string `json:"path" description:"File path to rename"`
NewPath string `json:"newPath" description:"New file path"`
}
func PostSpiderRenameFile(c *gin.Context, params *PostSpiderRenameFileParams) (response *VoidResponse, err error) {
@@ -637,8 +637,8 @@ func PostSpiderRenameFile(c *gin.Context, params *PostSpiderRenameFileParams) (r
}
type DeleteSpiderFileParams struct {
Id string `path:"id"`
Path string `json:"path"`
Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Path string `json:"path" description:"File path to delete"`
}
func DeleteSpiderFile(c *gin.Context, params *DeleteSpiderFileParams) (response *VoidResponse, err error) {
@@ -650,9 +650,9 @@ func DeleteSpiderFile(c *gin.Context, params *DeleteSpiderFileParams) (response
}
type PostSpiderCopyFileParams struct {
Id string `path:"id"`
Path string `json:"path"`
NewPath string `json:"new_path"`
Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Path string `json:"path" description:"File path to copy"`
NewPath string `json:"new_path" description:"New file path"`
}
func PostSpiderCopyFile(c *gin.Context, params *PostSpiderCopyFileParams) (response *VoidResponse, err error) {
@@ -664,7 +664,7 @@ func PostSpiderCopyFile(c *gin.Context, params *PostSpiderCopyFileParams) (respo
}
type PostSpiderExportParams struct {
Id string `path:"id"`
Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
}
func PostSpiderExport(c *gin.Context, _ *PostSpiderExportParams) (err error) {
@@ -676,13 +676,13 @@ func PostSpiderExport(c *gin.Context, _ *PostSpiderExportParams) (err error) {
}
type PostSpiderRunParams struct {
Id string `path:"id"`
Mode string `json:"mode"`
NodeIds []string `json:"node_ids"`
Cmd string `json:"cmd"`
Param string `json:"param"`
ScheduleId string `json:"schedule_id"`
Priority int `json:"priority"`
Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Mode string `json:"mode" description:"Run mode" enum:"random,all,selected-nodes"`
NodeIds []string `json:"node_ids" description:"Node IDs, used in selected-nodes mode"`
Cmd string `json:"cmd" description:"Command"`
Param string `json:"param" description:"Parameters"`
ScheduleId string `json:"schedule_id" description:"Schedule ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Priority int `json:"priority" description:"Priority" default:"5" minimum:"1" maximum:"10"`
}
func PostSpiderRun(c *gin.Context, params *PostSpiderRunParams) (response *Response[[]primitive.ObjectID], err error) {
@@ -733,9 +733,9 @@ func PostSpiderRun(c *gin.Context, params *PostSpiderRunParams) (response *Respo
}
type GetSpiderResultsParams struct {
Id string `path:"id"`
Page int `query:"page"`
Size int `query:"size"`
Id string `path:"id" description:"Spider ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Page int `query:"page" description:"Page" default:"1" minimum:"1"`
Size int `query:"size" description:"Size" default:"10" minimum:"1"`
}
func GetSpiderResults(c *gin.Context, params *GetSpiderResultsParams) (response *ListResponse[bson.M], err error) {

6
core/controllers/stats.go
View File

@@ -15,7 +15,7 @@ var statsDefaultQuery = bson.M{
}
type GetStatsOverviewParams struct {
Query bson.M `json:"query"`
Query bson.M `json:"query" description:"Query"`
}
func GetStatsOverview(_ *gin.Context, params *GetStatsOverviewParams) (response *Response[bson.M], err error) {
@@ -32,7 +32,7 @@ func GetStatsOverview(_ *gin.Context, params *GetStatsOverviewParams) (response
}
type GetStatsDailyParams struct {
Query bson.M `json:"query"`
Query bson.M `json:"query" description:"Query"`
}
func GetStatsDaily(_ *gin.Context, params *GetStatsDailyParams) (response *Response[bson.M], err error) {
@@ -49,7 +49,7 @@ func GetStatsDaily(_ *gin.Context, params *GetStatsDailyParams) (response *Respo
}
type GetStatsTasksParams struct {
Query bson.M `json:"query"`
Query bson.M `json:"query" description:"Query"`
}
func GetStatsTasks(_ *gin.Context, params *GetStatsTasksParams) (response *Response[bson.M], err error) {

18
core/controllers/task.go
View File

@@ -23,7 +23,7 @@ import (
)
type GetTaskByIdParams struct {
Id string `path:"id"`
Id string `path:"id" description:"Task ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
}
func GetTaskById(_ *gin.Context, params *GetTaskByIdParams) (response *Response[models.Task], err error) {
@@ -172,7 +172,7 @@ func GetTaskList(c *gin.Context, params *GetTaskListParams) (response *ListRespo
}
type DeleteTaskByIdParams struct {
Id string `path:"id"`
Id string `path:"id" description:"Task ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
}
func DeleteTaskById(_ *gin.Context, params *DeleteTaskByIdParams) (response *VoidResponse, err error) {
@@ -223,7 +223,7 @@ type DeleteTaskListParams struct {
Ids []string `json:"ids"`
}
func DeleteList(_ *gin.Context, params *DeleteTaskListParams) (response *VoidResponse, err error) {
func DeleteTaskList(_ *gin.Context, params *DeleteTaskListParams) (response *VoidResponse, err error) {
var ids []primitive.ObjectID
for _, id := range params.Ids {
id, err := primitive.ObjectIDFromHex(id)
@@ -333,7 +333,7 @@ func PostTaskRun(c *gin.Context, params *PostTaskRunParams) (response *Response[
}
type PostTaskRestartParams struct {
Id string `path:"id"`
Id string `path:"id" description:"Task ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
}
func PostTaskRestart(c *gin.Context, params *PostTaskRestartParams) (response *Response[[]primitive.ObjectID], err error) {
@@ -380,8 +380,8 @@ func PostTaskRestart(c *gin.Context, params *PostTaskRestartParams) (response *R
}
type PostTaskCancelParams struct {
Id string `path:"id"`
Force bool `json:"force,omitempty"`
Id string `path:"id" description:"Task ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Force bool `json:"force,omitempty" description:"Force cancel" default:"false"`
}
func PostTaskCancel(c *gin.Context, params *PostTaskCancelParams) (response *VoidResponse, err error) {
@@ -415,9 +415,9 @@ func PostTaskCancel(c *gin.Context, params *PostTaskCancelParams) (response *Voi
}
type GetTaskLogsParams struct {
Id string `path:"id"`
Page int `query:"page"`
Size int `query:"size"`
Id string `path:"id" description:"Task ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Page int `query:"page" description:"Page" default:"1" minimum:"1"`
Size int `query:"size" description:"Size" default:"10" minimum:"1"`
}
func GetTaskLogs(_ *gin.Context, params *GetTaskLogsParams) (response *ListResponse[string], err error) {

2
core/controllers/token.go
View File

@@ -11,7 +11,7 @@ import (
)
type PostTokenParams struct {
Data models.Token `json:"data"`
Data models.Token `json:"data" description:"The data to create" validate:"required"`
}
func PostToken(c *gin.Context, params *PostTokenParams) (response *Response[models.Token], err error) {

29
core/controllers/user.go
View File

@@ -89,11 +89,11 @@ func GetUserList(_ *gin.Context, params *GetListParams) (response *ListResponse[
}
type PostUserParams struct {
Username string `json:"username" validate:"required"`
Password string `json:"password" validate:"required"`
Role string `json:"role"`
RoleId primitive.ObjectID `json:"role_id"`
Email string `json:"email"`
Username string `json:"username" description:"Username" validate:"required"`
Password string `json:"password" description:"Password" validate:"required"`
Role string `json:"role" description:"Role"`
RoleId string `json:"role_id" description:"Role ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Email string `json:"email" description:"Email"`
}
func PostUser(c *gin.Context, params *PostUserParams) (response *Response[models.User], err error) {
@@ -105,8 +105,13 @@ func PostUser(c *gin.Context, params *PostUserParams) (response *Response[models
}
}
if !params.RoleId.IsZero() {
_, err := service.NewModelService[models.Role]().GetById(params.RoleId)
var roleId primitive.ObjectID
if params.RoleId != "" {
roleId, err = primitive.ObjectIDFromHex(params.RoleId)
if err != nil {
return GetErrorResponse[models.User](errors.BadRequestf("invalid role id: %v", err))
}
_, err = service.NewModelService[models.Role]().GetById(roleId)
if err != nil {
return GetErrorResponse[models.User](errors.BadRequestf("role not found: %v", err))
}
@@ -116,7 +121,7 @@ func PostUser(c *gin.Context, params *PostUserParams) (response *Response[models
Username: params.Username,
Password: utils.EncryptMd5(params.Password),
Role: params.Role,
RoleId: params.RoleId,
RoleId: roleId,
Email: params.Email,
}
model.SetCreated(u.Id)
@@ -143,12 +148,12 @@ func PutUserById(c *gin.Context, params *PutByIdParams[models.User]) (response *
}
type PostUserChangePasswordParams struct {
Id string `path:"id"`
Password string `json:"password" validate:"required"`
Id string `path:"id" description:"User ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"`
Password string `json:"password" description:"Password" validate:"required"`
}
func PostUserChangePassword(c *gin.Context, params *PostUserChangePasswordParams) (response *Response[models.User], err error) {
id, err := primitive.ObjectIDFromHex(c.Param("id"))
id, err := primitive.ObjectIDFromHex(params.Id)
if err != nil {
return GetErrorResponse[models.User](errors.BadRequestf("invalid user id: %v", err))
}
@@ -228,7 +233,7 @@ func PutUserMe(c *gin.Context, params *PutUserMeParams) (response *Response[mode
}
type PostUserMeChangePasswordParams struct {
Password string `json:"password" validate:"required"`
Password string `json:"password" description:"Password" validate:"required"`
}
func PostUserMeChangePassword(c *gin.Context, params *PostUserMeChangePasswordParams) (response *Response[models.User], err error) {

10
core/models/models/base.go
View File

@@ -8,11 +8,11 @@ import (
)
type BaseModel struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
CreatedAt time.Time `json:"created_ts,omitempty" bson:"created_ts,omitempty"`
CreatedBy primitive.ObjectID `json:"created_by,omitempty" bson:"created_by,omitempty"`
UpdatedAt time.Time `json:"updated_ts,omitempty" bson:"updated_ts,omitempty"`
UpdatedBy primitive.ObjectID `json:"updated_by,omitempty" bson:"updated_by,omitempty"`
Id primitive.ObjectID `json:"_id" bson:"_id" description:"ID"`
CreatedAt time.Time `json:"created_ts,omitempty" bson:"created_ts,omitempty" description:"Created timestamp"`
CreatedBy primitive.ObjectID `json:"created_by,omitempty" bson:"created_by,omitempty" description:"Created by"`
UpdatedAt time.Time `json:"updated_ts,omitempty" bson:"updated_ts,omitempty" description:"Updated timestamp"`
UpdatedBy primitive.ObjectID `json:"updated_by,omitempty" bson:"updated_by,omitempty" description:"Updated by"`
}
func (m *BaseModel) GetId() primitive.ObjectID {

62
core/models/models/database.go
View File

@@ -7,42 +7,42 @@ import (
type Database struct {
any `collection:"databases"`
BaseModel `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
DataSource string `json:"data_source" bson:"data_source"`
Host string `json:"host" bson:"host"`
Port int `json:"port" bson:"port"`
URI string `json:"uri,omitempty" bson:"uri,omitempty"`
Database string `json:"database,omitempty" bson:"database,omitempty"`
Username string `json:"username,omitempty" bson:"username,omitempty"`
Password string `json:"password,omitempty" bson:"-"`
EncryptedPassword string `json:"-,omitempty" bson:"encrypted_password,omitempty"`
Status string `json:"status" bson:"status"`
Error string `json:"error" bson:"error"`
Active bool `json:"active" bson:"active"`
ActiveAt time.Time `json:"active_ts" bson:"active_ts"`
IsDefault bool `json:"is_default" bson:"-"`
Name string `json:"name" bson:"name" description:"Name"`
Description string `json:"description" bson:"description" description:"Description"`
DataSource string `json:"data_source" bson:"data_source" description:"Data source"`
Host string `json:"host" bson:"host" description:"Host"`
Port int `json:"port" bson:"port" description:"Port"`
URI string `json:"uri,omitempty" bson:"uri,omitempty" description:"URI"`
Database string `json:"database,omitempty" bson:"database,omitempty" description:"Database"`
Username string `json:"username,omitempty" bson:"username,omitempty" description:"Username"`
Password string `json:"password,omitempty" bson:"-" binding:"-"`
EncryptedPassword string `json:"-,omitempty" bson:"encrypted_password,omitempty" description:"Encrypted password"`
Status string `json:"status" bson:"status" description:"Status"`
Error string `json:"error" bson:"error" description:"Error"`
Active bool `json:"active" bson:"active" description:"Active"`
ActiveAt time.Time `json:"active_ts" bson:"active_ts" description:"Active at"`
IsDefault bool `json:"is_default" bson:"-" binding:"-"`
MongoParams *struct {
AuthSource string `json:"auth_source,omitempty" bson:"auth_source,omitempty"`
AuthMechanism string `json:"auth_mechanism,omitempty" bson:"auth_mechanism,omitempty"`
} `json:"mongo_params,omitempty" bson:"mongo_params,omitempty"`
AuthSource string `json:"auth_source,omitempty" bson:"auth_source,omitempty" description:"Auth source"`
AuthMechanism string `json:"auth_mechanism,omitempty" bson:"auth_mechanism,omitempty" description:"Auth mechanism"`
} `json:"mongo_params,omitempty" bson:"mongo_params,omitempty" description:"Mongo params"`
PostgresParams *struct {
SSLMode string `json:"ssl_mode,omitempty" bson:"ssl_mode,omitempty"`
} `json:"postgres_params,omitempty" bson:"postgres_params,omitempty"`
SSLMode string `json:"ssl_mode,omitempty" bson:"ssl_mode,omitempty" description:"SSL mode"`
} `json:"postgres_params,omitempty" bson:"postgres_params,omitempty" description:"Postgres params"`
SnowflakeParams *struct {
Account string `json:"account,omitempty" bson:"account,omitempty"`
Schema string `json:"schema,omitempty" bson:"schema,omitempty"`
Warehouse string `json:"warehouse,omitempty" bson:"warehouse,omitempty"`
Role string `json:"role,omitempty" bson:"role,omitempty"`
} `json:"snowflake_params,omitempty" bson:"snowflake_params,omitempty"`
Account string `json:"account,omitempty" bson:"account,omitempty" description:"Account"`
Schema string `json:"schema,omitempty" bson:"schema,omitempty" description:"Schema"`
Warehouse string `json:"warehouse,omitempty" bson:"warehouse,omitempty" description:"Warehouse"`
Role string `json:"role,omitempty" bson:"role,omitempty" description:"Role"`
} `json:"snowflake_params,omitempty" bson:"snowflake_params,omitempty" description:"Snowflake params"`
CassandraParams *struct {
Keyspace string `json:"keyspace,omitempty" bson:"keyspace,omitempty"`
} `json:"cassandra_params,omitempty" bson:"cassandra_params,omitempty"`
Keyspace string `json:"keyspace,omitempty" bson:"keyspace,omitempty" description:"Keyspace"`
} `json:"cassandra_params,omitempty" bson:"cassandra_params,omitempty" description:"Cassandra params"`
HiveParams *struct {
Auth string `json:"auth,omitempty" bson:"auth,omitempty"`
} `json:"hive_params,omitempty" bson:"hive_params,omitempty"`
Auth string `json:"auth,omitempty" bson:"auth,omitempty" description:"Auth"`
} `json:"hive_params,omitempty" bson:"hive_params,omitempty" description:"Hive params"`
RedisParams *struct {
DB int `json:"db,omitempty" bson:"db,omitempty"`
} `json:"redis_params,omitempty" bson:"redis_params,omitempty"`
DB int `json:"db,omitempty" bson:"db,omitempty" description:"DB"`
} `json:"redis_params,omitempty" bson:"redis_params,omitempty" description:"Redis params"`
}

32
core/models/models/database_metric.go
View File

@@ -5,20 +5,20 @@ import "go.mongodb.org/mongo-driver/bson/primitive"
type DatabaseMetric struct {
any `collection:"database_metrics"`
BaseModel `bson:",inline"`
DatabaseId primitive.ObjectID `json:"database_id" bson:"database_id"`
CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"`
TotalMemory uint64 `json:"total_memory" bson:"total_memory"`
AvailableMemory uint64 `json:"available_memory" bson:"available_memory"`
UsedMemory uint64 `json:"used_memory" bson:"used_memory"`
UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"`
TotalDisk uint64 `json:"total_disk" bson:"total_disk"`
AvailableDisk uint64 `json:"available_disk" bson:"available_disk"`
UsedDisk uint64 `json:"used_disk" bson:"used_disk"`
UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"`
Connections int `json:"connections" bson:"connections"`
QueryPerSecond float64 `json:"query_per_second" bson:"query_per_second"`
TotalQuery uint64 `json:"total_query,omitempty" bson:"total_query,omitempty"`
CacheHitRatio float64 `json:"cache_hit_ratio" bson:"cache_hit_ratio"`
ReplicationLag float64 `json:"replication_lag" bson:"replication_lag"`
LockWaitTime float64 `json:"lock_wait_time" bson:"lock_wait_time"`
DatabaseId primitive.ObjectID `json:"database_id" bson:"database_id" description:"Database ID"`
CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent" description:"CPU usage percentage"`
TotalMemory uint64 `json:"total_memory" bson:"total_memory" description:"Total memory"`
AvailableMemory uint64 `json:"available_memory" bson:"available_memory" description:"Available memory"`
UsedMemory uint64 `json:"used_memory" bson:"used_memory" description:"Used memory"`
UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent" description:"Used memory percentage"`
TotalDisk uint64 `json:"total_disk" bson:"total_disk" description:"Total disk"`
AvailableDisk uint64 `json:"available_disk" bson:"available_disk" description:"Available disk"`
UsedDisk uint64 `json:"used_disk" bson:"used_disk" description:"Used disk"`
UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent" description:"Used disk percentage"`
Connections int `json:"connections" bson:"connections" description:"Connections"`
QueryPerSecond float64 `json:"query_per_second" bson:"query_per_second" description:"Query per second"`
TotalQuery uint64 `json:"total_query,omitempty" bson:"total_query,omitempty" description:"Total query"`
CacheHitRatio float64 `json:"cache_hit_ratio" bson:"cache_hit_ratio" description:"Cache hit ratio"`
ReplicationLag float64 `json:"replication_lag" bson:"replication_lag" description:"Replication lag"`
LockWaitTime float64 `json:"lock_wait_time" bson:"lock_wait_time" description:"Lock wait time"`
}

18
core/models/models/dependency.go
View File

@@ -7,13 +7,13 @@ import (
type Dependency struct {
any `collection:"dependencies"`
BaseModel `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
Type string `json:"type" bson:"type"`
Version string `json:"version" bson:"version"`
Status string `json:"status" bson:"status"`
Error string `json:"error,omitempty" bson:"error,omitempty"`
NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-"`
Versions []string `json:"versions,omitempty" bson:"-"`
Name string `json:"name" bson:"name" description:"Name"`
Description string `json:"description" bson:"description" description:"Description"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id" description:"Node ID"`
Type string `json:"type" bson:"type" description:"Type"`
Version string `json:"version" bson:"version" description:"Version"`
Status string `json:"status" bson:"status" description:"Status"`
Error string `json:"error,omitempty" bson:"error,omitempty" description:"Error"`
NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-" binding:"-"`
Versions []string `json:"versions,omitempty" bson:"-" binding:"-"`
}

16
core/models/models/dependency_config.go
View File

@@ -3,12 +3,12 @@ package models
type DependencyConfig struct {
any `collection:"dependency_configs"`
BaseModel `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
ExecCmd string `json:"exec_cmd" bson:"exec_cmd"`
PkgCmd string `json:"pkg_cmd" bson:"pkg_cmd"`
PkgSrcURL string `json:"pkg_src_url" bson:"pkg_src_url"`
Setup bool `json:"setup" bson:"-"`
TotalDependencies int `json:"total_dependencies" bson:"-"`
SearchReady bool `json:"search_ready" bson:"-"`
Key string `json:"key" bson:"key" description:"Key"`
Name string `json:"name" bson:"name" description:"Name"`
ExecCmd string `json:"exec_cmd" bson:"exec_cmd" description:"Exec cmd"`
PkgCmd string `json:"pkg_cmd" bson:"pkg_cmd" description:"Pkg cmd"`
PkgSrcURL string `json:"pkg_src_url" bson:"pkg_src_url" description:"Pkg src url"`
Setup bool `json:"setup" bson:"-" binding:"-"`
TotalDependencies int `json:"total_dependencies" bson:"-" binding:"-"`
SearchReady bool `json:"search_ready" bson:"-" binding:"-"`
}

18
core/models/models/dependency_config_setup.go
View File

@@ -5,15 +5,15 @@ import "go.mongodb.org/mongo-driver/bson/primitive"
type DependencyConfigSetup struct {
any `collection:"dependency_config_setups"`
BaseModel `bson:",inline"`
DependencyConfigId primitive.ObjectID `json:"dependency_config_id" bson:"dependency_config_id"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
Version string `json:"version" bson:"version"`
Drivers []DependencyDriver `json:"versions,omitempty" bson:"versions,omitempty"`
Status string `json:"status" bson:"status"`
Error string `json:"error,omitempty" bson:"error,omitempty"`
Node *Node `json:"node,omitempty" bson:"-"`
DependencyConfigId primitive.ObjectID `json:"dependency_config_id" bson:"dependency_config_id" description:"Dependency config ID"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id" description:"Node ID"`
Version string `json:"version" bson:"version" description:"Version"`
Drivers []DependencyDriver `json:"versions,omitempty" bson:"versions,omitempty" description:"Versions"`
Status string `json:"status" bson:"status" description:"Status"`
Error string `json:"error,omitempty" bson:"error,omitempty" description:"Error"`
Node *Node `json:"node,omitempty" bson:"-" binding:"-"`
}
type DependencyDriver struct {
Name string `json:"name" bson:"name"`
Version string `json:"version" bson:"version"`
Name string `json:"name" bson:"name" description:"Name"`
Version string `json:"version" bson:"version" description:"Version"`
}

4
core/models/models/dependency_log.go
View File

@@ -5,6 +5,6 @@ import "go.mongodb.org/mongo-driver/bson/primitive"
type DependencyLog struct {
any `collection:"dependency_logs"`
BaseModel `bson:",inline"`
TargetId primitive.ObjectID `json:"target_id" bson:"target_id"`
Content string `json:"content" bson:"content"`
TargetId primitive.ObjectID `json:"target_id" bson:"target_id" description:"Target ID"`
Content string `json:"content" bson:"content" description:"Content"`
}

6
core/models/models/dependency_pypi_project.go
View File

@@ -3,7 +3,7 @@ package models
type DependencyPypiProject struct {
any `collection:"dependency_pypi_projects"`
BaseModel `bson:",inline"`
Name string `json:"name" bson:"name"`
Version string `json:"version" bson:"version"`
LastSerial int `json:"_last-serial" bson:"last_serial"`
Name string `json:"name" bson:"name" description:"Name"`
Version string `json:"version" bson:"version" description:"Version"`
LastSerial int `json:"_last-serial" bson:"last_serial" description:"Last serial"`
}

8
core/models/models/dependency_repo.go
View File

@@ -3,8 +3,8 @@ package models
type DependencyRepo struct {
any `collection:"dependency_repos"`
BaseModel `bson:",inline"`
Name string `json:"name" bson:"name"`
Type string `json:"type" bson:"type"`
LatestVersion string `json:"latest_version" bson:"latest_version"`
AllVersions []string `json:"all_versions" bson:"all_versions"`
Name string `json:"name" bson:"name" description:"Name"`
Type string `json:"type" bson:"type" description:"Type"`
LatestVersion string `json:"latest_version" bson:"latest_version" description:"Latest version"`
AllVersions []string `json:"all_versions" bson:"all_versions" description:"All versions"`
}

4
core/models/models/environment.go
View File

@@ -3,6 +3,6 @@ package models
type Environment struct {
any `collection:"environments"`
BaseModel `bson:",inline"`
Key string `json:"key" bson:"key"`
Value string `json:"value" bson:"value"`
Key string `json:"key" bson:"key" description:"Key"`
Value string `json:"value" bson:"value" description:"Value"`
}

29
core/models/models/git.go
View File

@@ -1,26 +1,27 @@
package models
import (
"github.com/crawlab-team/crawlab/vcs"
"time"
"github.com/crawlab-team/crawlab/vcs"
)
type Git struct {
any `collection:"gits"`
BaseModel `bson:",inline"`
Url string `json:"url" bson:"url"`
Name string `json:"name" bson:"name"`
AuthType string `json:"auth_type" bson:"auth_type"`
Username string `json:"username" bson:"username"`
Password string `json:"password" bson:"password"`
CurrentBranch string `json:"current_branch" bson:"current_branch"`
Status string `json:"status" bson:"status"`
Error string `json:"error" bson:"error"`
Spiders []Spider `json:"spiders,omitempty" bson:"-"`
Refs []vcs.GitRef `json:"refs" bson:"refs"`
RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"`
CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"`
Url string `json:"url" bson:"url" description:"URL"`
Name string `json:"name" bson:"name" description:"Name"`
AuthType string `json:"auth_type" bson:"auth_type" description:"Auth type"`
Username string `json:"username" bson:"username" description:"Username"`
Password string `json:"password" bson:"password" description:"Password"`
CurrentBranch string `json:"current_branch" bson:"current_branch" description:"Current branch"`
Status string `json:"status" bson:"status" description:"Status"`
Error string `json:"error" bson:"error" description:"Error"`
Spiders []Spider `json:"spiders,omitempty" bson:"-" binding:"-"`
Refs []vcs.GitRef `json:"refs" bson:"refs" description:"Refs"`
RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at" description:"Refs updated at"`
CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs" description:"Clone logs"`
// settings
AutoPull bool `json:"auto_pull" bson:"auto_pull"`
AutoPull bool `json:"auto_pull" bson:"auto_pull" description:"Auto pull"`
}

18
core/models/models/llm_provider.go
View File

@@ -4,15 +4,15 @@ package models
type LLMProvider struct {
any `collection:"llm_providers"`
BaseModel `bson:",inline"`
Key string `json:"key" bson:"key"` // Provider key (e.g., "openai", "anthropic", "gemini")
Name string `json:"name" bson:"name"` // Display name for UI
Enabled bool `json:"enabled" bson:"enabled"` // Whether this provider is enabled
ApiKey string `json:"api_key" bson:"api_key"` // API key for the provider
ApiBaseUrl string `json:"api_base_url" bson:"api_base_url"` // API base URL for the provider
DeploymentName string `json:"deployment_name" bson:"deployment_name"` // Deployment name for the provider
ApiVersion string `json:"api_version" bson:"api_version"` // API version for the provider
Models []string `json:"models" bson:"models"` // Models supported by this provider
Unset bool `json:"unset" bson:"-"` // Whether the provider is unset
Key string `json:"key" bson:"key" description:"Provider key (e.g., 'openai', 'anthropic', 'gemini')"`
Name string `json:"name" bson:"name" description:"Display name for UI"`
Enabled bool `json:"enabled" bson:"enabled" description:"Whether this provider is enabled"`
ApiKey string `json:"api_key" bson:"api_key" description:"API key for the provider"`
ApiBaseUrl string `json:"api_base_url" bson:"api_base_url" description:"API base URL for the provider"`
DeploymentName string `json:"deployment_name" bson:"deployment_name" description:"Deployment name for the provider"`
ApiVersion string `json:"api_version" bson:"api_version" description:"API version for the provider"`
Models []string `json:"models" bson:"models" description:"Models supported by this provider"`
Unset bool `json:"unset" bson:"-"` // Whether the provider is unset
}
func (p *LLMProvider) IsUnset() bool {

30
core/models/models/metric.go
View File

@@ -7,19 +7,19 @@ import (
type Metric struct {
any `collection:"metrics"`
BaseModel `bson:",inline"`
Type string `json:"type" bson:"type"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"`
TotalMemory uint64 `json:"total_memory" bson:"total_memory"`
AvailableMemory uint64 `json:"available_memory" bson:"available_memory"`
UsedMemory uint64 `json:"used_memory" bson:"used_memory"`
UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"`
TotalDisk uint64 `json:"total_disk" bson:"total_disk"`
AvailableDisk uint64 `json:"available_disk" bson:"available_disk"`
UsedDisk uint64 `json:"used_disk" bson:"used_disk"`
UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"`
DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"`
DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"`
NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"`
NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"`
Type string `json:"type" bson:"type" description:"Type"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id" description:"Node ID"`
CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent" description:"CPU usage percentage"`
TotalMemory uint64 `json:"total_memory" bson:"total_memory" description:"Total memory"`
AvailableMemory uint64 `json:"available_memory" bson:"available_memory" description:"Available memory"`
UsedMemory uint64 `json:"used_memory" bson:"used_memory" description:"Used memory"`
UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent" description:"Used memory percentage"`
TotalDisk uint64 `json:"total_disk" bson:"total_disk" description:"Total disk"`
AvailableDisk uint64 `json:"available_disk" bson:"available_disk" description:"Available disk"`
UsedDisk uint64 `json:"used_disk" bson:"used_disk" description:"Used disk"`
UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent" description:"Used disk percentage"`
DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate" description:"Disk read bytes rate"`
DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate" description:"Disk write bytes rate"`
NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate" description:"Network bytes sent rate"`
NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate" description:"Network bytes recv rate"`
}

26
core/models/models/node.go
View File

@@ -7,17 +7,17 @@ import (
type Node struct {
any `collection:"nodes"`
BaseModel `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Ip string `json:"ip" bson:"ip"`
Mac string `json:"mac" bson:"mac"`
Hostname string `json:"hostname" bson:"hostname"`
Description string `json:"description" bson:"description"`
IsMaster bool `json:"is_master" bson:"is_master"`
Status string `json:"status" bson:"status"`
Enabled bool `json:"enabled" bson:"enabled"`
Active bool `json:"active" bson:"active"`
ActiveAt time.Time `json:"active_at" bson:"active_ts"`
CurrentRunners int `json:"current_runners" bson:"current_runners"`
MaxRunners int `json:"max_runners" bson:"max_runners"`
Key string `json:"key" bson:"key" description:"Key"`
Name string `json:"name" bson:"name" description:"Name"`
Ip string `json:"ip" bson:"ip" description:"IP"`
Mac string `json:"mac" bson:"mac" description:"MAC"`
Hostname string `json:"hostname" bson:"hostname" description:"Hostname"`
Description string `json:"description" bson:"description" description:"Description"`
IsMaster bool `json:"is_master" bson:"is_master" description:"Is master"`
Status string `json:"status" bson:"status" description:"Status"`
Enabled bool `json:"enabled" bson:"enabled" description:"Enabled"`
Active bool `json:"active" bson:"active" description:"Active"`
ActiveAt time.Time `json:"active_at" bson:"active_ts" description:"Active at"`
CurrentRunners int `json:"current_runners" bson:"current_runners" description:"Current runners"`
MaxRunners int `json:"max_runners" bson:"max_runners" description:"Max runners"`
}

22
core/models/models/notification_alert.go
View File

@@ -5,15 +5,15 @@ import "go.mongodb.org/mongo-driver/bson/primitive"
type NotificationAlert struct {
any `collection:"notification_alerts"`
BaseModel `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Enabled bool `json:"enabled" bson:"enabled"`
HasMetricTarget bool `json:"has_metric_target" bson:"has_metric_target"`
MetricTargetId primitive.ObjectID `json:"metric_target_id,omitempty" bson:"metric_target_id,omitempty"`
MetricName string `json:"metric_name" bson:"metric_name"`
Operator string `json:"operator" bson:"operator"`
LastingSeconds int `json:"lasting_seconds" bson:"lasting_seconds"`
TargetValue float32 `json:"target_value" bson:"target_value"`
Level string `json:"level" bson:"level"`
TemplateKey string `json:"template_key,omitempty" bson:"template_key,omitempty"`
Name string `json:"name" bson:"name" description:"Name"`
Description string `json:"description" bson:"description" description:"Description"`
Enabled bool `json:"enabled" bson:"enabled" description:"Enabled"`
HasMetricTarget bool `json:"has_metric_target" bson:"has_metric_target" description:"Has metric target"`
MetricTargetId primitive.ObjectID `json:"metric_target_id,omitempty" bson:"metric_target_id,omitempty" description:"Metric target ID"`
MetricName string `json:"metric_name" bson:"metric_name" description:"Metric name"`
Operator string `json:"operator" bson:"operator" description:"Operator"`
LastingSeconds int `json:"lasting_seconds" bson:"lasting_seconds" description:"Lasting seconds"`
TargetValue float32 `json:"target_value" bson:"target_value" description:"Target value"`
Level string `json:"level" bson:"level" description:"Level"`
TemplateKey string `json:"template_key,omitempty" bson:"template_key,omitempty" description:"Template key"`
}

24
core/models/models/notification_channel.go
View File

@@ -3,16 +3,16 @@ package models
type NotificationChannel struct {
any `collection:"notification_channels"`
BaseModel `bson:",inline"`
Type string `json:"type" bson:"type"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Provider string `json:"provider" bson:"provider"`
SMTPServer string `json:"smtp_server,omitempty" bson:"smtp_server,omitempty"`
SMTPPort int `json:"smtp_port,omitempty" bson:"smtp_port,omitempty"`
SMTPUsername string `json:"smtp_username,omitempty" bson:"smtp_username,omitempty"`
SMTPPassword string `json:"smtp_password,omitempty" bson:"smtp_password,omitempty"`
WebhookUrl string `json:"webhook_url,omitempty" bson:"webhook_url,omitempty"`
TelegramBotToken string `json:"telegram_bot_token,omitempty" bson:"telegram_bot_token,omitempty"`
TelegramChatId string `json:"telegram_chat_id,omitempty" bson:"telegram_chat_id,omitempty"`
GoogleOAuth2Json string `json:"google_oauth2_json,omitempty" bson:"google_oauth2_json,omitempty"`
Type string `json:"type" bson:"type" description:"Type" enum:"im,mail"`
Name string `json:"name" bson:"name" description:"Name"`
Description string `json:"description" bson:"description" description:"Description"`
Provider string `json:"provider" bson:"provider" description:"Provider"`
SMTPServer string `json:"smtp_server,omitempty" bson:"smtp_server,omitempty" description:"SMTP server"`
SMTPPort int `json:"smtp_port,omitempty" bson:"smtp_port,omitempty" description:"SMTP port"`
SMTPUsername string `json:"smtp_username,omitempty" bson:"smtp_username,omitempty" description:"SMTP username"`
SMTPPassword string `json:"smtp_password,omitempty" bson:"smtp_password,omitempty" description:"SMTP password"`
WebhookUrl string `json:"webhook_url,omitempty" bson:"webhook_url,omitempty" description:"Webhook URL"`
TelegramBotToken string `json:"telegram_bot_token,omitempty" bson:"telegram_bot_token,omitempty" description:"Telegram bot token"`
TelegramChatId string `json:"telegram_chat_id,omitempty" bson:"telegram_chat_id,omitempty" description:"Telegram chat ID"`
GoogleOAuth2Json string `json:"google_oauth2_json,omitempty" bson:"google_oauth2_json,omitempty" description:"Google OAuth2 JSON"`
}

28
core/models/models/notification_request.go
View File

@@ -5,18 +5,18 @@ import "go.mongodb.org/mongo-driver/bson/primitive"
type NotificationRequest struct {
any `collection:"notification_requests"`
BaseModel `bson:",inline"`
Status string `json:"status" bson:"status"`
Error string `json:"error,omitempty" bson:"error,omitempty"`
Title string `json:"title" bson:"title"`
Content string `json:"content" bson:"content"`
SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"`
SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"`
MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty"`
MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"`
MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"`
SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"`
ChannelId primitive.ObjectID `json:"channel_id" bson:"channel_id"`
Setting *NotificationSetting `json:"setting,omitempty" bson:"-"`
Channel *NotificationChannel `json:"channel,omitempty" bson:"-"`
Test bool `json:"test,omitempty" bson:"test,omitempty"`
Status string `json:"status" bson:"status" description:"Status"`
Error string `json:"error,omitempty" bson:"error,omitempty" description:"Error"`
Title string `json:"title" bson:"title" description:"Title"`
Content string `json:"content" bson:"content" description:"Content"`
SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty" description:"Sender email"`
SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty" description:"Sender name"`
MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty" description:"Mail to"`
MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty" description:"Mail CC"`
MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty" description:"Mail BCC"`
SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id" description:"Setting ID"`
ChannelId primitive.ObjectID `json:"channel_id" bson:"channel_id" description:"Channel ID"`
Setting *NotificationSetting `json:"setting,omitempty" bson:"-" binding:"-"`
Channel *NotificationChannel `json:"channel,omitempty" bson:"-" binding:"-"`
Test bool `json:"test,omitempty" bson:"test,omitempty" description:"Test"`
}

40
core/models/models/notification_setting.go
View File

@@ -5,30 +5,30 @@ import "go.mongodb.org/mongo-driver/bson/primitive"
type NotificationSetting struct {
any `collection:"notification_settings"`
BaseModel `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Enabled bool `json:"enabled" bson:"enabled"`
Name string `json:"name" bson:"name" description:"Name"`
Description string `json:"description" bson:"description" description:"Description"`
Enabled bool `json:"enabled" bson:"enabled" description:"Enabled"`
Title string `json:"title,omitempty" bson:"title,omitempty"`
Template string `json:"template" bson:"template"`
TemplateMode string `json:"template_mode" bson:"template_mode"`
TemplateMarkdown string `json:"template_markdown,omitempty" bson:"template_markdown,omitempty"`
TemplateRichText string `json:"template_rich_text,omitempty" bson:"template_rich_text,omitempty"`
TemplateRichTextJson string `json:"template_rich_text_json,omitempty" bson:"template_rich_text_json,omitempty"`
TemplateTheme string `json:"template_theme,omitempty" bson:"template_theme,omitempty"`
Title string `json:"title,omitempty" bson:"title,omitempty" description:"Title"`
Template string `json:"template" bson:"template" description:"Template"`
TemplateMode string `json:"template_mode" bson:"template_mode" description:"Template mode"`
TemplateMarkdown string `json:"template_markdown,omitempty" bson:"template_markdown,omitempty" description:"Template markdown"`
TemplateRichText string `json:"template_rich_text,omitempty" bson:"template_rich_text,omitempty" description:"Template rich text"`
TemplateRichTextJson string `json:"template_rich_text_json,omitempty" bson:"template_rich_text_json,omitempty" description:"Template rich text JSON"`
TemplateTheme string `json:"template_theme,omitempty" bson:"template_theme,omitempty" description:"Template theme"`
TaskTrigger string `json:"task_trigger" bson:"task_trigger"`
Trigger string `json:"trigger" bson:"trigger"`
TaskTrigger string `json:"task_trigger" bson:"task_trigger" description:"Task trigger"`
Trigger string `json:"trigger" bson:"trigger" description:"Trigger"`
SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"`
UseCustomSenderEmail bool `json:"use_custom_sender_email,omitempty" bson:"use_custom_sender_email,omitempty"`
SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"`
MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty"`
MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"`
MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"`
SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty" description:"Sender email"`
UseCustomSenderEmail bool `json:"use_custom_sender_email,omitempty" bson:"use_custom_sender_email,omitempty" description:"Use custom sender email"`
SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty" description:"Sender name"`
MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty" description:"Mail to"`
MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty" description:"Mail CC"`
MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty" description:"Mail BCC"`
ChannelIds []primitive.ObjectID `json:"channel_ids,omitempty" bson:"channel_ids,omitempty"`
ChannelIds []primitive.ObjectID `json:"channel_ids,omitempty" bson:"channel_ids,omitempty" description:"Channel IDs"`
Channels []NotificationChannel `json:"channels,omitempty" bson:"-"`
AlertId primitive.ObjectID `json:"alert_id,omitempty" bson:"alert_id,omitempty"`
AlertId primitive.ObjectID `json:"alert_id,omitempty" bson:"alert_id,omitempty" description:"Alert ID"`
}

12
core/models/models/permission.go
View File

@@ -5,10 +5,10 @@ import "go.mongodb.org/mongo-driver/bson/primitive"
type Permission struct {
any `collection:"permissions"`
BaseModel `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
Type string `json:"type" bson:"type"`
Routes []string `json:"routes" bson:"routes"`
Key string `json:"key" bson:"key" description:"Key"`
Name string `json:"name" bson:"name" description:"Name"`
Description string `json:"description" bson:"description" description:"Description"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id" description:"Role ID"`
Type string `json:"type" bson:"type" description:"Type"`
Routes []string `json:"routes" bson:"routes" description:"Routes"`
}

4
core/models/models/project.go
View File

@@ -3,7 +3,7 @@ package models
type Project struct {
any `collection:"projects"`
BaseModel `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Name string `json:"name" bson:"name" description:"Name"`
Description string `json:"description" bson:"description" description:"Description"`
Spiders int `json:"spiders" bson:"-"`
}

8
core/models/models/role.go
View File

@@ -3,10 +3,10 @@ package models
type Role struct {
any `collection:"roles"`
BaseModel `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Routes []string `json:"routes" bson:"routes"`
RootAdmin bool `json:"-" bson:"root_admin,omitempty"`
Name string `json:"name" bson:"name" description:"Name"`
Description string `json:"description" bson:"description" description:"Description"`
Routes []string `json:"routes" bson:"routes" description:"Routes"`
RootAdmin bool `json:"-" bson:"root_admin,omitempty" description:"Root admin"`
IsRootAdmin bool `json:"root_admin" bson:"-"`
Users int `json:"users" bson:"-"`
}

22
core/models/models/schedule.go
View File

@@ -8,15 +8,15 @@ import (
type Schedule struct {
any `collection:"schedules"`
BaseModel `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Cron string `json:"cron" bson:"cron"`
EntryId cron.EntryID `json:"entry_id" bson:"entry_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
Priority int `json:"priority" bson:"priority"`
Enabled bool `json:"enabled" bson:"enabled"`
Name string `json:"name" bson:"name" description:"Name"`
Description string `json:"description" bson:"description" description:"Description"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id" description:"Spider ID"`
Cron string `json:"cron" bson:"cron" description:"Cron"`
EntryId cron.EntryID `json:"entry_id" bson:"entry_id" description:"Entry ID"`
Cmd string `json:"cmd" bson:"cmd" description:"Cmd"`
Param string `json:"param" bson:"param" description:"Param"`
Mode string `json:"mode" bson:"mode" description:"Mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids" description:"Node IDs"`
Priority int `json:"priority" bson:"priority" description:"Priority"`
Enabled bool `json:"enabled" bson:"enabled" description:"Enabled"`
}

4
core/models/models/setting.go
View File

@@ -7,6 +7,6 @@ import (
type Setting struct {
any `collection:"settings"`
BaseModel `bson:",inline"`
Key string `json:"key" bson:"key"`
Value bson.M `json:"value" bson:"value"`
Key string `json:"key" bson:"key" description:"Key"`
Value bson.M `json:"value" bson:"value" description:"Value"`
}

42
core/models/models/spider.go
View File

@@ -7,34 +7,34 @@ import (
type Spider struct {
any `collection:"spiders"`
BaseModel `bson:",inline"`
Name string `json:"name" bson:"name"` // spider name
ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id (deprecated) # TODO: remove this field in the future
ColName string `json:"col_name,omitempty" bson:"col_name"` // data collection name
DbName string `json:"db_name,omitempty" bson:"db_name"` // database name
DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id
DataSource *Database `json:"data_source,omitempty" bson:"-"` // data source
Description string `json:"description" bson:"description"` // description
ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id
Mode string `json:"mode" bson:"mode"` // default Task.Mode
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds
GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id
GitRootPath string `json:"git_root_path" bson:"git_root_path"`
Name string `json:"name" bson:"name" description:"Spider name"`
ColId primitive.ObjectID `json:"col_id" bson:"col_id" description:"Data collection id" deprecated:"true"`
ColName string `json:"col_name,omitempty" bson:"col_name" description:"Data collection name"`
DbName string `json:"db_name,omitempty" bson:"db_name" description:"Database name"`
DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id" description:"Data source id"`
DataSource *Database `json:"data_source,omitempty" bson:"-"`
Description string `json:"description" bson:"description" description:"Description"`
ProjectId primitive.ObjectID `json:"project_id" bson:"project_id" description:"Project ID"`
Mode string `json:"mode" bson:"mode" description:"Default task mode" enum:"random,all,selected-nodes"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids" description:"Default node ids, used in selected-nodes mode"`
GitId primitive.ObjectID `json:"git_id" bson:"git_id" description:"Related Git ID"`
GitRootPath string `json:"git_root_path" bson:"git_root_path" description:"Git root path"`
Git *Git `json:"git,omitempty" bson:"-"`
Template string `json:"template,omitempty" bson:"template,omitempty"` // spider template
TemplateParams *SpiderTemplateParams `json:"template_params,omitempty" bson:"template_params,omitempty"`
Template string `json:"template,omitempty" bson:"template,omitempty" description:"Spider template"`
TemplateParams *SpiderTemplateParams `json:"template_params,omitempty" bson:"template_params,omitempty" description:"Spider template params"`
// stats
Stat *SpiderStat `json:"stat,omitempty" bson:"-"`
// execution
Cmd string `json:"cmd" bson:"cmd"` // execute command
Param string `json:"param" bson:"param"` // default task param
Priority int `json:"priority" bson:"priority"`
Cmd string `json:"cmd" bson:"cmd" description:"Execute command"`
Param string `json:"param" bson:"param" description:"Default task param"`
Priority int `json:"priority" bson:"priority" description:"Priority" default:"5" minimum:"1" maximum:"10"`
}
type SpiderTemplateParams struct {
ProjectName string `json:"project_name,omitempty" bson:"project_name,omitempty"`
SpiderName string `json:"spider_name,omitempty" bson:"spider_name,omitempty"`
StartUrls string `json:"start_urls,omitempty" bson:"start_urls,omitempty"`
AllowedDomains string `json:"allowed_domains,omitempty" bson:"allowed_domains,omitempty"`
ProjectName string `json:"project_name,omitempty" bson:"project_name,omitempty" description:"Project name"`
SpiderName string `json:"spider_name,omitempty" bson:"spider_name,omitempty" description:"Spider name"`
StartUrls string `json:"start_urls,omitempty" bson:"start_urls,omitempty" description:"Start urls"`
AllowedDomains string `json:"allowed_domains,omitempty" bson:"allowed_domains,omitempty" description:"Allowed domains"`
}

18
core/models/models/spider_stat.go
View File

@@ -7,14 +7,14 @@ import (
type SpiderStat struct {
any `collection:"spider_stats"`
BaseModel `bson:",inline"`
LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"`
LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty" description:"Last task ID"`
LastTask *Task `json:"last_task,omitempty" bson:"-"`
Tasks int `json:"tasks" bson:"tasks"`
Results int `json:"results" bson:"results"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second
AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second
AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second
AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second
Tasks int `json:"tasks" bson:"tasks" description:"Task count"`
Results int `json:"results" bson:"results" description:"Result count"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty" description:"Wait duration (in second)"`
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty" description:"Runtime duration (in second)"`
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty" description:"Total duration (in second)"`
AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second
AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second
AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second
}

22
core/models/models/task.go
View File

@@ -7,17 +7,17 @@ import (
type Task struct {
any `collection:"tasks"`
BaseModel `bson:",inline"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Status string `json:"status" bson:"status"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Error string `json:"error" bson:"error"`
Pid int `json:"pid" bson:"pid"`
ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"`
Type string `json:"type" bson:"type"`
Mode string `json:"mode" bson:"mode"`
Priority int `json:"priority" bson:"priority"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id" description:"Spider ID"`
Status string `json:"status" bson:"status" description:"Status"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id" description:"Node ID"`
Cmd string `json:"cmd" bson:"cmd" description:"Command"`
Param string `json:"param" bson:"param" description:"Parameter"`
Error string `json:"error" bson:"error" description:"Error"`
Pid int `json:"pid" bson:"pid" description:"Process ID"`
ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id" description:"Schedule ID"`
Type string `json:"type" bson:"type" description:"Type"`
Mode string `json:"mode" bson:"mode" description:"Mode"`
Priority int `json:"priority" bson:"priority" description:"Priority"`
NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-"`
Stat *TaskStat `json:"stat,omitempty" bson:"-"`
Spider *Spider `json:"spider,omitempty" bson:"-"`

12
core/models/models/task_stat.go
View File

@@ -7,10 +7,10 @@ import (
type TaskStat struct {
any `collection:"task_stats"`
BaseModel `bson:",inline"`
StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"`
EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond
ResultCount int64 `json:"result_count" bson:"result_count"`
StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty" description:"Start time"`
EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty" description:"End time"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty" description:"Wait duration (in millisecond)"`
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty" description:"Runtime duration (in millisecond)"`
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty" description:"Total duration (in millisecond)"`
ResultCount int64 `json:"result_count" bson:"result_count" description:"Result count"`
}

4
core/models/models/token.go
View File

@@ -3,6 +3,6 @@ package models
type Token struct {
any `collection:"tokens"`
BaseModel `bson:",inline"`
Name string `json:"name" bson:"name"`
Token string `json:"token" bson:"token"`
Name string `json:"name" bson:"name" description:"Name"`
Token string `json:"token" bson:"token" description:"Token"`
}

16
core/models/models/user.go
View File

@@ -5,14 +5,14 @@ import "go.mongodb.org/mongo-driver/bson/primitive"
type User struct {
any `collection:"users"`
BaseModel `bson:",inline"`
Username string `json:"username" bson:"username"`
Password string `json:"-" bson:"password"`
Role string `json:"role" bson:"role"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
FirstName string `json:"first_name" bson:"first_name"`
LastName string `json:"last_name" bson:"last_name"`
Email string `json:"email" bson:"email"`
RootAdmin bool `json:"root_admin,omitempty" bson:"root_admin"`
Username string `json:"username" bson:"username" description:"Username"`
Password string `json:"-" bson:"password" description:"Password"`
Role string `json:"role" bson:"role" description:"Role"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id" description:"Role ID"`
FirstName string `json:"first_name" bson:"first_name" description:"First name"`
LastName string `json:"last_name" bson:"last_name" description:"Last name"`
Email string `json:"email" bson:"email" description:"Email"`
RootAdmin bool `json:"root_admin,omitempty" bson:"root_admin" description:"Root admin"`
RootAdminRole bool `json:"root_admin_role,omitempty" bson:"-"`
Routes []string `json:"routes,omitempty" bson:"-"`
}

4
core/models/models/user_role.go
View File

@@ -7,6 +7,6 @@ import (
type UserRole struct {
any `collection:"user_roles"`
BaseModel `bson:",inline"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id" description:"Role ID"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id" description:"User ID"`
}

13
core/utils/string.go
View File

@@ -32,9 +32,7 @@ func ToSnakeCase(s string) string {
}
func ToPascalCase(s string) string {
s = strings.TrimSpace(s)
s = strings.ReplaceAll(s, "_", " ")
s = cases.Title(language.English).String(s)
s = Capitalize(s)
s = strings.ReplaceAll(s, " ", "")
return s
}
@@ -45,6 +43,15 @@ func ToKebabCase(s string) string {
return replaceChars(s, []string{" ", "_", "."}, "-")
}
func Capitalize(s string) string {
s = strings.TrimSpace(s)
for _, char := range []string{" ", "_", "-", "/"} {
s = strings.ReplaceAll(s, char, " ")
}
s = cases.Title(language.English).String(s)
return s
}
// splitStringWithQuotes splits a string with quotes
// Parameters:
// - s: the string to split