diff --git a/core/controllers/base_file.go b/core/controllers/base_file.go index fc8657ed..1384d1f0 100644 --- a/core/controllers/base_file.go +++ b/core/controllers/base_file.go @@ -3,21 +3,17 @@ package controllers import ( "errors" "fmt" + "io" + "mime/multipart" + "os" + "sync" + "github.com/crawlab-team/crawlab/core/fs" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/gin-gonic/gin" - "io" - "os" - "sync" ) -type GetBaseFileListDirParams struct { - Path string `path:"path"` -} - -func GetBaseFileListDir(rootPath string, params *GetBaseFileListDirParams) (response *Response[[]interfaces.FsFileInfo], err error) { - path := params.Path - +func GetBaseFileListDir(rootPath, path string) (response *Response[[]interfaces.FsFileInfo], err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { return GetErrorResponse[[]interfaces.FsFileInfo](err) @@ -30,17 +26,10 @@ func GetBaseFileListDir(rootPath string, params *GetBaseFileListDirParams) (resp } } - //HandleSuccessWithData(c, files) - return GetDataResponse[[]interfaces.FsFileInfo](files) + return GetDataResponse(files) } -type GetBaseFileFileParams struct { - Path string `path:"path"` -} - -func GetBaseFileFile(rootPath string, params *GetBaseFileFileParams) (response *Response[string], err error) { - path := params.Path - +func GetBaseFileContent(rootPath, path string) (response *Response[string], err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { return GetErrorResponse[string](err) @@ -51,94 +40,75 @@ func GetBaseFileFile(rootPath string, params *GetBaseFileFileParams) (response * return GetErrorResponse[string](err) } - return GetDataResponse[string](string(data)) + return GetDataResponse(string(data)) } -func GetBaseFileFileInfo(rootPath string, c *gin.Context) { - path := c.Query("path") - +func GetBaseFileInfo(rootPath, path string) (response *Response[interfaces.FsFileInfo], err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[interfaces.FsFileInfo](err) } info, err := fsSvc.GetFileInfo(path) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[interfaces.FsFileInfo](err) } - HandleSuccessWithData(c, info) + return GetDataResponse(info) } -func PostBaseFileSaveFile(rootPath string, c *gin.Context) { - fsSvc, err := fs.GetBaseFileFsSvc(rootPath) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - if c.GetHeader("Content-Type") == "application/json" { - var payload struct { - Path string `json:"path"` - Data string `json:"data"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := fsSvc.Save(payload.Path, []byte(payload.Data)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } else { - path, ok := c.GetPostForm("path") - if !ok { - HandleErrorBadRequest(c, errors.New("missing required field 'path'")) - return - } - file, err := c.FormFile("file") - if err != nil { - HandleErrorBadRequest(c, err) - return - } - f, err := file.Open() - if err != nil { - HandleErrorBadRequest(c, err) - return - } - fileData, err := io.ReadAll(f) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := fsSvc.Save(path, fileData); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - HandleSuccess(c) +type PostBaseFileSaveOneParams struct { + Path string `json:"path" form:"path"` + Data string `json:"data"` } -func PostBaseFileSaveFiles(rootPath string, c *gin.Context) { +func PostBaseFileSaveOne(rootPath, path, data string) (response *Response[any], err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[any](err) } - form, err := c.MultipartForm() - if err != nil { - HandleErrorBadRequest(c, err) - return + if err := fsSvc.Save(path, []byte(data)); err != nil { + return GetErrorResponse[any](err) } + + return GetDataResponse(any(data)) +} + +func PostBaseFileSaveOneForm(rootPath, path string, file *multipart.FileHeader) (response *Response[any], err error) { + fsSvc, err := fs.GetBaseFileFsSvc(rootPath) + if err != nil { + return GetErrorResponse[any](err) + } + + f, err := file.Open() + if err != nil { + return GetErrorResponse[any](err) + } + + fileData, err := io.ReadAll(f) + if err != nil { + return GetErrorResponse[any](err) + } + + if err := fsSvc.Save(path, fileData); err != nil { + return GetErrorResponse[any](err) + } + + return GetDataResponse[any](nil) +} + +func PostBaseFileSaveMany(rootPath string, form *multipart.Form) (response *Response[any], err error) { + fsSvc, err := fs.GetBaseFileFsSvc(rootPath) + if err != nil { + return GetErrorResponse[any](err) + } + wg := sync.WaitGroup{} wg.Add(len(form.File)) for path := range form.File { go func(path string) { - file, err := c.FormFile(path) + file := form.File[path][0] if err != nil { logger.Warnf("invalid file header: %s", path) logger.Error(err.Error()) @@ -170,125 +140,84 @@ func PostBaseFileSaveFiles(rootPath string, c *gin.Context) { } wg.Wait() - HandleSuccess(c) + return GetDataResponse[any](nil) } -func PostBaseFileSaveDir(rootPath string, c *gin.Context) { - var payload struct { - Path string `json:"path"` - NewPath string `json:"new_path"` - Data string `json:"data"` +func PostBaseFileSaveDir(rootPath, path string) (response *Response[any], err error) { + fsSvc, err := fs.GetBaseFileFsSvc(rootPath) + if err != nil { + return GetErrorResponse[any](err) } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return + + if err := fsSvc.CreateDir(path); err != nil { + return GetErrorResponse[any](err) + } + + return GetDataResponse[any](nil) +} + +func PostBaseFileRename(rootPath, path, newPath string) (response *Response[any], err error) { + fsSvc, err := fs.GetBaseFileFsSvc(rootPath) + if err != nil { + return GetErrorResponse[any](err) + } + + if err := fsSvc.Rename(path, newPath); err != nil { + return GetErrorResponse[any](err) + } + + return GetDataResponse[any](nil) +} + +func DeleteBaseFile(rootPath, path string) (response *Response[any], err error) { + if path == "~" { + path = "." } fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[any](err) } - if err := fsSvc.CreateDir(payload.Path); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func PostBaseFileRenameFile(rootPath string, c *gin.Context) { - var payload struct { - Path string `json:"path"` - NewPath string `json:"new_path"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - fsSvc, err := fs.GetBaseFileFsSvc(rootPath) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := fsSvc.Rename(payload.Path, payload.NewPath); err != nil { - HandleErrorInternalServerError(c, err) - return - } -} - -func DeleteBaseFileFile(rootPath string, c *gin.Context) { - var payload struct { - Path string `json:"path"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - if payload.Path == "~" { - payload.Path = "." - } - - fsSvc, err := fs.GetBaseFileFsSvc(rootPath) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := fsSvc.Delete(payload.Path); err != nil { - HandleErrorInternalServerError(c, err) - return + if err := fsSvc.Delete(path); err != nil { + return GetErrorResponse[any](err) } _, err = fsSvc.GetFileInfo(".") if err != nil { _ = fsSvc.CreateDir("/") } - HandleSuccess(c) + return GetDataResponse[any](nil) } -func PostBaseFileCopyFile(rootPath string, c *gin.Context) { - var payload struct { - Path string `json:"path"` - NewPath string `json:"new_path"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - +func PostBaseFileCopy(rootPath, path, newPath string) (response *Response[any], err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[any](err) } - if err := fsSvc.Copy(payload.Path, payload.NewPath); err != nil { - HandleErrorInternalServerError(c, err) - return + if err := fsSvc.Copy(path, newPath); err != nil { + return GetErrorResponse[any](err) } - HandleSuccess(c) + return GetDataResponse[any](nil) } -func PostBaseFileExport(rootPath string, c *gin.Context) { +func PostBaseFileExport(rootPath string, c *gin.Context) (err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - HandleErrorBadRequest(c, err) - return + return err } // zip file path zipFilePath, err := fsSvc.Export() if err != nil { - HandleErrorInternalServerError(c, err) - return + return err } // download c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilePath)) c.File(zipFilePath) + + return nil } diff --git a/core/controllers/login.go b/core/controllers/login.go index 082ff200..1a174b12 100644 --- a/core/controllers/login.go +++ b/core/controllers/login.go @@ -7,30 +7,26 @@ import ( "github.com/gin-gonic/gin" ) -func PostLogin(c *gin.Context) { - var payload struct { - Username string `json:"username"` - Password string `json:"password"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - userSvc, err := user.GetUserService() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - token, loggedInUser, err := userSvc.Login(payload.Username, payload.Password) - if err != nil { - HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) - return - } - c.Set(constants.UserContextKey, loggedInUser) - HandleSuccessWithData(c, token) +type PostLoginParams struct { + Username string `json:"username" validate:"required"` + Password string `json:"password" validate:"required"` } -func PostLogout(c *gin.Context) { - c.Set(constants.UserContextKey, nil) - HandleSuccess(c) +func PostLogin(c *gin.Context, params *PostLoginParams) (response *Response[string], err error) { + userSvc, err := user.GetUserService() + if err != nil { + return GetErrorResponse[string](err) + } + + token, loggedInUser, err := userSvc.Login(params.Username, params.Password) + if err != nil { + return GetErrorResponse[string](errors.ErrorUserUnauthorized) + } + + c.Set(constants.UserContextKey, loggedInUser) + return GetDataResponse(token) +} + +func PostLogout(_ *gin.Context) (response *Response[any], err error) { + return GetDataResponse[any](nil) } diff --git a/core/controllers/router.go b/core/controllers/router.go index 1b7846a2..95aa7805 100644 --- a/core/controllers/router.go +++ b/core/controllers/router.go @@ -271,7 +271,7 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodGet, Path: "/:id/files/get", - HandlerFunc: GetSpiderFile, + HandlerFunc: GetSpiderFileContent, }, { Method: http.MethodGet, @@ -324,8 +324,6 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetSpiderResults, }, }...)) - groups.AnonymousGroup.GET("/openapi.json", GetOpenAPI) - return RegisterController(groups.AuthGroup, "/schedules", NewController[models.Schedule]([]Action{ { Method: http.MethodPost, diff --git a/core/controllers/schedule.go b/core/controllers/schedule.go index a3a0056d..4a5bf363 100644 --- a/core/controllers/schedule.go +++ b/core/controllers/schedule.go @@ -2,22 +2,23 @@ package controllers import ( errors2 "errors" + "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/schedule" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/gin-gonic/gin" + "github.com/juju/errors" "go.mongodb.org/mongo-driver/bson/primitive" ) -func PostSchedule(c *gin.Context) { - var s models.Schedule - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } +type PostScheduleParams struct { + Data models.Schedule `json:"data"` +} +func PostSchedule(c *gin.Context, params *PostScheduleParams) (response *Response[models.Schedule], err error) { + s := params.Data u := GetUserFromContext(c) modelSvc := service.NewModelService[models.Schedule]() @@ -26,141 +27,155 @@ func PostSchedule(c *gin.Context) { s.SetUpdated(u.Id) id, err := modelSvc.InsertOne(s) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Schedule](err) } s.Id = id if s.Enabled { scheduleSvc := schedule.GetScheduleService() if err := scheduleSvc.Enable(s, u.Id); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Schedule](err) } } - HandleSuccessWithData(c, s) + return GetDataResponse(s) } -func PutScheduleById(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +type PutScheduleByIdParams struct { + Id string `path:"id"` + Data models.Schedule `json:"data"` +} + +func PutScheduleById(c *gin.Context, params *PutScheduleByIdParams) (response *Response[models.Schedule], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return - } - var s models.Schedule - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[models.Schedule](errors.BadRequestf("invalid schedule id: %v", err)) } + + s := params.Data if s.Id != id { - HandleErrorBadRequest(c, errors2.New("id in path does not match id in body")) - return + return GetErrorResponse[models.Schedule](errors2.New("id in path does not match id in body")) } modelSvc := service.NewModelService[models.Schedule]() err = modelSvc.ReplaceById(id, s) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Schedule](err) } scheduleSvc := schedule.GetScheduleService() - u := GetUserFromContext(c) if s.Enabled { if err := scheduleSvc.Enable(s, u.Id); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Schedule](err) } } else { if err := scheduleSvc.Disable(s, u.Id); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Schedule](err) } } - HandleSuccessWithData(c, s) + return GetDataResponse(s) } -func PostScheduleEnable(c *gin.Context) { - postScheduleEnableDisableFunc(true)(c) +type PostScheduleEnableDisableParams struct { + Id string `path:"id"` } -func PostScheduleDisable(c *gin.Context) { - postScheduleEnableDisableFunc(false)(c) +func PostScheduleEnable(c *gin.Context, params *PostScheduleEnableDisableParams) (response *Response[any], err error) { + userId := GetUserFromContext(c).Id + return postScheduleEnableDisableFunc(true, userId, params) } -func postScheduleEnableDisableFunc(isEnable bool) func(c *gin.Context) { - return func(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - svc := schedule.GetScheduleService() - s, err := service.NewModelService[models.Schedule]().GetById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - u := GetUserFromContext(c) - if isEnable { - err = svc.Enable(*s, u.Id) - } else { - err = svc.Disable(*s, u.Id) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) - } +func PostScheduleDisable(c *gin.Context, params *PostScheduleEnableDisableParams) (response *Response[any], err error) { + userId := GetUserFromContext(c).Id + return postScheduleEnableDisableFunc(false, userId, params) } -func PostScheduleRun(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +func postScheduleEnableDisableFunc(isEnable bool, userId primitive.ObjectID, params *PostScheduleEnableDisableParams) (response *Response[any], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[any](errors.BadRequestf("invalid schedule id: %v", err)) } - // options - var opts interfaces.SpiderRunOptions - if err := c.ShouldBindJSON(&opts); err != nil { - HandleErrorInternalServerError(c, err) - return + svc := schedule.GetScheduleService() + s, err := service.NewModelService[models.Schedule]().GetById(id) + if err != nil { + return GetErrorResponse[any](err) } - if opts.ScheduleId.IsZero() { - opts.ScheduleId = id + + if isEnable { + err = svc.Enable(*s, userId) + } else { + err = svc.Disable(*s, userId) + } + if err != nil { + return GetErrorResponse[any](err) + } + + return GetDataResponse[any](nil) +} + +type PostScheduleRunParams struct { + Id string `path:"id"` + Mode string `json:"mode"` + NodeIds []string `json:"node_ids"` + Cmd string `json:"cmd"` + Param string `json:"param"` + Priority int `json:"priority"` +} + +func PostScheduleRun(c *gin.Context, params *PostScheduleRunParams) (response *Response[[]primitive.ObjectID], err error) { + userId := GetUserFromContext(c).Id + return postScheduleRunFunc(params, userId) +} + +func postScheduleRunFunc(params *PostScheduleRunParams, userId primitive.ObjectID) (response *Response[[]primitive.ObjectID], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) + if err != nil { + return GetErrorResponse[[]primitive.ObjectID](errors.BadRequestf("invalid schedule id: %v", err)) + } + + var nodeIds []primitive.ObjectID + for _, nodeId := range params.NodeIds { + nodeId, err := primitive.ObjectIDFromHex(nodeId) + if err != nil { + return GetErrorResponse[[]primitive.ObjectID](errors.BadRequestf("invalid node id: %v", err)) + } + nodeIds = append(nodeIds, nodeId) + } + + opts := interfaces.SpiderRunOptions{ + Mode: params.Mode, + NodeIds: nodeIds, + Cmd: params.Cmd, + Param: params.Param, + Priority: params.Priority, + ScheduleId: id, + UserId: userId, } // schedule sch, err := service.NewModelService[models.Schedule]().GetById(id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } // spider s, err := service.NewModelService[models.Spider]().GetById(sch.SpiderId) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } // user - if u := GetUserFromContext(c); u != nil { - opts.UserId = u.GetId() - } + opts.UserId = userId // schedule tasks taskIds, err := admin.GetSpiderAdminService().Schedule(s.Id, &opts) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } - HandleSuccessWithData(c, taskIds) + return GetDataResponse(taskIds) } diff --git a/core/controllers/setting.go b/core/controllers/setting.go index de691ecb..f04bb56b 100644 --- a/core/controllers/setting.go +++ b/core/controllers/setting.go @@ -1,7 +1,6 @@ package controllers import ( - "errors" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/gin-gonic/gin" @@ -9,84 +8,70 @@ import ( "go.mongodb.org/mongo-driver/mongo" ) -func GetSetting(c *gin.Context) { - // key - key := c.Param("key") - - // setting - s, err := service.NewModelService[models.Setting]().GetOne(bson.M{"key": key}, nil) - if err != nil { - if errors.Is(err, mongo.ErrNoDocuments) { - HandleSuccess(c) - return - } - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, s) +type GetSettingParams struct { + Key string `path:"key" validate:"required"` } -func PostSetting(c *gin.Context) { - // key - key := c.Param("key") - - // settings - var s models.Setting - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorInternalServerError(c, err) - return +func GetSetting(_ *gin.Context, params *GetSettingParams) (response *Response[models.Setting], err error) { + // setting + s, err := service.NewModelService[models.Setting]().GetOne(bson.M{"key": params.Key}, nil) + if err != nil { + if err == mongo.ErrNoDocuments { + return GetDataResponse(models.Setting{}) + } + return GetErrorResponse[models.Setting](err) } + return GetDataResponse(*s) +} + +type PostSettingParams struct { + Key string `path:"key" validate:"required"` + Data models.Setting `json:"data"` +} + +func PostSetting(c *gin.Context, params *PostSettingParams) (response *Response[models.Setting], err error) { + s := params.Data if s.Key == "" { - s.Key = key + s.Key = params.Key } u := GetUserFromContext(c) - s.SetCreated(u.Id) s.SetUpdated(u.Id) id, err := service.NewModelService[models.Setting]().InsertOne(s) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Setting](err) } s.Id = id - HandleSuccessWithData(c, s) + return GetDataResponse(s) } -func PutSetting(c *gin.Context) { - // key - key := c.Param("key") - - // settings - var s models.Setting - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorInternalServerError(c, err) - return - } +type PutSettingParams struct { + Key string `path:"key" validate:"required"` + Data models.Setting `json:"data"` +} +func PutSetting(c *gin.Context, params *PutSettingParams) (response *Response[models.Setting], err error) { modelSvc := service.NewModelService[models.Setting]() // setting - _s, err := modelSvc.GetOne(bson.M{"key": key}, nil) + existingSetting, err := modelSvc.GetOne(bson.M{"key": params.Key}, nil) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Setting](err) } u := GetUserFromContext(c) // save - _s.Value = s.Value - _s.SetUpdated(u.Id) - err = modelSvc.ReplaceOne(bson.M{"key": key}, *_s) + existingSetting.Value = params.Data.Value + existingSetting.SetUpdated(u.Id) + err = modelSvc.ReplaceOne(bson.M{"key": params.Key}, *existingSetting) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Setting](err) } - HandleSuccess(c) + return GetDataResponse(*existingSetting) } diff --git a/core/controllers/spider.go b/core/controllers/spider.go index 06008862..8bbbb5c3 100644 --- a/core/controllers/spider.go +++ b/core/controllers/spider.go @@ -2,6 +2,7 @@ package controllers import ( "math" + "mime/multipart" "os" "path/filepath" "sync" @@ -528,105 +529,189 @@ func DeleteSpiderList(_ *gin.Context, params *DeleteSpiderListParams) (response return GetDataResponse(models.Spider{}) } -func GetSpiderListDir(c *gin.Context, params *GetBaseFileListDirParams) (response *Response[[]interfaces.FsFileInfo], err error) { - rootPath, err := getSpiderRootPathByContext(c) - if err != nil { - HandleErrorForbidden(c, err) - return - } - return GetBaseFileListDir(rootPath, params) +type GetSpiderListDirParams struct { + Id string `path:"id"` + Path string `query:"path"` } -func GetSpiderFile(c *gin.Context, params *GetBaseFileFileParams) (response *Response[string], err error) { +func GetSpiderListDir(c *gin.Context, params *GetSpiderListDirParams) (response *Response[[]interfaces.FsFileInfo], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[[]interfaces.FsFileInfo](err) } - return GetBaseFileFile(rootPath, params) + return GetBaseFileListDir(rootPath, params.Path) } -func GetSpiderFileInfo(c *gin.Context) { +type GetSpiderFileContentParams struct { + Id string `path:"id"` + Path string `query:"path"` +} + +func GetSpiderFileContent(c *gin.Context, params *GetSpiderFileContentParams) (response *Response[string], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[string](err) } - GetBaseFileFileInfo(rootPath, c) + return GetBaseFileContent(rootPath, params.Path) } -func PostSpiderSaveFile(c *gin.Context) { +type GetSpiderFileInfoParams struct { + Id string `path:"id"` + Path string `query:"path"` +} + +func GetSpiderFileInfo(c *gin.Context, params *GetSpiderFileInfoParams) (response *Response[interfaces.FsFileInfo], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[interfaces.FsFileInfo](err) } - PostBaseFileSaveFile(rootPath, c) + return GetBaseFileInfo(rootPath, params.Path) } -func PostSpiderSaveFiles(c *gin.Context) { +type PostSpiderSaveFileParams struct { + Id string `path:"id"` + Path string `json:"path"` + Data string `json:"data"` + File *multipart.FileHeader `form:"file"` +} + +func PostSpiderSaveFile(c *gin.Context, params *PostSpiderSaveFileParams) (response *Response[any], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[any](err) + } + if c.GetHeader("Content-Type") == "application/json" { + return PostBaseFileSaveOne(rootPath, params.Path, params.Data) + } else { + return PostBaseFileSaveOneForm(rootPath, params.Path, params.File) } - targetDirectory := c.PostForm("targetDirectory") - PostBaseFileSaveFiles(filepath.Join(rootPath, targetDirectory), c) } -func PostSpiderSaveDir(c *gin.Context) { +type PostSpiderSaveFilesParams struct { + Id string `path:"id"` + TargetDirectory string `form:"targetDirectory"` +} + +func PostSpiderSaveFiles(c *gin.Context, params *PostSpiderSaveFilesParams) (response *Response[any], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[any](err) } - PostBaseFileSaveDir(rootPath, c) + form, err := c.MultipartForm() + if err != nil { + return GetErrorResponse[any](err) + } + return PostBaseFileSaveMany(filepath.Join(rootPath, params.TargetDirectory), form) } -func PostSpiderRenameFile(c *gin.Context) { +type PostSpiderSaveDirParams struct { + Id string `path:"id"` + Path string `json:"path"` +} + +func PostSpiderSaveDir(c *gin.Context, params *PostSpiderSaveDirParams) (response *Response[any], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[any](err) } - PostBaseFileRenameFile(rootPath, c) + return PostBaseFileSaveDir(rootPath, params.Path) } -func DeleteSpiderFile(c *gin.Context) { +type PostSpiderRenameFileParams struct { + Id string `path:"id"` + Path string `json:"path"` + NewPath string `json:"newPath"` +} + +func PostSpiderRenameFile(c *gin.Context, params *PostSpiderRenameFileParams) (response *Response[any], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[any](err) } - DeleteBaseFileFile(rootPath, c) + return PostBaseFileRename(rootPath, params.Path, params.NewPath) } -func PostSpiderCopyFile(c *gin.Context) { +type DeleteSpiderFileParams struct { + Id string `path:"id"` + Path string `json:"path"` +} + +func DeleteSpiderFile(c *gin.Context, params *DeleteSpiderFileParams) (response *Response[any], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[any](err) } - PostBaseFileCopyFile(rootPath, c) + return DeleteBaseFile(rootPath, params.Path) } -func PostSpiderExport(c *gin.Context) { +type PostSpiderCopyFileParams struct { + Id string `path:"id"` + Path string `json:"path"` + NewPath string `json:"new_path"` +} + +func PostSpiderCopyFile(c *gin.Context, params *PostSpiderCopyFileParams) (response *Response[any], err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - HandleErrorForbidden(c, err) - return + return GetErrorResponse[any](err) } - PostBaseFileExport(rootPath, c) + return PostBaseFileCopy(rootPath, params.Path, params.NewPath) } -func PostSpiderRun(c *gin.Context) (response *Response[[]primitive.ObjectID], err error) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +type PostSpiderExportParams struct { + Id string `path:"id"` +} + +func PostSpiderExport(c *gin.Context, params *PostSpiderExportParams) (err error) { + rootPath, err := getSpiderRootPathByContext(c) + if err != nil { + return err + } + return PostBaseFileExport(rootPath, c) +} + +type PostSpiderRunParams struct { + Id string `path:"id"` + Mode string `json:"mode"` + NodeIds []string `json:"node_ids"` + Cmd string `json:"cmd"` + Param string `json:"param"` + ScheduleId string `json:"schedule_id"` + Priority int `json:"priority"` +} + +func PostSpiderRun(c *gin.Context, params *PostSpiderRunParams) (response *Response[[]primitive.ObjectID], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { return GetErrorResponse[[]primitive.ObjectID](errors.BadRequestf("invalid id format")) } // options - var opts interfaces.SpiderRunOptions + var nodeIds []primitive.ObjectID + if len(params.NodeIds) > 0 { + for _, id := range params.NodeIds { + nodeId, err := primitive.ObjectIDFromHex(id) + if err != nil { + return GetErrorResponse[[]primitive.ObjectID](errors.BadRequestf("invalid node id format")) + } + nodeIds = append(nodeIds, nodeId) + } + } + var scheduleId primitive.ObjectID + if params.ScheduleId != "" { + scheduleId, err = primitive.ObjectIDFromHex(params.ScheduleId) + if err != nil { + return GetErrorResponse[[]primitive.ObjectID](errors.BadRequestf("invalid schedule id format")) + } + } + opts := interfaces.SpiderRunOptions{ + Mode: params.Mode, + NodeIds: nodeIds, + Cmd: params.Cmd, + Param: params.Param, + ScheduleId: scheduleId, + Priority: params.Priority, + } if err := c.ShouldBindJSON(&opts); err != nil { return GetErrorResponse[[]primitive.ObjectID](err) } @@ -642,21 +727,22 @@ func PostSpiderRun(c *gin.Context) (response *Response[[]primitive.ObjectID], er return GetErrorResponse[[]primitive.ObjectID](err) } - HandleSuccessWithData(c, taskIds) return GetDataResponse(taskIds) } -func GetSpiderResults(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +type GetSpiderResultsParams struct { + Id string `path:"id"` +} + +func GetSpiderResults(c *gin.Context, params *GetSpiderResultsParams) (response *ListResponse[bson.M], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorListResponse[bson.M](errors.BadRequestf("invalid id format")) } s, err := service.NewModelService[models.Spider]().GetById(id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[bson.M](err) } // params @@ -672,17 +758,15 @@ func GetSpiderResults(c *gin.Context) { Limit: pagination.Size, })).All(&results) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[bson.M](err) } total, err := mongo2.GetMongoCol(s.ColName).Count(mongo2.GetMongoQuery(query)) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[bson.M](err) } - HandleSuccessWithListData(c, results, total) + return GetListResponse(results, total) } func getSpiderFsSvc(s *models.Spider) (svc interfaces.FsService, err error) { diff --git a/core/controllers/stats.go b/core/controllers/stats.go index 92ec6452..14a64f4e 100644 --- a/core/controllers/stats.go +++ b/core/controllers/stats.go @@ -1,10 +1,11 @@ package controllers import ( + "time" + "github.com/crawlab-team/crawlab/core/stats" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" - "time" ) var statsDefaultQuery = bson.M{ @@ -13,29 +14,53 @@ var statsDefaultQuery = bson.M{ }, } -func GetStatsOverview(c *gin.Context) { - data, err := stats.GetStatsService().GetOverviewStats(statsDefaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, data) +type GetStatsOverviewParams struct { + Query bson.M `json:"query"` } -func GetStatsDaily(c *gin.Context) { - data, err := stats.GetStatsService().GetDailyStats(statsDefaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return +func GetStatsOverview(_ *gin.Context, params *GetStatsOverviewParams) (response *Response[bson.M], err error) { + query := statsDefaultQuery + if params.Query != nil { + query = params.Query } - HandleSuccessWithData(c, data) + + data, err := stats.GetStatsService().GetOverviewStats(query) + if err != nil { + return GetErrorResponse[bson.M](err) + } + return GetDataResponse(data.(bson.M)) } -func GetStatsTasks(c *gin.Context) { - data, err := stats.GetStatsService().GetTaskStats(statsDefaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, data) +type GetStatsDailyParams struct { + Query bson.M `json:"query"` +} + +func GetStatsDaily(_ *gin.Context, params *GetStatsDailyParams) (response *Response[bson.M], err error) { + query := statsDefaultQuery + if params.Query != nil { + query = params.Query + } + + data, err := stats.GetStatsService().GetDailyStats(query) + if err != nil { + return GetErrorResponse[bson.M](err) + } + return GetDataResponse(data.(bson.M)) +} + +type GetStatsTasksParams struct { + Query bson.M `json:"query"` +} + +func GetStatsTasks(_ *gin.Context, params *GetStatsTasksParams) (response *Response[bson.M], err error) { + query := statsDefaultQuery + if params.Query != nil { + query = params.Query + } + + data, err := stats.GetStatsService().GetTaskStats(query) + if err != nil { + return GetErrorResponse[bson.M](err) + } + return GetDataResponse(data.(bson.M)) } diff --git a/core/controllers/sync.go b/core/controllers/sync.go index e7a77f8e..81555c53 100644 --- a/core/controllers/sync.go +++ b/core/controllers/sync.go @@ -1,30 +1,36 @@ package controllers import ( + "path/filepath" + + "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/utils" "github.com/gin-gonic/gin" - "net/http" - "path/filepath" ) -func GetSyncScan(c *gin.Context) { - id := c.Param("id") - path := c.Query("path") +type GetSyncScanParams struct { + Id string `path:"id" validate:"required"` + Path string `query:"path"` +} +func GetSyncScan(_ *gin.Context, params *GetSyncScanParams) (response *Response[map[string]entity.FsFileInfo], err error) { workspacePath := utils.GetWorkspace() - dirPath := filepath.Join(workspacePath, id, path) + dirPath := filepath.Join(workspacePath, params.Id, params.Path) files, err := utils.ScanDirectory(dirPath) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[map[string]entity.FsFileInfo](err) } - c.AbortWithStatusJSON(http.StatusOK, files) + return GetDataResponse(files) } -func GetSyncDownload(c *gin.Context) { - id := c.Param("id") - path := c.Query("path") - workspacePath := utils.GetWorkspace() - filePath := filepath.Join(workspacePath, id, path) - c.File(filePath) +type GetSyncDownloadParams struct { + Id string `path:"id" validate:"required"` + Path string `query:"path" validate:"required"` +} + +func GetSyncDownload(c *gin.Context, params *GetSyncDownloadParams) (err error) { + workspacePath := utils.GetWorkspace() + filePath := filepath.Join(workspacePath, params.Id, params.Path) + c.File(filePath) + return nil } diff --git a/core/controllers/system_info.go b/core/controllers/system_info.go index 45ff983a..b95b0900 100644 --- a/core/controllers/system_info.go +++ b/core/controllers/system_info.go @@ -6,10 +6,10 @@ import ( "github.com/gin-gonic/gin" ) -func GetSystemInfo(c *gin.Context) { - info := &entity.SystemInfo{ +func GetSystemInfo(c *gin.Context) (response *Response[entity.SystemInfo], err error) { + info := entity.SystemInfo{ Edition: utils.GetEdition(), Version: utils.GetVersion(), } - HandleSuccessWithData(c, info) + return GetDataResponse(info) } diff --git a/core/controllers/task.go b/core/controllers/task.go index e341841f..23299c7f 100644 --- a/core/controllers/task.go +++ b/core/controllers/task.go @@ -2,6 +2,11 @@ package controllers import ( "errors" + "os" + "path/filepath" + "strings" + "sync" + "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" @@ -15,35 +20,31 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" - "os" - "path/filepath" - "strings" - "sync" ) -func GetTaskById(c *gin.Context) { +type GetTaskByIdParams struct { + Id string `path:"id"` +} + +func GetTaskById(_ *gin.Context, params *GetTaskByIdParams) (response *Response[models.Task], err error) { // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[models.Task](err) } // task t, err := service.NewModelService[models.Task]().GetById(id) if errors.Is(err, mongo2.ErrNoDocuments) { - HandleErrorNotFound(c, err) - return + return GetErrorResponse[models.Task](err) } if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Task](err) } // skip if task status is pending if t.Status == constants.TaskStatusPending { - HandleSuccessWithData(c, t) - return + return GetDataResponse(*t) } // spider @@ -64,40 +65,41 @@ func GetTaskById(c *gin.Context) { // task stat t.Stat, _ = service.NewModelService[models.TaskStat]().GetById(id) - HandleSuccessWithData(c, t) + return GetDataResponse(*t) } -func GetTaskList(c *gin.Context, params *GetListParams) { - withStats := c.Query("stats") - if withStats == "" { - NewController[models.Task]().GetList(c, params) - return +type GetTaskListParams struct { + *GetListParams + Stats bool `query:"stats"` +} + +func GetTaskList(c *gin.Context, params *GetTaskListParams) (response *ListResponse[models.Task], err error) { + if params.Stats { + return NewController[models.Task]().GetList(c, params.GetListParams) } - // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) + // get query + query, err := GetFilterQueryFromListParams(params.GetListParams) + if err != nil { + return GetErrorListResponse[models.Task](err) + } // get tasks tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo3.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, + Sort: params.Sort, + Skip: params.Size * (params.Page - 1), + Limit: params.Size, }) if err != nil { if errors.Is(err, mongo2.ErrNoDocuments) { - HandleErrorNotFound(c, err) - } else { - HandleErrorInternalServerError(c, err) + return GetErrorListResponse[models.Task](err) } - return + return GetErrorListResponse[models.Task](err) } // check empty list if len(tasks) == 0 { - HandleSuccessWithListData(c, nil, 0) - return + return GetListResponse[models.Task](nil, 0) } // ids @@ -111,8 +113,7 @@ func GetTaskList(c *gin.Context, params *GetListParams) { // total count total, err := service.NewModelService[models.Task]().Count(query) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.Task](err) } // stat list @@ -122,8 +123,7 @@ func GetTaskList(c *gin.Context, params *GetListParams) { }, }, nil) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.Task](err) } // cache stat list to dict @@ -139,8 +139,7 @@ func GetTaskList(c *gin.Context, params *GetListParams) { }, }, nil) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.Task](err) } // cache spider list to dict @@ -164,15 +163,17 @@ func GetTaskList(c *gin.Context, params *GetListParams) { } } - // response - HandleSuccessWithListData(c, tasks, total) + return GetListResponse(tasks, total) } -func DeleteTaskById(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) +type DeleteTaskByIdParams struct { + Id string `path:"id"` +} + +func DeleteTaskById(_ *gin.Context, params *DeleteTaskByIdParams) (response *Response[any], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[any](err) } // delete in db @@ -201,8 +202,7 @@ func DeleteTaskById(c *gin.Context) { return nil }); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[any](err) } // delete task logs @@ -211,23 +211,28 @@ func DeleteTaskById(c *gin.Context) { logger.Warnf("failed to remove task log directory: %s", logPath) } - HandleSuccess(c) + return GetDataResponse[any](nil) } -func DeleteList(c *gin.Context) { - var payload struct { - Ids []primitive.ObjectID `json:"ids"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return +type DeleteTaskListParams struct { + Ids []string `json:"ids"` +} + +func DeleteList(_ *gin.Context, params *DeleteTaskListParams) (response *Response[any], err error) { + var ids []primitive.ObjectID + for _, id := range params.Ids { + id, err := primitive.ObjectIDFromHex(id) + if err != nil { + return GetErrorResponse[any](err) + } + ids = append(ids, id) } if err := mongo3.RunTransaction(func(context mongo2.SessionContext) error { // delete tasks if err := service.NewModelService[models.Task]().DeleteMany(bson.M{ "_id": bson.M{ - "$in": payload.Ids, + "$in": ids, }, }); err != nil { return err @@ -236,7 +241,7 @@ func DeleteList(c *gin.Context) { // delete task stats if err := service.NewModelService[models.Task]().DeleteMany(bson.M{ "_id": bson.M{ - "$in": payload.Ids, + "$in": ids, }, }); err != nil { logger.Warnf("delete task stat error: %s", err.Error()) @@ -245,56 +250,66 @@ func DeleteList(c *gin.Context) { return nil }); err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[any](err) } // delete tasks logs wg := sync.WaitGroup{} - wg.Add(len(payload.Ids)) - for _, id := range payload.Ids { - go func(id string) { + wg.Add(len(ids)) + for _, id := range ids { + go func(taskId primitive.ObjectID) { // delete task logs - logPath := filepath.Join(utils.GetTaskLogPath(), id) + logPath := filepath.Join(utils.GetTaskLogPath(), taskId.Hex()) if err := os.RemoveAll(logPath); err != nil { logger.Warnf("failed to remove task log directory: %s", logPath) } wg.Done() - }(id.Hex()) + }(id) } wg.Wait() - HandleSuccess(c) + return GetDataResponse[any](nil) } -func PostTaskRun(c *gin.Context) { - // task - var t models.Task - if err := c.ShouldBindJSON(&t); err != nil { - HandleErrorBadRequest(c, err) - return +type PostTaskRunParams struct { + SpiderId string `json:"spider_id" validate:"required"` + Mode string `json:"mode"` + NodeIds []string `json:"node_ids"` + Cmd string `json:"cmd"` + Param string `json:"param"` + Priority int `json:"priority"` +} + +func PostTaskRun(c *gin.Context, params *PostTaskRunParams) (response *Response[[]primitive.ObjectID], err error) { + spiderId, err := primitive.ObjectIDFromHex(params.SpiderId) + if err != nil { + return GetErrorResponse[[]primitive.ObjectID](err) } - // validate spider id - if t.SpiderId.IsZero() { - HandleErrorBadRequest(c, errors.New("spider id is required")) - return + var nodeIds []primitive.ObjectID + if params.NodeIds != nil { + for _, nodeId := range params.NodeIds { + nodeId, err := primitive.ObjectIDFromHex(nodeId) + if err != nil { + return GetErrorResponse[[]primitive.ObjectID](err) + } + nodeIds = append(nodeIds, nodeId) + } } // spider - s, err := service.NewModelService[models.Spider]().GetById(t.SpiderId) + s, err := service.NewModelService[models.Spider]().GetById(spiderId) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } // options opts := &interfaces.SpiderRunOptions{ - Mode: t.Mode, - NodeIds: t.NodeIds, - Cmd: t.Cmd, - Param: t.Param, - Priority: t.Priority, + Mode: params.Mode, + NodeIds: nodeIds, + Cmd: params.Cmd, + Param: params.Param, + Priority: params.Priority, } // user @@ -306,27 +321,27 @@ func PostTaskRun(c *gin.Context) { adminSvc := admin.GetSpiderAdminService() taskIds, err := adminSvc.Schedule(s.Id, opts) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } - HandleSuccessWithData(c, taskIds) - + return GetDataResponse(taskIds) } -func PostTaskRestart(c *gin.Context) { +type PostTaskRestartParams struct { + Id string `path:"id"` +} + +func PostTaskRestart(c *gin.Context, params *PostTaskRestartParams) (response *Response[[]primitive.ObjectID], err error) { // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } // task t, err := service.NewModelService[models.Task]().GetById(id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } // options @@ -353,89 +368,73 @@ func PostTaskRestart(c *gin.Context) { adminSvc := admin.GetSpiderAdminService() taskIds, err := adminSvc.Schedule(t.SpiderId, opts) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[[]primitive.ObjectID](err) } - HandleSuccessWithData(c, taskIds) + return GetDataResponse(taskIds) } -func PostTaskCancel(c *gin.Context) { - type Payload struct { - Force bool `json:"force,omitempty"` - } +type PostTaskCancelParams struct { + Id string `path:"id"` + Force bool `json:"force,omitempty"` +} +func PostTaskCancel(c *gin.Context, params *PostTaskCancelParams) (response *Response[any], err error) { // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // payload - var p Payload - if err := c.ShouldBindJSON(&p); err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[any](err) } // task t, err := service.NewModelService[models.Task]().GetById(id) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[any](err) } // validate if !utils.IsCancellable(t.Status) { - HandleErrorInternalServerError(c, errors.New("task is not cancellable")) - return + return GetErrorResponse[any](errors.New("task is not cancellable")) } u := GetUserFromContext(c) // cancel schedulerSvc := scheduler.GetTaskSchedulerService() - err = schedulerSvc.Cancel(id, u.Id, p.Force) + err = schedulerSvc.Cancel(id, u.Id, params.Force) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[any](err) } - HandleSuccess(c) + return GetDataResponse[any](nil) } -func GetTaskLogs(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } +type GetTaskLogsParams struct { + Id string `path:"id"` + Page int `query:"page"` + Size int `query:"size"` +} - // pagination - p, err := GetPagination(c) +func GetTaskLogs(_ *gin.Context, params *GetTaskLogsParams) (response *ListResponse[string], err error) { + // id + id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorListResponse[string](err) } // logs logDriver := log.GetFileLogDriver() - logs, err := logDriver.Find(id.Hex(), "", (p.Page-1)*p.Size, p.Size) + logs, err := logDriver.Find(id.Hex(), "", (params.Page-1)*params.Size, params.Size) if err != nil { if strings.HasSuffix(err.Error(), "Status:404 Not Found") { - HandleSuccess(c) - return + return GetListResponse[string](nil, 0) } - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[string](err) } total, err := logDriver.Count(id.Hex(), "") if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[string](err) } - HandleSuccessWithListData(c, logs, total) + return GetListResponse(logs, total) } diff --git a/core/controllers/token.go b/core/controllers/token.go index bfa079a3..1fab41c2 100644 --- a/core/controllers/token.go +++ b/core/controllers/token.go @@ -1,55 +1,51 @@ package controllers import ( - "errors" - "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/user" "github.com/gin-gonic/gin" + "github.com/juju/errors" mongo2 "go.mongodb.org/mongo-driver/mongo" ) -func PostToken(c *gin.Context) { - var t models.Token - if err := c.ShouldBindJSON(&t); err != nil { - HandleErrorBadRequest(c, err) - return - } +type PostTokenParams struct { + Data models.Token `json:"data"` +} + +func PostToken(c *gin.Context, params *PostTokenParams) (response *Response[models.Token], err error) { + t := params.Data svc, err := user.GetUserService() if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Token](err) } + u := GetUserFromContext(c) t.SetCreated(u.Id) t.SetUpdated(u.Id) t.Token, err = svc.MakeToken(u) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Token](err) } - _, err = service.NewModelService[models.Token]().InsertOne(t) + + id, err := service.NewModelService[models.Token]().InsertOne(t) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorResponse[models.Token](err) } - HandleSuccess(c) + t.Id = id + + return GetDataResponse(t) } -func GetTokenList(c *gin.Context) { +func GetTokenList(c *gin.Context, params *GetListParams) (response *ListResponse[models.Token], err error) { // Get current user from context u := GetUserFromContext(c) - // Get pagination, filter query, and sort options - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - - // If query is nil, initialize it - if query == nil { - query = make(map[string]interface{}) + // Get filter query + query, err := GetFilterQueryFromListParams(params) + if err != nil { + return GetErrorListResponse[models.Token](errors.BadRequestf("invalid request parameters: %v", err)) } // Add filter for tokens created by the current user @@ -57,26 +53,22 @@ func GetTokenList(c *gin.Context) { // Get tokens with pagination tokens, err := service.NewModelService[models.Token]().GetMany(query, &mongo.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, + Sort: params.Sort, + Skip: params.Size * (params.Page - 1), + Limit: params.Size, }) if err != nil { - if errors.Is(err, mongo2.ErrNoDocuments) { - HandleSuccessWithListData(c, nil, 0) - } else { - HandleErrorInternalServerError(c, err) + if err == mongo2.ErrNoDocuments { + return GetListResponse([]models.Token{}, 0) } - return + return GetErrorListResponse[models.Token](err) } // Count total tokens for pagination total, err := service.NewModelService[models.Token]().Count(query) if err != nil { - HandleErrorInternalServerError(c, err) - return + return GetErrorListResponse[models.Token](err) } - // Return tokens with total count - HandleSuccessWithListData(c, tokens, total) + return GetListResponse(tokens, total) } diff --git a/core/controllers/user.go b/core/controllers/user.go index 184a0dd2..435b534f 100644 --- a/core/controllers/user.go +++ b/core/controllers/user.go @@ -15,11 +15,10 @@ import ( mongo2 "go.mongodb.org/mongo-driver/mongo" ) -func GetUserById(c *gin.Context, params *GetByIdParams) (response *Response[models.User], err error) { +func GetUserById(_ *gin.Context, params *GetByIdParams) (response *Response[models.User], err error) { id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - HandleErrorBadRequest(c, err) - return + return GetErrorResponse[models.User](errors.BadRequestf("invalid user id: %v", err)) } return getUserById(id) } @@ -80,7 +79,7 @@ func GetUserList(_ *gin.Context, params *GetListParams) (response *ListResponse[ } // response - return GetListResponse[models.User](users, total) + return GetListResponse(users, total) } type PostUserParams struct { @@ -126,7 +125,7 @@ func PostUser(c *gin.Context, params *PostUserParams) (response *Response[models return GetErrorResponse[models.User](err) } - return GetDataResponse[models.User](*result) + return GetDataResponse(*result) } func PutUserById(c *gin.Context, params *PutByIdParams[models.User]) (response *Response[models.User], err error) { @@ -168,7 +167,7 @@ func DeleteUserById(_ *gin.Context, params *DeleteByIdParams) (response *Respons return GetErrorResponse[models.User](err) } - return GetDataResponse[models.User](models.User{}) + return GetDataResponse(models.User{}) } func DeleteUserList(_ *gin.Context, params *DeleteListParams) (response *Response[models.User], err error) { @@ -205,7 +204,7 @@ func DeleteUserList(_ *gin.Context, params *DeleteListParams) (response *Respons return GetErrorResponse[models.User](err) } - return GetDataResponse[models.User](models.User{}) + return GetDataResponse(models.User{}) } func GetUserMe(c *gin.Context) (response *Response[models.User], err error) { @@ -253,7 +252,7 @@ func getUserById(userId primitive.ObjectID) (response *Response[models.User], er } } - return GetDataResponse[models.User](*user) + return GetDataResponse(*user) } func getUserByIdWithRoutes(userId primitive.ObjectID) (response *Response[models.User], err error) { @@ -284,7 +283,7 @@ func getUserByIdWithRoutes(userId primitive.ObjectID) (response *Response[models user.Routes = role.Routes } - return GetDataResponse[models.User](*user) + return GetDataResponse(*user) } func putUser(userId, by primitive.ObjectID, user models.User) (response *Response[models.User], err error) { @@ -319,7 +318,7 @@ func putUser(userId, by primitive.ObjectID, user models.User) (response *Respons } // handle success - return GetDataResponse[models.User](user) + return GetDataResponse(user) } func postUserChangePassword(userId, by primitive.ObjectID, password string) (response *Response[models.User], err error) { @@ -338,5 +337,5 @@ func postUserChangePassword(userId, by primitive.ObjectID, password string) (res return GetErrorResponse[models.User](err) } - return GetDataResponse[models.User](models.User{}) + return GetDataResponse(models.User{}) }