diff --git a/core/controllers/base_file.go b/core/controllers/base_file.go index 1384d1f0..c43e6ec4 100644 --- a/core/controllers/base_file.go +++ b/core/controllers/base_file.go @@ -62,46 +62,46 @@ type PostBaseFileSaveOneParams struct { Data string `json:"data"` } -func PostBaseFileSaveOne(rootPath, path, data string) (response *Response[any], err error) { +func PostBaseFileSaveOne(rootPath, path, data string) (response *VoidResponse, err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if err := fsSvc.Save(path, []byte(data)); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } - return GetDataResponse(any(data)) + return GetVoidResponse() } -func PostBaseFileSaveOneForm(rootPath, path string, file *multipart.FileHeader) (response *Response[any], err error) { +func PostBaseFileSaveOneForm(rootPath, path string, file *multipart.FileHeader) (response *VoidResponse, err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } f, err := file.Open() if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } fileData, err := io.ReadAll(f) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if err := fsSvc.Save(path, fileData); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } - return GetDataResponse[any](nil) + return GetVoidResponse() } -func PostBaseFileSaveMany(rootPath string, form *multipart.Form) (response *Response[any], err error) { +func PostBaseFileSaveMany(rootPath string, form *multipart.Form) (response *VoidResponse, err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } wg := sync.WaitGroup{} @@ -140,67 +140,67 @@ func PostBaseFileSaveMany(rootPath string, form *multipart.Form) (response *Resp } wg.Wait() - return GetDataResponse[any](nil) + return GetVoidResponse() } -func PostBaseFileSaveDir(rootPath, path string) (response *Response[any], err error) { +func PostBaseFileSaveDir(rootPath, path string) (response *VoidResponse, err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if err := fsSvc.CreateDir(path); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } - return GetDataResponse[any](nil) + return GetVoidResponse() } -func PostBaseFileRename(rootPath, path, newPath string) (response *Response[any], err error) { +func PostBaseFileRename(rootPath, path, newPath string) (response *VoidResponse, err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if err := fsSvc.Rename(path, newPath); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } - return GetDataResponse[any](nil) + return GetVoidResponse() } -func DeleteBaseFile(rootPath, path string) (response *Response[any], err error) { +func DeleteBaseFile(rootPath, path string) (response *VoidResponse, err error) { if path == "~" { path = "." } fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if err := fsSvc.Delete(path); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } _, err = fsSvc.GetFileInfo(".") if err != nil { _ = fsSvc.CreateDir("/") } - return GetDataResponse[any](nil) + return GetVoidResponse() } -func PostBaseFileCopy(rootPath, path, newPath string) (response *Response[any], err error) { +func PostBaseFileCopy(rootPath, path, newPath string) (response *VoidResponse, err error) { fsSvc, err := fs.GetBaseFileFsSvc(rootPath) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if err := fsSvc.Copy(path, newPath); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } - return GetDataResponse[any](nil) + return GetVoidResponse() } func PostBaseFileExport(rootPath string, c *gin.Context) (err error) { diff --git a/core/controllers/export.go b/core/controllers/export.go index 3282cacc..fe91e531 100644 --- a/core/controllers/export.go +++ b/core/controllers/export.go @@ -11,18 +11,22 @@ import ( ) type PostExportParams struct { - Type string `path:"type" validate:"required"` - Target string `query:"target" validate:"required"` - Filter interfaces.Filter `query:"filter"` + Type string `path:"type" validate:"required"` + Target string `query:"target" validate:"required"` + Conditions string `query:"conditions" description:"Filter conditions. Format: [{\"key\":\"name\",\"op\":\"eq\",\"value\":\"test\"}]"` } func PostExport(_ *gin.Context, params *PostExportParams) (response *Response[string], err error) { + query, err := GetFilterQueryFromConditionString(params.Conditions) + if err != nil { + return GetErrorResponse[string](err) + } var exportId string switch params.Type { case constants.ExportTypeCsv: - exportId, err = export.GetCsvService().Export(params.Type, params.Target, params.Filter) + exportId, err = export.GetCsvService().Export(params.Type, params.Target, query) case constants.ExportTypeJson: - exportId, err = export.GetJsonService().Export(params.Type, params.Target, params.Filter) + exportId, err = export.GetJsonService().Export(params.Type, params.Target, query) default: return GetErrorResponse[string](errors.BadRequestf("invalid export type: %s", params.Type)) } diff --git a/core/controllers/filter.go b/core/controllers/filter.go index d616f2f6..0afcbe5f 100644 --- a/core/controllers/filter.go +++ b/core/controllers/filter.go @@ -10,13 +10,39 @@ import ( ) type GetFilterColFieldOptionsParams struct { + Col string `path:"col" validate:"required"` + Conditions string `query:"conditions" description:"Filter conditions. Format: [{\"key\":\"name\",\"op\":\"eq\",\"value\":\"test\"}]"` +} + +func GetFilterColFieldOptions(c *gin.Context, params *GetFilterColFieldOptionsParams) (response *Response[[]entity.FilterSelectOption], err error) { + return GetFilterColFieldOptionsWithValueLabel(c, &GetFilterColFieldOptionsWithValueLabelParams{ + Col: params.Col, + Conditions: params.Conditions, + }) +} + +type GetFilterColFieldOptionsWithValueParams struct { + Col string `path:"col" validate:"required"` + Value string `path:"value"` + Conditions string `query:"conditions" description:"Filter conditions. Format: [{\"key\":\"name\",\"op\":\"eq\",\"value\":\"test\"}]"` +} + +func GetFilterColFieldOptionsWithValue(c *gin.Context, params *GetFilterColFieldOptionsWithValueParams) (response *Response[[]entity.FilterSelectOption], err error) { + return GetFilterColFieldOptionsWithValueLabel(c, &GetFilterColFieldOptionsWithValueLabelParams{ + Col: params.Col, + Value: params.Value, + Conditions: params.Conditions, + }) +} + +type GetFilterColFieldOptionsWithValueLabelParams struct { Col string `path:"col" validate:"required"` Value string `path:"value"` Label string `path:"label"` Conditions string `query:"conditions" description:"Filter conditions. Format: [{\"key\":\"name\",\"op\":\"eq\",\"value\":\"test\"}]"` } -func GetFilterColFieldOptions(_ *gin.Context, params *GetFilterColFieldOptionsParams) (response *Response[[]entity.FilterSelectOption], err error) { +func GetFilterColFieldOptionsWithValueLabel(_ *gin.Context, params *GetFilterColFieldOptionsWithValueLabelParams) (response *Response[[]entity.FilterSelectOption], err error) { value := params.Value if value == "" { value = "_id" diff --git a/core/controllers/health.go b/core/controllers/health.go index bf687a5e..37e71db4 100644 --- a/core/controllers/health.go +++ b/core/controllers/health.go @@ -1,20 +1,17 @@ package controllers import ( - "errors" "github.com/gin-gonic/gin" "net/http" ) -func GetHealthFn(healthFn func() bool) func(c *gin.Context) error { - return func(c *gin.Context) (err error) { +func GetHealthFn(healthFn func() bool) func(c *gin.Context) { + return func(c *gin.Context) { if healthFn() { - c.Writer.Write([]byte("ok")) + _, _ = c.Writer.Write([]byte("ok")) c.AbortWithStatus(http.StatusOK) - return } - c.Writer.Write([]byte("not ready")) + _, _ = c.Writer.Write([]byte("not ready")) c.AbortWithStatus(http.StatusServiceUnavailable) - return errors.New("not ready") } } diff --git a/core/controllers/login.go b/core/controllers/login.go index 1a174b12..b95806f4 100644 --- a/core/controllers/login.go +++ b/core/controllers/login.go @@ -27,6 +27,6 @@ func PostLogin(c *gin.Context, params *PostLoginParams) (response *Response[stri return GetDataResponse(token) } -func PostLogout(_ *gin.Context) (response *Response[any], err error) { - return GetDataResponse[any](nil) +func PostLogout(_ *gin.Context) (response *VoidResponse, err error) { + return GetVoidResponse() } diff --git a/core/controllers/router.go b/core/controllers/router.go index 95aa7805..fc6c5450 100644 --- a/core/controllers/router.go +++ b/core/controllers/router.go @@ -29,7 +29,7 @@ func GetGlobalFizzWrapper() *openapi.FizzWrapper { // NewRouterGroups initializes the router groups with their respective middleware func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) { // Create OpenAPI wrapper - globalWrapper = openapi.NewFizzWrapper(app) + globalWrapper = openapi.GetFizzWrapper(app) return &RouterGroups{ AuthGroup: app.Group("/", middlewares.AuthorizationMiddleware()), @@ -485,12 +485,12 @@ func InitRoutes(app *gin.Engine) (err error) { { Method: http.MethodGet, Path: "/:col/:value", - HandlerFunc: GetFilterColFieldOptions, + HandlerFunc: GetFilterColFieldOptionsWithValue, }, { Method: http.MethodGet, Path: "/:col/:value/:label", - HandlerFunc: GetFilterColFieldOptions, + HandlerFunc: GetFilterColFieldOptionsWithValueLabel, }, }) RegisterActions(groups.AuthGroup, "/settings", []Action{ @@ -543,13 +543,6 @@ func InitRoutes(app *gin.Engine) (err error) { }) // Register public routes that don't require authentication - RegisterActions(groups.AnonymousGroup, "/health", []Action{ - { - Path: "", - Method: http.MethodGet, - HandlerFunc: GetHealthFn(func() bool { return true }), - }, - }) RegisterActions(groups.AnonymousGroup, "/system-info", []Action{ { Path: "", @@ -570,6 +563,9 @@ func InitRoutes(app *gin.Engine) (err error) { }, }) + // Register health check route + groups.AnonymousGroup.GET("/health", GetHealthFn(func() bool { return true })) + // Register OpenAPI documentation route groups.AnonymousGroup.GET("/openapi.json", GetOpenAPI) diff --git a/core/controllers/schedule.go b/core/controllers/schedule.go index 4a5bf363..c98d6f95 100644 --- a/core/controllers/schedule.go +++ b/core/controllers/schedule.go @@ -83,26 +83,26 @@ type PostScheduleEnableDisableParams struct { Id string `path:"id"` } -func PostScheduleEnable(c *gin.Context, params *PostScheduleEnableDisableParams) (response *Response[any], err error) { +func PostScheduleEnable(c *gin.Context, params *PostScheduleEnableDisableParams) (response *VoidResponse, err error) { userId := GetUserFromContext(c).Id return postScheduleEnableDisableFunc(true, userId, params) } -func PostScheduleDisable(c *gin.Context, params *PostScheduleEnableDisableParams) (response *Response[any], err error) { +func PostScheduleDisable(c *gin.Context, params *PostScheduleEnableDisableParams) (response *VoidResponse, err error) { userId := GetUserFromContext(c).Id return postScheduleEnableDisableFunc(false, userId, params) } -func postScheduleEnableDisableFunc(isEnable bool, userId primitive.ObjectID, params *PostScheduleEnableDisableParams) (response *Response[any], err error) { +func postScheduleEnableDisableFunc(isEnable bool, userId primitive.ObjectID, params *PostScheduleEnableDisableParams) (response *VoidResponse, err error) { id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - return GetErrorResponse[any](errors.BadRequestf("invalid schedule id: %v", err)) + return GetErrorVoidResponse(errors.BadRequestf("invalid schedule id: %v", err)) } svc := schedule.GetScheduleService() s, err := service.NewModelService[models.Schedule]().GetById(id) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if isEnable { @@ -111,10 +111,10 @@ func postScheduleEnableDisableFunc(isEnable bool, userId primitive.ObjectID, par err = svc.Disable(*s, userId) } if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } - return GetDataResponse[any](nil) + return GetVoidResponse() } type PostScheduleRunParams struct { diff --git a/core/controllers/spider.go b/core/controllers/spider.go index 8bbbb5c3..722efeb3 100644 --- a/core/controllers/spider.go +++ b/core/controllers/spider.go @@ -575,10 +575,10 @@ type PostSpiderSaveFileParams struct { File *multipart.FileHeader `form:"file"` } -func PostSpiderSaveFile(c *gin.Context, params *PostSpiderSaveFileParams) (response *Response[any], err error) { +func PostSpiderSaveFile(c *gin.Context, params *PostSpiderSaveFileParams) (response *VoidResponse, err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } if c.GetHeader("Content-Type") == "application/json" { return PostBaseFileSaveOne(rootPath, params.Path, params.Data) @@ -592,14 +592,14 @@ type PostSpiderSaveFilesParams struct { TargetDirectory string `form:"targetDirectory"` } -func PostSpiderSaveFiles(c *gin.Context, params *PostSpiderSaveFilesParams) (response *Response[any], err error) { +func PostSpiderSaveFiles(c *gin.Context, params *PostSpiderSaveFilesParams) (response *VoidResponse, err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } form, err := c.MultipartForm() if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } return PostBaseFileSaveMany(filepath.Join(rootPath, params.TargetDirectory), form) } @@ -609,10 +609,10 @@ type PostSpiderSaveDirParams struct { Path string `json:"path"` } -func PostSpiderSaveDir(c *gin.Context, params *PostSpiderSaveDirParams) (response *Response[any], err error) { +func PostSpiderSaveDir(c *gin.Context, params *PostSpiderSaveDirParams) (response *VoidResponse, err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } return PostBaseFileSaveDir(rootPath, params.Path) } @@ -623,10 +623,10 @@ type PostSpiderRenameFileParams struct { NewPath string `json:"newPath"` } -func PostSpiderRenameFile(c *gin.Context, params *PostSpiderRenameFileParams) (response *Response[any], err error) { +func PostSpiderRenameFile(c *gin.Context, params *PostSpiderRenameFileParams) (response *VoidResponse, err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } return PostBaseFileRename(rootPath, params.Path, params.NewPath) } @@ -636,10 +636,10 @@ type DeleteSpiderFileParams struct { Path string `json:"path"` } -func DeleteSpiderFile(c *gin.Context, params *DeleteSpiderFileParams) (response *Response[any], err error) { +func DeleteSpiderFile(c *gin.Context, params *DeleteSpiderFileParams) (response *VoidResponse, err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } return DeleteBaseFile(rootPath, params.Path) } @@ -650,10 +650,10 @@ type PostSpiderCopyFileParams struct { NewPath string `json:"new_path"` } -func PostSpiderCopyFile(c *gin.Context, params *PostSpiderCopyFileParams) (response *Response[any], err error) { +func PostSpiderCopyFile(c *gin.Context, params *PostSpiderCopyFileParams) (response *VoidResponse, err error) { rootPath, err := getSpiderRootPathByContext(c) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } return PostBaseFileCopy(rootPath, params.Path, params.NewPath) } diff --git a/core/controllers/task.go b/core/controllers/task.go index 23299c7f..25fb80e8 100644 --- a/core/controllers/task.go +++ b/core/controllers/task.go @@ -170,10 +170,10 @@ type DeleteTaskByIdParams struct { Id string `path:"id"` } -func DeleteTaskById(_ *gin.Context, params *DeleteTaskByIdParams) (response *Response[any], err error) { +func DeleteTaskById(_ *gin.Context, params *DeleteTaskByIdParams) (response *VoidResponse, err error) { id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } // delete in db @@ -202,7 +202,7 @@ func DeleteTaskById(_ *gin.Context, params *DeleteTaskByIdParams) (response *Res return nil }); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } // delete task logs @@ -211,19 +211,19 @@ func DeleteTaskById(_ *gin.Context, params *DeleteTaskByIdParams) (response *Res logger.Warnf("failed to remove task log directory: %s", logPath) } - return GetDataResponse[any](nil) + return GetVoidResponse() } type DeleteTaskListParams struct { Ids []string `json:"ids"` } -func DeleteList(_ *gin.Context, params *DeleteTaskListParams) (response *Response[any], err error) { +func DeleteList(_ *gin.Context, params *DeleteTaskListParams) (response *VoidResponse, err error) { var ids []primitive.ObjectID for _, id := range params.Ids { id, err := primitive.ObjectIDFromHex(id) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } ids = append(ids, id) } @@ -250,7 +250,7 @@ func DeleteList(_ *gin.Context, params *DeleteTaskListParams) (response *Respons return nil }); err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } // delete tasks logs @@ -268,7 +268,7 @@ func DeleteList(_ *gin.Context, params *DeleteTaskListParams) (response *Respons } wg.Wait() - return GetDataResponse[any](nil) + return GetVoidResponse() } type PostTaskRunParams struct { @@ -379,22 +379,22 @@ type PostTaskCancelParams struct { Force bool `json:"force,omitempty"` } -func PostTaskCancel(c *gin.Context, params *PostTaskCancelParams) (response *Response[any], err error) { +func PostTaskCancel(c *gin.Context, params *PostTaskCancelParams) (response *VoidResponse, err error) { // id id, err := primitive.ObjectIDFromHex(params.Id) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } // task t, err := service.NewModelService[models.Task]().GetById(id) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } // validate if !utils.IsCancellable(t.Status) { - return GetErrorResponse[any](errors.New("task is not cancellable")) + return GetErrorVoidResponse(errors.New("task is not cancellable")) } u := GetUserFromContext(c) @@ -403,10 +403,10 @@ func PostTaskCancel(c *gin.Context, params *PostTaskCancelParams) (response *Res schedulerSvc := scheduler.GetTaskSchedulerService() err = schedulerSvc.Cancel(id, u.Id, params.Force) if err != nil { - return GetErrorResponse[any](err) + return GetErrorVoidResponse(err) } - return GetDataResponse[any](nil) + return GetVoidResponse() } type GetTaskLogsParams struct { diff --git a/core/controllers/utils.go b/core/controllers/utils.go index ece41ddf..b61bf8b1 100644 --- a/core/controllers/utils.go +++ b/core/controllers/utils.go @@ -263,6 +263,12 @@ type ListResponse[T any] struct { Error string `json:"error"` } +type VoidResponse struct { + Status string `json:"status"` + Message string `json:"message"` + Error string `json:"error"` +} + func GetDataResponse[T any](model T) (res *Response[T], err error) { return &Response[T]{ Status: constants.HttpResponseStatusOk, @@ -280,6 +286,13 @@ func GetListResponse[T any](models []T, total int) (res *ListResponse[T], err er }, nil } +func GetVoidResponse() (res *VoidResponse, err error) { + return &VoidResponse{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageSuccess, + }, nil +} + func GetErrorResponse[T any](err error) (res *Response[T], err2 error) { return &Response[T]{ Status: constants.HttpResponseStatusOk, @@ -288,6 +301,14 @@ func GetErrorResponse[T any](err error) (res *Response[T], err2 error) { }, err } +func GetErrorVoidResponse(err error) (res *VoidResponse, err2 error) { + return &VoidResponse{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageError, + Error: err.Error(), + }, err +} + func GetErrorListResponse[T any](err error) (res *ListResponse[T], err2 error) { return &ListResponse[T]{ Status: constants.HttpResponseStatusOk, diff --git a/core/entity/export.go b/core/entity/export.go index 5bd06a7b..62605659 100644 --- a/core/entity/export.go +++ b/core/entity/export.go @@ -1,21 +1,21 @@ package entity import ( - "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" "time" ) type Export struct { - Id string `json:"id"` - Type string `json:"type"` - Target string `json:"target"` - Filter interfaces.Filter `json:"filter"` - Status string `json:"status"` - StartTs time.Time `json:"start_ts"` - EndTs time.Time `json:"end_ts"` - FileName string `json:"file_name"` - DownloadPath string `json:"-"` - Limit int `json:"-"` + Id string `json:"id"` + Type string `json:"type"` + Target string `json:"target"` + Query bson.M `json:"query"` + Status string `json:"status"` + StartTs time.Time `json:"start_ts"` + EndTs time.Time `json:"end_ts"` + FileName string `json:"file_name"` + DownloadPath string `json:"-"` + Limit int `json:"-"` } func (e *Export) GetId() string { @@ -30,8 +30,8 @@ func (e *Export) GetTarget() string { return e.Target } -func (e *Export) GetFilter() interfaces.Filter { - return e.Filter +func (e *Export) GetQuery() bson.M { + return e.Query } func (e *Export) GetStatus() string { diff --git a/core/export/csv_service.go b/core/export/csv_service.go index aac21876..83bf962d 100644 --- a/core/export/csv_service.go +++ b/core/export/csv_service.go @@ -36,7 +36,7 @@ func (svc *CsvService) GenerateId() (exportId string, err error) { return exportId, nil } -func (svc *CsvService) Export(exportType, target string, filter interfaces.Filter) (exportId string, err error) { +func (svc *CsvService) Export(exportType, target string, query bson.M) (exportId string, err error) { // generate export id exportId, err = svc.GenerateId() if err != nil { @@ -48,7 +48,7 @@ func (svc *CsvService) Export(exportType, target string, filter interfaces.Filte Id: exportId, Type: exportType, Target: target, - Filter: filter, + Query: query, Status: constants.TaskStatusRunning, StartTs: time.Now(), FileName: svc.getFileName(exportId), @@ -90,18 +90,8 @@ func (svc *CsvService) export(export *entity.Export) { // mongo collection col := mongo.GetMongoCol(export.Target) - // mongo query - query, err := utils.FilterToQuery(export.Filter) - if err != nil { - export.Status = constants.TaskStatusError - export.EndTs = time.Now() - svc.Errorf("export error (id: %s): %v", export.Id, err) - svc.cache.Set(export.Id, export) - return - } - // mongo cursor - cur := col.Find(query, nil).GetCursor() + cur := col.Find(export.Query, nil).GetCursor() // csv writer csvWriter, csvFile, err := svc.getCsvWriter(export) @@ -126,7 +116,7 @@ func (svc *CsvService) export(export *entity.Export) { } // write csv header row - columns, err := svc.getColumns(query, export) + columns, err := svc.getColumns(export.Query, export) err = csvWriter.Write(columns) if err != nil { export.Status = constants.TaskStatusError diff --git a/core/export/json_service.go b/core/export/json_service.go index 0929d9e2..66813b47 100644 --- a/core/export/json_service.go +++ b/core/export/json_service.go @@ -11,6 +11,7 @@ import ( "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/utils" "github.com/hashicorp/go-uuid" + "go.mongodb.org/mongo-driver/bson" mongo2 "go.mongodb.org/mongo-driver/mongo" "os" "path" @@ -31,7 +32,7 @@ func (svc *JsonService) GenerateId() (exportId string, err error) { return exportId, nil } -func (svc *JsonService) Export(exportType, target string, filter interfaces.Filter) (exportId string, err error) { +func (svc *JsonService) Export(exportType, target string, query bson.M) (exportId string, err error) { // generate export id exportId, err = svc.GenerateId() if err != nil { @@ -43,7 +44,7 @@ func (svc *JsonService) Export(exportType, target string, filter interfaces.Filt Id: exportId, Type: exportType, Target: target, - Filter: filter, + Query: query, Status: constants.TaskStatusRunning, StartTs: time.Now(), FileName: svc.getFileName(exportId), @@ -85,18 +86,8 @@ func (svc *JsonService) export(export *entity.Export) { // mongo collection col := mongo.GetMongoCol(export.Target) - // mongo query - query, err := utils.FilterToQuery(export.Filter) - if err != nil { - export.Status = constants.TaskStatusError - export.EndTs = time.Now() - svc.Errorf("export error (id: %s): %v", export.Id, err) - svc.cache.Set(export.Id, export) - return - } - // mongo cursor - cur := col.Find(query, nil).GetCursor() + cur := col.Find(export.Query, nil).GetCursor() // data var jsonData []interface{} diff --git a/core/interfaces/export.go b/core/interfaces/export.go index 2205b842..b9951711 100644 --- a/core/interfaces/export.go +++ b/core/interfaces/export.go @@ -1,12 +1,15 @@ package interfaces -import "time" +import ( + "go.mongodb.org/mongo-driver/bson" + "time" +) type Export interface { GetId() string GetType() string GetTarget() string - GetFilter() Filter + GetQuery() bson.M GetStatus() string GetStartTs() time.Time GetEndTs() time.Time diff --git a/core/interfaces/export_service.go b/core/interfaces/export_service.go index d48ae2b8..a21dfa6e 100644 --- a/core/interfaces/export_service.go +++ b/core/interfaces/export_service.go @@ -1,7 +1,9 @@ package interfaces +import "go.mongodb.org/mongo-driver/bson" + type ExportService interface { GenerateId() (exportId string, err error) - Export(exportType, target string, filter Filter) (exportId string, err error) + Export(exportType, target string, query bson.M) (exportId string, err error) GetExport(exportId string) (export Export, err error) } diff --git a/core/openapi/wrapper.go b/core/openapi/wrapper.go index 8f8214a1..86ea281e 100644 --- a/core/openapi/wrapper.go +++ b/core/openapi/wrapper.go @@ -4,10 +4,10 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/utils" - "github.com/loopfz/gadgeto/tonic" - "github.com/gin-gonic/gin" + "github.com/loopfz/gadgeto/tonic" "github.com/wI2L/fizz" + "sync" ) // FizzWrapper wraps an existing Gin Engine to add OpenAPI functionality @@ -17,9 +17,9 @@ type FizzWrapper struct { logger interfaces.Logger } -// NewFizzWrapper creates a new wrapper around an existing Gin Engine +// newFizzWrapper creates a new wrapper around an existing Gin Engine // This approach ensures we don't break existing functionality -func NewFizzWrapper(engine *gin.Engine) *FizzWrapper { +func newFizzWrapper(engine *gin.Engine) *FizzWrapper { // Create a new Fizz instance using the existing Gin engine f := fizz.NewFromEngine(engine) return &FizzWrapper{ @@ -116,3 +116,13 @@ func (w *FizzWrapper) buildOperationOptions(id, summary, description string, res return opts } + +var wrapper *FizzWrapper +var wrapperOnce sync.Once + +func GetFizzWrapper(app *gin.Engine) *FizzWrapper { + wrapperOnce.Do(func() { + wrapper = newFizzWrapper(app) + }) + return wrapper +}