From b2f50dbceade70219f80035d526e4f7674ab4be6 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 17 Oct 2024 20:54:25 +0800 Subject: [PATCH] fix: results data display issue --- core/controllers/result_v2.go | 119 ------------------------------- core/controllers/router_v2.go | 17 +---- core/controllers/spider_v2.go | 61 ++++------------ core/controllers/utils_filter.go | 16 +++++ 4 files changed, 32 insertions(+), 181 deletions(-) delete mode 100644 core/controllers/result_v2.go diff --git a/core/controllers/result_v2.go b/core/controllers/result_v2.go deleted file mode 100644 index 662ef1f4..00000000 --- a/core/controllers/result_v2.go +++ /dev/null @@ -1,119 +0,0 @@ -package controllers - -import ( - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/result" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" -) - -func GetResultList(c *gin.Context) { - // data collection id - dcId, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // data source id - var dsId primitive.ObjectID - dsIdStr := c.Query("data_source_id") - if dsIdStr != "" { - dsId, err = primitive.ObjectIDFromHex(dsIdStr) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - } - - // data collection - dc, err := service.NewModelServiceV2[models2.DataCollectionV2]().GetById(dcId) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // data source - ds, err := service.NewModelServiceV2[models2.DatabaseV2]().GetById(dsId) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - ds = &models2.DatabaseV2{} - } else { - HandleErrorInternalServerError(c, err) - return - } - } - - // spider - sq := bson.M{ - "col_id": dc.Id, - "data_source_id": ds.Id, - } - s, err := service.NewModelServiceV2[models2.SpiderV2]().GetOne(sq, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // service - svc, err := result.GetResultService(s.Id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // params - pagination := MustGetPagination(c) - query := getResultListQuery(c) - - // get results - data, err := svc.List(query, &generic.ListOptions{ - Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}}, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, - }) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - HandleSuccessWithListData(c, nil, 0) - return - } - HandleErrorInternalServerError(c, err) - return - } - - // validate results - if len(data) == 0 { - HandleSuccessWithListData(c, nil, 0) - return - } - - // total count - total, err := svc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // response - HandleSuccessWithListData(c, data, total) -} - -func getResultListQuery(c *gin.Context) (q generic.ListQuery) { - f, err := GetFilter(c) - if err != nil { - return q - } - for _, cond := range f.Conditions { - q = append(q, generic.ListQueryCondition{ - Key: cond.Key, - Op: cond.Op, - Value: utils.NormalizeObjectId(cond.Value), - }) - } - return q -} diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index 36ee2a26..55cb78d6 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -173,16 +173,10 @@ func InitRoutes(app *gin.Engine) (err error) { Path: "/:id/run", HandlerFunc: PostSpiderRun, }, - { Method: http.MethodGet, - Path: "/:id/data-source", - HandlerFunc: GetSpiderDataSource, - }, - { - Method: http.MethodPost, - Path: "/:id/data-source/:ds_id", - HandlerFunc: PostSpiderDataSource, + Path: "/:id/results", + HandlerFunc: GetSpiderResults, }, }...)) RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models2.TaskV2]([]Action{ @@ -262,13 +256,6 @@ func InitRoutes(app *gin.Engine) (err error) { }, }...)) - RegisterActions(groups.AuthGroup, "/results", []Action{ - { - Method: http.MethodGet, - Path: "/:id", - HandlerFunc: GetResultList, - }, - }) RegisterActions(groups.AuthGroup, "/export", []Action{ { Method: http.MethodPost, diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go index 76d27038..0bb5ac23 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider_v2.go @@ -9,6 +9,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/crawlab/db/generic" "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/trace" "github.com/gin-gonic/gin" @@ -670,77 +671,43 @@ func PostSpiderRun(c *gin.Context) { HandleSuccessWithData(c, taskIds) } -func GetSpiderDataSource(c *gin.Context) { - // spider id +func GetSpiderResults(c *gin.Context) { id, err := primitive.ObjectIDFromHex(c.Param("id")) if err != nil { HandleErrorBadRequest(c, err) return } - // spider s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return } - // data source - ds, err := service.NewModelServiceV2[models2.DatabaseV2]().GetById(s.DataSourceId) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - HandleSuccess(c) - return - } - HandleErrorInternalServerError(c, err) - return - } + // params + pagination := MustGetPagination(c) + query := getResultListQuery(c) - HandleSuccessWithData(c, ds) -} + col := mongo.GetMongoCol(s.ColName) -func PostSpiderDataSource(c *gin.Context) { - // spider id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // data source id - dsId, err := primitive.ObjectIDFromHex(c.Param("ds_id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider - s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) + var results []bson.M + err = col.Find(utils.GetMongoQuery(query), utils.GetMongoOpts(&generic.ListOptions{ + Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}}, + Skip: pagination.Size * (pagination.Page - 1), + Limit: pagination.Size, + })).All(&results) if err != nil { HandleErrorInternalServerError(c, err) return } - // data source - if !dsId.IsZero() { - _, err = service.NewModelServiceV2[models2.DatabaseV2]().GetById(dsId) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - // save data source id - u := GetUserFromContextV2(c) - s.DataSourceId = dsId - s.SetUpdatedBy(u.Id) - _, err = service.NewModelServiceV2[models2.SpiderV2]().InsertOne(*s) + total, err := mongo.GetMongoCol(s.ColName).Count(utils.GetMongoQuery(query)) if err != nil { HandleErrorInternalServerError(c, err) return } - HandleSuccess(c) + HandleSuccessWithListData(c, results, total) } func getSpiderFsSvc(s *models2.SpiderV2) (svc interfaces.FsServiceV2, err error) { diff --git a/core/controllers/utils_filter.go b/core/controllers/utils_filter.go index 9a7b619c..a4fef2d1 100644 --- a/core/controllers/utils_filter.go +++ b/core/controllers/utils_filter.go @@ -6,6 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/crawlab/db/generic" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -121,3 +122,18 @@ func MustGetFilterAll(c *gin.Context) (res bool) { } return res } + +func getResultListQuery(c *gin.Context) (q generic.ListQuery) { + f, err := GetFilter(c) + if err != nil { + return q + } + for _, cond := range f.Conditions { + q = append(q, generic.ListQueryCondition{ + Key: cond.Key, + Op: cond.Op, + Value: utils.NormalizeObjectId(cond.Value), + }) + } + return q +}