diff --git a/core/controllers/router.go b/core/controllers/router.go index dcfa8dad..b514bdfd 100644 --- a/core/controllers/router.go +++ b/core/controllers/router.go @@ -377,7 +377,7 @@ func InitRoutes(app *gin.Engine) (err error) { Method: http.MethodGet, Path: "/:id/results", Name: "Get Spider Results", - Description: "Get the results of a spider", + Description: "Get the scraped or crawled results data of a spider", HandlerFunc: GetSpiderResults, }, }...)) @@ -475,6 +475,13 @@ func InitRoutes(app *gin.Engine) (err error) { Description: "Get the logs of a task", HandlerFunc: GetTaskLogs, }, + { + Method: http.MethodGet, + Path: "/:id/results", + Name: "Get Task Results", + Description: "Get the scraped or crawled results data of a task", + HandlerFunc: GetTaskResults, + }, }...)) RegisterController(groups.AuthGroup.Group("", "Users", "APIs for users management"), "/users", NewController[models.User]([]Action{ { diff --git a/core/controllers/spider.go b/core/controllers/spider.go index 09ed677e..64c4438e 100644 --- a/core/controllers/spider.go +++ b/core/controllers/spider.go @@ -746,12 +746,16 @@ func GetSpiderResults(c *gin.Context, params *GetSpiderResultsParams) (response return GetErrorListResponse[bson.M](err) } - query := GetResultListQuery(c) + query := ConvertToBsonMFromContext(c) + if query == nil { + query = bson.M{} + } + query["_sid"] = s.Id col := mongo2.GetMongoCol(s.ColName) var results []bson.M - err = col.Find(mongo2.GetMongoQuery(query), mongo2.GetMongoOpts(&mongo2.ListOptions{ + err = col.Find(query, mongo2.GetMongoOpts(&mongo2.ListOptions{ Sort: []mongo2.ListSort{{"_id", mongo2.SortDirectionDesc}}, Skip: params.Size * (params.Page - 1), Limit: params.Size, @@ -760,7 +764,7 @@ func GetSpiderResults(c *gin.Context, params *GetSpiderResultsParams) (response return GetErrorListResponse[bson.M](err) } - total, err := mongo2.GetMongoCol(s.ColName).Count(mongo2.GetMongoQuery(query)) + total, err := mongo2.GetMongoCol(s.ColName).Count(query) if err != nil { return GetErrorListResponse[bson.M](err) } diff --git a/core/controllers/task.go b/core/controllers/task.go index e3618b92..9112bd62 100644 --- a/core/controllers/task.go +++ b/core/controllers/task.go @@ -1,7 +1,6 @@ package controllers import ( - "errors" "os" "path/filepath" "strings" @@ -11,15 +10,16 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" - mongo3 "github.com/crawlab-team/crawlab/core/mongo" + mongo2 "github.com/crawlab-team/crawlab/core/mongo" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/task/log" "github.com/crawlab-team/crawlab/core/task/scheduler" "github.com/crawlab-team/crawlab/core/utils" "github.com/gin-gonic/gin" + "github.com/juju/errors" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo" ) type GetTaskByIdParams struct { @@ -35,7 +35,7 @@ func GetTaskById(_ *gin.Context, params *GetTaskByIdParams) (response *Response[ // task t, err := service.NewModelService[models.Task]().GetById(id) - if errors.Is(err, mongo2.ErrNoDocuments) { + if errors.Is(err, mongo.ErrNoDocuments) { return GetErrorResponse[models.Task](err) } if err != nil { @@ -87,13 +87,13 @@ func GetTaskList(c *gin.Context, params *GetTaskListParams) (response *ListRespo } // get tasks - tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo3.FindOptions{ + tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo2.FindOptions{ Sort: sort, Skip: params.Size * (params.Page - 1), Limit: params.Size, }) if err != nil { - if errors.Is(err, mongo2.ErrNoDocuments) { + if errors.Is(err, mongo.ErrNoDocuments) { return GetErrorListResponse[models.Task](err) } return GetErrorListResponse[models.Task](err) @@ -179,7 +179,7 @@ func DeleteTaskById(_ *gin.Context, params *DeleteTaskByIdParams) (response *Voi } // delete in db - if err := mongo3.RunTransaction(func(context mongo2.SessionContext) (err error) { + if err := mongo2.RunTransaction(func(context mongo.SessionContext) (err error) { // delete task _, err = service.NewModelService[models.Task]().GetById(id) if err != nil { @@ -230,7 +230,7 @@ func DeleteTaskList(_ *gin.Context, params *DeleteTaskListParams) (response *Voi ids = append(ids, id) } - if err := mongo3.RunTransaction(func(context mongo2.SessionContext) error { + if err := mongo2.RunTransaction(func(context mongo.SessionContext) error { // delete tasks if err := service.NewModelService[models.Task]().DeleteMany(bson.M{ "_id": bson.M{ @@ -440,3 +440,51 @@ func GetTaskLogs(_ *gin.Context, params *GetTaskLogsParams) (response *ListRespo return GetListResponse(logs, total) } + +type GetTaskResultsParams struct { + Id string `path:"id" description:"Task ID" format:"objectid" pattern:"^[0-9a-fA-F]{24}$"` + Page int `query:"page" description:"Page" default:"1" minimum:"1"` + Size int `query:"size" description:"Size" default:"10" minimum:"1"` +} + +func GetTaskResults(c *gin.Context, params *GetSpiderResultsParams) (response *ListResponse[bson.M], err error) { + id, err := primitive.ObjectIDFromHex(params.Id) + if err != nil { + return GetErrorListResponse[bson.M](errors.BadRequestf("invalid id format")) + } + + t, err := service.NewModelService[models.Task]().GetById(id) + if err != nil { + return GetErrorListResponse[bson.M](err) + } + + s, err := service.NewModelService[models.Spider]().GetById(t.SpiderId) + if err != nil { + return GetErrorListResponse[bson.M](err) + } + + query := ConvertToBsonMFromContext(c) + if query == nil { + query = bson.M{} + } + query["_tid"] = t.Id + + col := mongo2.GetMongoCol(s.ColName) + + var results []bson.M + err = col.Find(query, mongo2.GetMongoOpts(&mongo2.ListOptions{ + Sort: []mongo2.ListSort{{"_id", mongo2.SortDirectionDesc}}, + Skip: params.Size * (params.Page - 1), + Limit: params.Size, + })).All(&results) + if err != nil { + return GetErrorListResponse[bson.M](err) + } + + total, err := mongo2.GetMongoCol(s.ColName).Count(query) + if err != nil { + return GetErrorListResponse[bson.M](err) + } + + return GetListResponse(results, total) +} diff --git a/core/controllers/utils.go b/core/controllers/utils.go index 2925197d..0b77feaf 100644 --- a/core/controllers/utils.go +++ b/core/controllers/utils.go @@ -177,6 +177,9 @@ func ConvertToBsonMFromContext(c *gin.Context) (q bson.M) { func GetResultListQuery(c *gin.Context) (q mongo.ListQuery) { f := GetFilterFromContext(c) + if f == nil { + return nil + } for _, cond := range f.Conditions { q = append(q, mongo.ListQueryCondition{ Key: cond.Key,