git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix: results data display issue

Browse Source
This commit is contained in:
Marvin Zhang
2024-10-17 20:54:25 +08:00
parent dd9ffdc212
commit 6ff5e7aa76
4 changed files with 32 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
core/controllers/result_v2.go
View File

@@ -1,119 +0,0 @@
package controllers
import (
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/result"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
mongo2 "go.mongodb.org/mongo-driver/mongo"
)
func GetResultList(c *gin.Context) {
// data collection id
dcId, err := primitive.ObjectIDFromHex(c.Param("id"))
if err != nil {
HandleErrorBadRequest(c, err)
return
}
// data source id
var dsId primitive.ObjectID
dsIdStr := c.Query("data_source_id")
if dsIdStr != "" {
dsId, err = primitive.ObjectIDFromHex(dsIdStr)
if err != nil {
HandleErrorBadRequest(c, err)
return
}
}
// data collection
dc, err := service.NewModelServiceV2[models2.DataCollectionV2]().GetById(dcId)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// data source
ds, err := service.NewModelServiceV2[models2.DatabaseV2]().GetById(dsId)
if err != nil {
if err.Error() == mongo2.ErrNoDocuments.Error() {
ds = &models2.DatabaseV2{}
} else {
HandleErrorInternalServerError(c, err)
return
}
}
// spider
sq := bson.M{
"col_id": dc.Id,
"data_source_id": ds.Id,
}
s, err := service.NewModelServiceV2[models2.SpiderV2]().GetOne(sq, nil)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// service
svc, err := result.GetResultService(s.Id)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// params
pagination := MustGetPagination(c)
query := getResultListQuery(c)
// get results
data, err := svc.List(query, &generic.ListOptions{
Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}},
Skip: pagination.Size * (pagination.Page - 1),
Limit: pagination.Size,
})
if err != nil {
if err.Error() == mongo2.ErrNoDocuments.Error() {
HandleSuccessWithListData(c, nil, 0)
return
}
HandleErrorInternalServerError(c, err)
return
}
// validate results
if len(data) == 0 {
HandleSuccessWithListData(c, nil, 0)
return
}
// total count
total, err := svc.Count(query)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// response
HandleSuccessWithListData(c, data, total)
}
func getResultListQuery(c *gin.Context) (q generic.ListQuery) {
f, err := GetFilter(c)
if err != nil {
return q
}
for _, cond := range f.Conditions {
q = append(q, generic.ListQueryCondition{
Key: cond.Key,
Op: cond.Op,
Value: utils.NormalizeObjectId(cond.Value),
})
}
return q
}

17
core/controllers/router_v2.go
View File

@@ -173,16 +173,10 @@ func InitRoutes(app *gin.Engine) (err error) {
Path: "/:id/run",
HandlerFunc: PostSpiderRun,
},
{
Method: http.MethodGet,
Path: "/:id/data-source",
HandlerFunc: GetSpiderDataSource,
},
{
Method: http.MethodPost,
Path: "/:id/data-source/:ds_id",
HandlerFunc: PostSpiderDataSource,
Path: "/:id/results",
HandlerFunc: GetSpiderResults,
},
}...))
RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models2.TaskV2]([]Action{
@@ -262,13 +256,6 @@ func InitRoutes(app *gin.Engine) (err error) {
},
}...))
RegisterActions(groups.AuthGroup, "/results", []Action{
{
Method: http.MethodGet,
Path: "/:id",
HandlerFunc: GetResultList,
},
})
RegisterActions(groups.AuthGroup, "/export", []Action{
{
Method: http.MethodPost,

61
core/controllers/spider_v2.go
View File

@@ -9,6 +9,7 @@ import (
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/spider/admin"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/trace"
"github.com/gin-gonic/gin"
@@ -670,77 +671,43 @@ func PostSpiderRun(c *gin.Context) {
HandleSuccessWithData(c, taskIds)
}
func GetSpiderDataSource(c *gin.Context) {
// spider id
func GetSpiderResults(c *gin.Context) {
id, err := primitive.ObjectIDFromHex(c.Param("id"))
if err != nil {
HandleErrorBadRequest(c, err)
return
}
// spider
s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// data source
ds, err := service.NewModelServiceV2[models2.DatabaseV2]().GetById(s.DataSourceId)
if err != nil {
if err.Error() == mongo2.ErrNoDocuments.Error() {
HandleSuccess(c)
return
}
HandleErrorInternalServerError(c, err)
return
}
// params
pagination := MustGetPagination(c)
query := getResultListQuery(c)
HandleSuccessWithData(c, ds)
}
col := mongo.GetMongoCol(s.ColName)
func PostSpiderDataSource(c *gin.Context) {
// spider id
id, err := primitive.ObjectIDFromHex(c.Param("id"))
if err != nil {
HandleErrorBadRequest(c, err)
return
}
// data source id
dsId, err := primitive.ObjectIDFromHex(c.Param("ds_id"))
if err != nil {
HandleErrorBadRequest(c, err)
return
}
// spider
s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id)
var results []bson.M
err = col.Find(utils.GetMongoQuery(query), utils.GetMongoOpts(&generic.ListOptions{
Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}},
Skip: pagination.Size * (pagination.Page - 1),
Limit: pagination.Size,
})).All(&results)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// data source
if !dsId.IsZero() {
_, err = service.NewModelServiceV2[models2.DatabaseV2]().GetById(dsId)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
}
// save data source id
u := GetUserFromContextV2(c)
s.DataSourceId = dsId
s.SetUpdatedBy(u.Id)
_, err = service.NewModelServiceV2[models2.SpiderV2]().InsertOne(*s)
total, err := mongo.GetMongoCol(s.ColName).Count(utils.GetMongoQuery(query))
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
HandleSuccess(c)
HandleSuccessWithListData(c, results, total)
}
func getSpiderFsSvc(s *models2.SpiderV2) (svc interfaces.FsServiceV2, err error) {

16
core/controllers/utils_filter.go
View File

@@ -6,6 +6,7 @@ import (
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
@@ -121,3 +122,18 @@ func MustGetFilterAll(c *gin.Context) (res bool) {
}
return res
}
func getResultListQuery(c *gin.Context) (q generic.ListQuery) {
f, err := GetFilter(c)
if err != nil {
return q
}
for _, cond := range f.Conditions {
q = append(q, generic.ListQueryCondition{
Key: cond.Key,
Op: cond.Op,
Value: utils.NormalizeObjectId(cond.Value),
})
}
return q
}