diff --git a/core/controllers/schedule.go b/core/controllers/schedule.go index e7c1e71d..8656cf13 100644 --- a/core/controllers/schedule.go +++ b/core/controllers/schedule.go @@ -2,8 +2,6 @@ package controllers import ( errors2 "errors" - mongo2 "github.com/crawlab-team/crawlab/core/mongo" - "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "github.com/crawlab-team/crawlab/core/interfaces" @@ -45,26 +43,11 @@ func GetScheduleById(_ *gin.Context, params *GetByIdParams) (response *Response[ func GetScheduleList(_ *gin.Context, params *GetListParams) (response *ListResponse[models.Schedule], err error) { query := ConvertToBsonMFromListParams(params) - sort, err := GetSortOptionFromString(params.Sort) if err != nil { return GetErrorListResponse[models.Schedule](errors.BadRequestf("invalid request parameters: %v", err)) } - - schedules, err := service.NewModelService[models.Schedule]().GetMany(query, &mongo2.FindOptions{ - Sort: sort, - Skip: params.Size * (params.Page - 1), - Limit: params.Size, - }) - if err != nil { - if !errors.Is(err, mongo.ErrNoDocuments) { - return GetErrorListResponse[models.Schedule](err) - } - return GetListResponse[models.Schedule]([]models.Schedule{}, 0) - } - if len(schedules) == 0 { - return GetListResponse[models.Schedule]([]models.Schedule{}, 0) - } + skip, limit := GetSkipLimitFromListParams(params) // total count total, err := service.NewModelService[models.Schedule]().Count(query) @@ -72,46 +55,23 @@ func GetScheduleList(_ *gin.Context, params *GetListParams) (response *ListRespo return GetErrorListResponse[models.Schedule](err) } - // ids - var ids []primitive.ObjectID - var spiderIds []primitive.ObjectID - for _, s := range schedules { - ids = append(ids, s.Id) - if !s.SpiderId.IsZero() { - spiderIds = append(spiderIds, s.SpiderId) - } + // check total + if total == 0 { + return GetEmptyListResponse[models.Schedule]() } - // spider dict cache - var spiders []models.Spider - if len(spiderIds) > 0 { - spiders, err = service.NewModelService[models.Spider]().GetMany(bson.M{"_id": bson.M{"$in": spiderIds}}, nil) - if err != nil { - return GetErrorListResponse[models.Schedule](err) - } - } - dictSpider := map[primitive.ObjectID]models.Spider{} - for _, p := range spiders { - dictSpider[p.Id] = p + // aggregation pipelines + pipelines := service.GetPaginationPipeline(query, sort, skip, limit) + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...) + + // perform query + var schedules []models.Schedule + err = service.GetCollection[models.Schedule]().Aggregate(pipelines, nil).All(&schedules) + if err != nil { + return GetErrorListResponse[models.Schedule](err) } - // iterate list again - var data []models.Schedule - for _, s := range schedules { - // spider - if !s.SpiderId.IsZero() { - p, ok := dictSpider[s.SpiderId] - if ok { - s.Spider = &p - } - } - - // add to list - data = append(data, s) - } - - // response - return GetListResponse(data, total) + return GetListResponse(schedules, total) } type PostScheduleParams struct { diff --git a/core/controllers/spider.go b/core/controllers/spider.go index 12cb6502..a6bafba8 100644 --- a/core/controllers/spider.go +++ b/core/controllers/spider.go @@ -1,7 +1,6 @@ package controllers import ( - "math" "mime/multipart" "os" "path/filepath" @@ -81,39 +80,14 @@ func GetSpiderById(_ *gin.Context, params *GetByIdParams) (response *Response[mo } // GetSpiderList handles getting a list of spiders with optional stats -func GetSpiderList(c *gin.Context, params *GetListParams) (response *ListResponse[models.Spider], err error) { - // get list - withStats := c.Query("stats") - if withStats == "" { - return getSpiderList(params) - } - - // get list with stats - return getSpiderListWithStats(params) -} - -func getSpiderList(params *GetListParams) (response *ListResponse[models.Spider], err error) { +func GetSpiderList(_ *gin.Context, params *GetListParams) (response *ListResponse[models.Spider], err error) { + // query parameters query := ConvertToBsonMFromListParams(params) - sort, err := GetSortOptionFromString(params.Sort) if err != nil { return GetErrorListResponse[models.Spider](errors.BadRequestf("invalid request parameters: %v", err)) } - - spiders, err := service.NewModelService[models.Spider]().GetMany(query, &mongo2.FindOptions{ - Sort: sort, - Skip: params.Size * (params.Page - 1), - Limit: params.Size, - }) - if err != nil { - if !errors.Is(err, mongo.ErrNoDocuments) { - return GetErrorListResponse[models.Spider](err) - } - return GetListResponse[models.Spider]([]models.Spider{}, 0) - } - if len(spiders) == 0 { - return GetListResponse[models.Spider]([]models.Spider{}, 0) - } + skip, limit := GetSkipLimitFromListParams(params) // total count total, err := service.NewModelService[models.Spider]().Count(query) @@ -121,161 +95,29 @@ func getSpiderList(params *GetListParams) (response *ListResponse[models.Spider] return GetErrorListResponse[models.Spider](err) } - // ids - var ids []primitive.ObjectID - var gitIds []primitive.ObjectID - var projectIds []primitive.ObjectID - for _, s := range spiders { - ids = append(ids, s.Id) - if !s.GitId.IsZero() { - gitIds = append(gitIds, s.GitId) - } - if !s.ProjectId.IsZero() { - projectIds = append(projectIds, s.ProjectId) - } + // check total + if total == 0 { + return GetEmptyListResponse[models.Spider]() } - // project dict cache - var projects []models.Project - if len(projectIds) > 0 { - projects, err = service.NewModelService[models.Project]().GetMany(bson.M{"_id": bson.M{"$in": projectIds}}, nil) - if err != nil { - return GetErrorListResponse[models.Spider](err) - } - } - dictProject := map[primitive.ObjectID]models.Project{} - for _, p := range projects { - dictProject[p.Id] = p + // aggregation pipelines + pipelines := service.GetPaginationPipeline(query, sort, skip, limit) + pipelines = append(pipelines, service.GetJoinPipeline[models.SpiderStat]("_id", "_id", "_stat")...) + pipelines = append(pipelines, service.GetJoinPipeline[models.Task]("_stat.last_task_id", "_id", "_last_task")...) + pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_last_task._id", "_id", "_last_task._stat")...) + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Project]()...) + if utils.IsPro() { + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Git]()...) } - // git dict cache - var gits []models.Git - if len(gitIds) > 0 && utils.IsPro() { - gits, err = service.NewModelService[models.Git]().GetMany(bson.M{"_id": bson.M{"$in": gitIds}}, nil) - if err != nil { - return GetErrorListResponse[models.Spider](err) - } - } - dictGit := map[primitive.ObjectID]models.Git{} - for _, g := range gits { - dictGit[g.Id] = g - } - - // iterate list again - var data []models.Spider - for _, s := range spiders { - // project - if !s.ProjectId.IsZero() { - p, ok := dictProject[s.ProjectId] - if ok { - s.Project = &p - } - } - - // git - if !s.GitId.IsZero() && utils.IsPro() { - g, ok := dictGit[s.GitId] - if ok { - s.Git = &g - } - } - - // add to list - data = append(data, s) - } - - // response - return GetListResponse(data, total) -} - -func getSpiderListWithStats(params *GetListParams) (response *ListResponse[models.Spider], err error) { - response, err = getSpiderList(params) + // perform query + var spiders []models.Spider + err = service.GetCollection[models.Spider]().Aggregate(pipelines, nil).All(&spiders) if err != nil { return GetErrorListResponse[models.Spider](err) } - // spider ids - var ids []primitive.ObjectID - for _, s := range response.Data { - ids = append(ids, s.Id) - } - - // spider stat dict - spiderStats, err := service.NewModelService[models.SpiderStat]().GetMany(bson.M{"_id": bson.M{"$in": ids}}, nil) - if err != nil { - return GetErrorListResponse[models.Spider](err) - } - dictSpiderStat := map[primitive.ObjectID]models.SpiderStat{} - - // task dict and task stat dict - var lastTasks []models.Task - var lastTaskIds []primitive.ObjectID - for _, st := range spiderStats { - if st.Tasks > 0 { - taskCount := int64(st.Tasks) - st.AverageWaitDuration = int64(math.Round(float64(st.WaitDuration) / float64(taskCount))) - st.AverageRuntimeDuration = int64(math.Round(float64(st.RuntimeDuration) / float64(taskCount))) - st.AverageTotalDuration = int64(math.Round(float64(st.TotalDuration) / float64(taskCount))) - } - dictSpiderStat[st.Id] = st - - if !st.LastTaskId.IsZero() { - lastTaskIds = append(lastTaskIds, st.LastTaskId) - } - } - dictLastTask := map[primitive.ObjectID]models.Task{} - dictLastTaskStat := map[primitive.ObjectID]models.TaskStat{} - if len(lastTaskIds) > 0 { - // task list - queryTask := bson.M{ - "_id": bson.M{ - "$in": lastTaskIds, - }, - } - lastTasks, err = service.NewModelService[models.Task]().GetMany(queryTask, nil) - if err != nil { - return GetErrorListResponse[models.Spider](err) - } - - // task stats list - taskStats, err := service.NewModelService[models.TaskStat]().GetMany(queryTask, nil) - if err != nil { - return GetErrorListResponse[models.Spider](err) - } - - for _, st := range taskStats { - dictLastTaskStat[st.Id] = st - } - - for _, t := range lastTasks { - st, ok := dictLastTaskStat[t.Id] - if ok { - t.Stat = &st - } - dictLastTask[t.SpiderId] = t - } - } - - // iterate list again - for i, s := range response.Data { - // spider stat - st, ok := dictSpiderStat[s.Id] - if ok { - s.Stat = &st - } - - // last task and stat - if !s.Stat.LastTaskId.IsZero() { - t, ok := dictLastTask[s.Stat.LastTaskId] - if ok { - s.Stat.LastTask = &t - } - } - - response.Data[i] = s - } - - return response, nil + return GetListResponse(spiders, total) } // PostSpider handles creating a new spider diff --git a/core/controllers/task.go b/core/controllers/task.go index d6646bba..6c85dae4 100644 --- a/core/controllers/task.go +++ b/core/controllers/task.go @@ -33,138 +33,62 @@ func GetTaskById(_ *gin.Context, params *GetTaskByIdParams) (response *Response[ return GetErrorResponse[models.Task](err) } - // task - t, err := service.NewModelService[models.Task]().GetById(id) - if errors.Is(err, mongo.ErrNoDocuments) { - return GetErrorResponse[models.Task](err) - } + // aggregation pipelines + pipelines := service.GetByIdPipeline(id) + pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_id", "_id", "_stat")...) + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Node]()...) + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...) + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Schedule]()...) + + // perform query + var tasks []models.Task + err = service.GetCollection[models.Task]().Aggregate(pipelines, nil).All(&tasks) if err != nil { return GetErrorResponse[models.Task](err) } - // skip if task status is pending - if t.Status == constants.TaskStatusPending { - return GetDataResponse(*t) - } - - // spider - if !t.SpiderId.IsZero() { - t.Spider, _ = service.NewModelService[models.Spider]().GetById(t.SpiderId) - } - - // schedule - if !t.ScheduleId.IsZero() { - t.Schedule, _ = service.NewModelService[models.Schedule]().GetById(t.ScheduleId) - } - - // node - if !t.NodeId.IsZero() { - t.Node, _ = service.NewModelService[models.Node]().GetById(t.NodeId) - } - - // task stat - t.Stat, _ = service.NewModelService[models.TaskStat]().GetById(id) - - return GetDataResponse(*t) -} - -type GetTaskListParams struct { - *GetListParams - Stats bool `query:"stats"` -} - -func GetTaskList(c *gin.Context, params *GetTaskListParams) (response *ListResponse[models.Task], err error) { - if params.Stats { - return NewController[models.Task]().GetList(c, params.GetListParams) - } - - // get query - query := ConvertToBsonMFromListParams(params.GetListParams) - - sort, err := GetSortOptionFromString(params.GetListParams.Sort) - if err != nil { - return GetErrorListResponse[models.Task](err) - } - - // get tasks - tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo2.FindOptions{ - Sort: sort, - Skip: params.Size * (params.Page - 1), - Limit: params.Size, - }) - if err != nil { - if errors.Is(err, mongo.ErrNoDocuments) { - return GetErrorListResponse[models.Task](err) - } - return GetErrorListResponse[models.Task](err) - } - - // check empty list + // check results if len(tasks) == 0 { - return GetListResponse[models.Task](nil, 0) + return nil, errors.NotFoundf("task %s not found", params.Id) } - // ids - var taskIds []primitive.ObjectID - var spiderIds []primitive.ObjectID - for _, t := range tasks { - taskIds = append(taskIds, t.Id) - spiderIds = append(spiderIds, t.SpiderId) - } + return GetDataResponse(tasks[0]) +} - // total count +func GetTaskList(_ *gin.Context, params *GetListParams) (response *ListResponse[models.Task], err error) { + // query parameters + query := ConvertToBsonMFromListParams(params) + sort, err := GetSortOptionFromString(params.Sort) + if err != nil { + return GetErrorListResponse[models.Task](err) + } + skip, limit := GetSkipLimitFromListParams(params) + + // total total, err := service.NewModelService[models.Task]().Count(query) if err != nil { return GetErrorListResponse[models.Task](err) } - // stat list - stats, err := service.NewModelService[models.TaskStat]().GetMany(bson.M{ - "_id": bson.M{ - "$in": taskIds, - }, - }, nil) + // check total + if total == 0 { + return GetEmptyListResponse[models.Task]() + } + + // aggregation pipelines + pipelines := service.GetPaginationPipeline(query, sort, skip, limit) + pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_id", "_id", "_stat")...) + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Node]()...) + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...) + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Schedule]()...) + + // perform query + var tasks []models.Task + err = service.GetCollection[models.Task]().Aggregate(pipelines, nil).All(&tasks) if err != nil { return GetErrorListResponse[models.Task](err) } - // cache stat list to dict - statsDict := map[primitive.ObjectID]models.TaskStat{} - for _, s := range stats { - statsDict[s.Id] = s - } - - // spider list - spiders, err := service.NewModelService[models.Spider]().GetMany(bson.M{ - "_id": bson.M{ - "$in": spiderIds, - }, - }, nil) - if err != nil { - return GetErrorListResponse[models.Task](err) - } - - // cache spider list to dict - spiderDict := map[primitive.ObjectID]models.Spider{} - for _, s := range spiders { - spiderDict[s.Id] = s - } - - // iterate list again - for i, t := range tasks { - // task stat - ts, ok := statsDict[t.Id] - if ok { - tasks[i].Stat = &ts - } - - // spider - s, ok := spiderDict[t.SpiderId] - if ok { - tasks[i].Spider = &s - } - } - return GetListResponse(tasks, total) } diff --git a/core/controllers/utils.go b/core/controllers/utils.go index ddfa0d39..a0e09c6e 100644 --- a/core/controllers/utils.go +++ b/core/controllers/utils.go @@ -281,6 +281,10 @@ func SortsToOption(sorts []entity.Sort) (sort bson.D, err error) { return sort, nil } +func GetSkipLimitFromListParams(params *GetListParams) (skip int, limit int) { + return params.Size * (params.Page - 1), params.Size +} + type BaseResponse interface { GetData() interface{} GetDataString() string diff --git a/core/models/models/spider.go b/core/models/models/spider.go index 3ebbf7de..93f50635 100644 --- a/core/models/models/spider.go +++ b/core/models/models/spider.go @@ -21,18 +21,15 @@ type Spider struct { Template string `json:"template,omitempty" bson:"template,omitempty" description:"Spider template"` TemplateParams *SpiderTemplateParams `json:"template_params,omitempty" bson:"template_params,omitempty" description:"Spider template params"` - // stats - Stat *SpiderStat `json:"stat,omitempty" bson:"-"` - // execution Cmd string `json:"cmd" bson:"cmd" description:"Execute command"` Param string `json:"param" bson:"param" description:"Default task param"` Priority int `json:"priority" bson:"priority" description:"Priority" default:"5" minimum:"1" maximum:"10"` - // associated data - Project *Project `json:"project,omitempty" bson:"-"` - Git *Git `json:"git,omitempty" bson:"-"` - DataSource *Database `json:"data_source,omitempty" bson:"-"` + Stat *SpiderStat `json:"stat,omitempty" bson:"_stat,omitempty"` + LastTask *Task `json:"last_task,omitempty" bson:"_last_task,omitempty"` + Project *Project `json:"project,omitempty" bson:"_project,omitempty"` + Git *Git `json:"git,omitempty" bson:"_git,omitempty"` } type SpiderTemplateParams struct { diff --git a/core/models/models/spider_stat.go b/core/models/models/spider_stat.go index 9fd71bbe..4a214e6b 100644 --- a/core/models/models/spider_stat.go +++ b/core/models/models/spider_stat.go @@ -8,7 +8,6 @@ type SpiderStat struct { any `collection:"spider_stats"` BaseModel `bson:",inline"` LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty" description:"Last task ID"` - LastTask *Task `json:"last_task,omitempty" bson:"-"` Tasks int `json:"tasks" bson:"tasks" description:"Task count"` Results int `json:"results" bson:"results" description:"Result count"` WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty" description:"Wait duration (in second)"` diff --git a/core/models/models/task.go b/core/models/models/task.go index 17e2e422..8dae2a26 100644 --- a/core/models/models/task.go +++ b/core/models/models/task.go @@ -20,8 +20,8 @@ type Task struct { NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-"` // associated data - Stat *TaskStat `json:"stat,omitempty" bson:"-"` - Spider *Spider `json:"spider,omitempty" bson:"-"` - Schedule *Schedule `json:"schedule,omitempty" bson:"-"` - Node *Node `json:"node,omitempty" bson:"-"` + Stat *TaskStat `json:"stat,omitempty" bson:"_stat,omitempty"` + Node *Node `json:"node,omitempty" bson:"_node,omitempty"` + Spider *Spider `json:"spider,omitempty" bson:"_spider,omitempty"` + Schedule *Schedule `json:"schedule,omitempty" bson:"_schedule,omitempty"` } diff --git a/core/models/service/base_service.go b/core/models/service/base_service.go index dee6e5e2..aa9c7427 100644 --- a/core/models/service/base_service.go +++ b/core/models/service/base_service.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/crawlab-team/crawlab/core/mongo" + "github.com/crawlab-team/crawlab/core/utils" "reflect" "sync" @@ -303,13 +304,17 @@ func GetCollectionNameByInstance(v any) string { return field.Tag.Get("collection") } -func getCollectionName[T any]() string { +func GetCollectionName[T any]() string { var instance T t := reflect.TypeOf(instance) field := t.Field(0) return field.Tag.Get("collection") } +func GetCollection[T any]() *mongo.Col { + return mongo.GetMongoCol(GetCollectionName[T]()) +} + // NewModelService return singleton instance of ModelService func NewModelService[T any]() *ModelService[T] { typeName := fmt.Sprintf("%T", *new(T)) @@ -324,7 +329,7 @@ func NewModelService[T any]() *ModelService[T] { var instance *ModelService[T] onceMap[typeName].Do(func() { - collectionName := getCollectionName[T]() + collectionName := GetCollectionName[T]() collection := mongo.GetMongoCol(collectionName) instance = &ModelService[T]{col: collection} instanceMap[typeName] = instance @@ -351,3 +356,80 @@ func NewModelServiceWithColName[T any](colName string) *ModelService[T] { return instanceMap[colName].(*ModelService[T]) } + +func GetDefaultJoinPipeline[T any]() []bson.D { + return []bson.D{ + GetDefaultLookupPipeline[T](), + GetDefaultUnwindPipeline[T](), + } +} + +func GetJoinPipeline[T any](localField, foreignField, as string) []bson.D { + return []bson.D{ + GetLookupPipeline[T](localField, foreignField, as), + GetUnwindPipeline(as), + } +} + +func GetDefaultLookupPipeline[T any]() bson.D { + var model T + typ := reflect.TypeOf(model) + name := utils.ToSnakeCase(typ.Name()) + return GetLookupByNamePipeline[T](name) +} + +func GetLookupByNamePipeline[T any](name string) bson.D { + localField := fmt.Sprintf("%s_id", name) + foreignField := "_id" + as := fmt.Sprintf("_%s", name) + return GetLookupPipeline[T](localField, foreignField, as) +} + +func GetLookupPipeline[T any](localField, foreignField, as string) bson.D { + return bson.D{{ + Key: "$lookup", + Value: bson.M{ + "from": GetCollectionName[T](), + "localField": localField, + "foreignField": foreignField, + "as": as, + }}, + } +} + +func GetDefaultUnwindPipeline[T any]() bson.D { + var model T + typ := reflect.TypeOf(model) + name := utils.ToSnakeCase(typ.Name()) + as := fmt.Sprintf("_%s", name) + return GetUnwindPipeline(as) +} + +func GetUnwindPipeline(as string) bson.D { + return bson.D{{ + Key: "$unwind", + Value: bson.M{ + "path": fmt.Sprintf("$%s", as), + "preserveNullAndEmptyArrays": true, + }}, + } +} + +func GetPaginationPipeline(query bson.M, sort bson.D, skip, limit int) []bson.D { + if query == nil { + query = bson.M{} + } + return []bson.D{ + {{Key: "$match", Value: query}}, + {{Key: "$sort", Value: sort}}, + {{Key: "$skip", Value: skip}}, + {{Key: "$limit", Value: limit}}, + } +} + +func GetByIdPipeline(id primitive.ObjectID) []bson.D { + return []bson.D{ + {{Key: "$match", Value: bson.M{"_id": id}}}, + {{Key: "$limit", Value: 1}}, + } +} diff --git a/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue b/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue index ba36663b..40089492 100644 --- a/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue +++ b/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue @@ -1,5 +1,5 @@ + + diff --git a/frontend/crawlab-ui/src/components/ui/select/Select.vue b/frontend/crawlab-ui/src/components/ui/select/Select.vue deleted file mode 100644 index 13fea16a..00000000 --- a/frontend/crawlab-ui/src/components/ui/select/Select.vue +++ /dev/null @@ -1,30 +0,0 @@ - - - diff --git a/frontend/crawlab-ui/src/interfaces/models/spider.d.ts b/frontend/crawlab-ui/src/interfaces/models/spider.d.ts index fcb99739..f369c510 100644 --- a/frontend/crawlab-ui/src/interfaces/models/spider.d.ts +++ b/frontend/crawlab-ui/src/interfaces/models/spider.d.ts @@ -18,20 +18,22 @@ export declare global { description?: string; update_ts?: string; create_ts?: string; - last_task?: Task; - stat?: SpiderStat; incremental_sync?: boolean; auto_install?: boolean; git_id?: string; git_root_path?: string; - git?: Git; template?: SpiderTemplateName; template_params?: SpiderTemplateParams; + + // associated data + stat?: SpiderStat; + last_task?: Task; + project?: Project; + git?: Git; } interface SpiderStat { _id: number; - last_task?: Task; tasks: number; results: number; wait_duration: number; diff --git a/frontend/crawlab-ui/src/views/spider/list/useSpiderList.tsx b/frontend/crawlab-ui/src/views/spider/list/useSpiderList.tsx index f46da81a..84bb1ef6 100644 --- a/frontend/crawlab-ui/src/views/spider/list/useSpiderList.tsx +++ b/frontend/crawlab-ui/src/views/spider/list/useSpiderList.tsx @@ -52,12 +52,6 @@ const useSpiderList = () => { const { allListSelectOptions: allProjectListSelectOptions } = useProject(store); - // const allProjectList = computed(() => store.state.project.allList); - - // all project dict - const allProjectDict = computed>( - () => store.getters['project/allDict'] - ); // nav actions const navActions = computed(() => [ @@ -138,10 +132,12 @@ const useSpiderList = () => { icon: ['fa', 'project-diagram'], width: '120', value: (row: Spider) => { - if (!row.project_id) return; - const p = allProjectDict.value.get(row.project_id); + if (!row.project?._id) return; return ( - + ); }, hasFilter: true, @@ -176,7 +172,7 @@ const useSpiderList = () => { icon: ['fa', 'heartbeat'], width: '120', value: (row: Spider) => { - const { _id, status, error } = row.stat?.last_task || {}; + const { _id, status, error } = row.last_task || {}; if (!status) return; return ( { icon: ['fa', 'clock'], width: '160', value: (row: Spider) => { - const time = row.stat?.last_task?.stat?.started_at; + const time = row.last_task?.stat?.started_at; if (!time) return; return ; }, @@ -338,7 +334,7 @@ const useSpiderList = () => { } as UseListOptions; // init - setupListComponent(ns, store, ['node', 'project']); + setupListComponent(ns, store); return { ...useList(ns, store, opts), diff --git a/frontend/crawlab-ui/src/views/task/detail/actions/groups/TaskDetailActionGroupNav.vue b/frontend/crawlab-ui/src/views/task/detail/actions/groups/TaskDetailActionGroupNav.vue index e51b9e85..0223210a 100644 --- a/frontend/crawlab-ui/src/views/task/detail/actions/groups/TaskDetailActionGroupNav.vue +++ b/frontend/crawlab-ui/src/views/task/detail/actions/groups/TaskDetailActionGroupNav.vue @@ -1,7 +1,7 @@ @@ -22,7 +20,7 @@ defineOptions({ name: 'ClTaskDetailActionGroupNav' }); { } }; - // all node dict - const allNodeDict = computed>( - () => store.getters['node/allDict'] - ); - - // all spider dict - const allSpiderDict = computed>( - () => store.getters['spider/allDict'] - ); - - // all schedule dict - const allScheduleDict = computed>( - () => store.getters['schedule/allDict'] - ); - // nav actions const navActions = computed(() => [ { @@ -183,15 +168,14 @@ const useTaskList = () => { icon: ['fa', 'server'], width: '160', value: (row: Task) => { - if (!row.node_id) return; - const node = allNodeDict.value.get(row.node_id); + const { node } = row; if (!node) return; return ( { - await router.push(`/nodes/${node?._id}`); + await router.push(`/nodes/${node._id}`); }} /> ); @@ -203,13 +187,10 @@ const useTaskList = () => { icon: ['fa', 'spider'], width: '160', value: (row: Task) => { - if (!row.spider_id) return; - const spider = row.spider || allSpiderDict.value.get(row.spider_id); + const { spider } = row; + if (!spider) return; return ( - + ); }, }, @@ -219,12 +200,12 @@ const useTaskList = () => { icon: ['fa', 'clock'], width: '160', value: (row: Task) => { - if (!row.schedule_id) return; - const schedule = allScheduleDict.value.get(row.schedule_id); + const { schedule } = row; + if (!schedule) return; return ( ); }, @@ -244,13 +225,9 @@ const useTaskList = () => { icon: ['fa', 'terminal'], width: '160', value: (row: Task) => { - return ( - - ); + const { spider } = row; + if (!spider) return; + return ; }, }, { @@ -290,7 +267,10 @@ const useTaskList = () => { icon: ['fa', 'clock'], width: '120', value: (row: Task) => { - if (!row.stat?.started_at || row.stat?.started_at.startsWith('000')) { + if ( + !row.stat?.started_at || + row.stat?.started_at.startsWith('000') + ) { return; } return ; @@ -477,7 +457,7 @@ const useTaskList = () => { } as UseListOptions; // init - setupListComponent(ns, store, ['node', 'project', 'spider', 'schedule']); + setupListComponent(ns, store); return { ...useList(ns, store, opts),