diff --git a/core/controllers/schedule.go b/core/controllers/schedule.go
index e7c1e71d..8656cf13 100644
--- a/core/controllers/schedule.go
+++ b/core/controllers/schedule.go
@@ -2,8 +2,6 @@ package controllers
import (
errors2 "errors"
- mongo2 "github.com/crawlab-team/crawlab/core/mongo"
- "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"github.com/crawlab-team/crawlab/core/interfaces"
@@ -45,26 +43,11 @@ func GetScheduleById(_ *gin.Context, params *GetByIdParams) (response *Response[
func GetScheduleList(_ *gin.Context, params *GetListParams) (response *ListResponse[models.Schedule], err error) {
query := ConvertToBsonMFromListParams(params)
-
sort, err := GetSortOptionFromString(params.Sort)
if err != nil {
return GetErrorListResponse[models.Schedule](errors.BadRequestf("invalid request parameters: %v", err))
}
-
- schedules, err := service.NewModelService[models.Schedule]().GetMany(query, &mongo2.FindOptions{
- Sort: sort,
- Skip: params.Size * (params.Page - 1),
- Limit: params.Size,
- })
- if err != nil {
- if !errors.Is(err, mongo.ErrNoDocuments) {
- return GetErrorListResponse[models.Schedule](err)
- }
- return GetListResponse[models.Schedule]([]models.Schedule{}, 0)
- }
- if len(schedules) == 0 {
- return GetListResponse[models.Schedule]([]models.Schedule{}, 0)
- }
+ skip, limit := GetSkipLimitFromListParams(params)
// total count
total, err := service.NewModelService[models.Schedule]().Count(query)
@@ -72,46 +55,23 @@ func GetScheduleList(_ *gin.Context, params *GetListParams) (response *ListRespo
return GetErrorListResponse[models.Schedule](err)
}
- // ids
- var ids []primitive.ObjectID
- var spiderIds []primitive.ObjectID
- for _, s := range schedules {
- ids = append(ids, s.Id)
- if !s.SpiderId.IsZero() {
- spiderIds = append(spiderIds, s.SpiderId)
- }
+ // check total
+ if total == 0 {
+ return GetEmptyListResponse[models.Schedule]()
}
- // spider dict cache
- var spiders []models.Spider
- if len(spiderIds) > 0 {
- spiders, err = service.NewModelService[models.Spider]().GetMany(bson.M{"_id": bson.M{"$in": spiderIds}}, nil)
- if err != nil {
- return GetErrorListResponse[models.Schedule](err)
- }
- }
- dictSpider := map[primitive.ObjectID]models.Spider{}
- for _, p := range spiders {
- dictSpider[p.Id] = p
+ // aggregation pipelines
+ pipelines := service.GetPaginationPipeline(query, sort, skip, limit)
+ pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...)
+
+ // perform query
+ var schedules []models.Schedule
+ err = service.GetCollection[models.Schedule]().Aggregate(pipelines, nil).All(&schedules)
+ if err != nil {
+ return GetErrorListResponse[models.Schedule](err)
}
- // iterate list again
- var data []models.Schedule
- for _, s := range schedules {
- // spider
- if !s.SpiderId.IsZero() {
- p, ok := dictSpider[s.SpiderId]
- if ok {
- s.Spider = &p
- }
- }
-
- // add to list
- data = append(data, s)
- }
-
- // response
- return GetListResponse(data, total)
+ return GetListResponse(schedules, total)
}
type PostScheduleParams struct {
diff --git a/core/controllers/spider.go b/core/controllers/spider.go
index 12cb6502..a6bafba8 100644
--- a/core/controllers/spider.go
+++ b/core/controllers/spider.go
@@ -1,7 +1,6 @@
package controllers
import (
- "math"
"mime/multipart"
"os"
"path/filepath"
@@ -81,39 +80,14 @@ func GetSpiderById(_ *gin.Context, params *GetByIdParams) (response *Response[mo
}
// GetSpiderList handles getting a list of spiders with optional stats
-func GetSpiderList(c *gin.Context, params *GetListParams) (response *ListResponse[models.Spider], err error) {
- // get list
- withStats := c.Query("stats")
- if withStats == "" {
- return getSpiderList(params)
- }
-
- // get list with stats
- return getSpiderListWithStats(params)
-}
-
-func getSpiderList(params *GetListParams) (response *ListResponse[models.Spider], err error) {
+func GetSpiderList(_ *gin.Context, params *GetListParams) (response *ListResponse[models.Spider], err error) {
+ // query parameters
query := ConvertToBsonMFromListParams(params)
-
sort, err := GetSortOptionFromString(params.Sort)
if err != nil {
return GetErrorListResponse[models.Spider](errors.BadRequestf("invalid request parameters: %v", err))
}
-
- spiders, err := service.NewModelService[models.Spider]().GetMany(query, &mongo2.FindOptions{
- Sort: sort,
- Skip: params.Size * (params.Page - 1),
- Limit: params.Size,
- })
- if err != nil {
- if !errors.Is(err, mongo.ErrNoDocuments) {
- return GetErrorListResponse[models.Spider](err)
- }
- return GetListResponse[models.Spider]([]models.Spider{}, 0)
- }
- if len(spiders) == 0 {
- return GetListResponse[models.Spider]([]models.Spider{}, 0)
- }
+ skip, limit := GetSkipLimitFromListParams(params)
// total count
total, err := service.NewModelService[models.Spider]().Count(query)
@@ -121,161 +95,29 @@ func getSpiderList(params *GetListParams) (response *ListResponse[models.Spider]
return GetErrorListResponse[models.Spider](err)
}
- // ids
- var ids []primitive.ObjectID
- var gitIds []primitive.ObjectID
- var projectIds []primitive.ObjectID
- for _, s := range spiders {
- ids = append(ids, s.Id)
- if !s.GitId.IsZero() {
- gitIds = append(gitIds, s.GitId)
- }
- if !s.ProjectId.IsZero() {
- projectIds = append(projectIds, s.ProjectId)
- }
+ // check total
+ if total == 0 {
+ return GetEmptyListResponse[models.Spider]()
}
- // project dict cache
- var projects []models.Project
- if len(projectIds) > 0 {
- projects, err = service.NewModelService[models.Project]().GetMany(bson.M{"_id": bson.M{"$in": projectIds}}, nil)
- if err != nil {
- return GetErrorListResponse[models.Spider](err)
- }
- }
- dictProject := map[primitive.ObjectID]models.Project{}
- for _, p := range projects {
- dictProject[p.Id] = p
+ // aggregation pipelines
+ pipelines := service.GetPaginationPipeline(query, sort, skip, limit)
+ pipelines = append(pipelines, service.GetJoinPipeline[models.SpiderStat]("_id", "_id", "_stat")...)
+ pipelines = append(pipelines, service.GetJoinPipeline[models.Task]("_stat.last_task_id", "_id", "_last_task")...)
+ pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_last_task._id", "_id", "_last_task._stat")...)
+ pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Project]()...)
+ if utils.IsPro() {
+ pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Git]()...)
}
- // git dict cache
- var gits []models.Git
- if len(gitIds) > 0 && utils.IsPro() {
- gits, err = service.NewModelService[models.Git]().GetMany(bson.M{"_id": bson.M{"$in": gitIds}}, nil)
- if err != nil {
- return GetErrorListResponse[models.Spider](err)
- }
- }
- dictGit := map[primitive.ObjectID]models.Git{}
- for _, g := range gits {
- dictGit[g.Id] = g
- }
-
- // iterate list again
- var data []models.Spider
- for _, s := range spiders {
- // project
- if !s.ProjectId.IsZero() {
- p, ok := dictProject[s.ProjectId]
- if ok {
- s.Project = &p
- }
- }
-
- // git
- if !s.GitId.IsZero() && utils.IsPro() {
- g, ok := dictGit[s.GitId]
- if ok {
- s.Git = &g
- }
- }
-
- // add to list
- data = append(data, s)
- }
-
- // response
- return GetListResponse(data, total)
-}
-
-func getSpiderListWithStats(params *GetListParams) (response *ListResponse[models.Spider], err error) {
- response, err = getSpiderList(params)
+ // perform query
+ var spiders []models.Spider
+ err = service.GetCollection[models.Spider]().Aggregate(pipelines, nil).All(&spiders)
if err != nil {
return GetErrorListResponse[models.Spider](err)
}
- // spider ids
- var ids []primitive.ObjectID
- for _, s := range response.Data {
- ids = append(ids, s.Id)
- }
-
- // spider stat dict
- spiderStats, err := service.NewModelService[models.SpiderStat]().GetMany(bson.M{"_id": bson.M{"$in": ids}}, nil)
- if err != nil {
- return GetErrorListResponse[models.Spider](err)
- }
- dictSpiderStat := map[primitive.ObjectID]models.SpiderStat{}
-
- // task dict and task stat dict
- var lastTasks []models.Task
- var lastTaskIds []primitive.ObjectID
- for _, st := range spiderStats {
- if st.Tasks > 0 {
- taskCount := int64(st.Tasks)
- st.AverageWaitDuration = int64(math.Round(float64(st.WaitDuration) / float64(taskCount)))
- st.AverageRuntimeDuration = int64(math.Round(float64(st.RuntimeDuration) / float64(taskCount)))
- st.AverageTotalDuration = int64(math.Round(float64(st.TotalDuration) / float64(taskCount)))
- }
- dictSpiderStat[st.Id] = st
-
- if !st.LastTaskId.IsZero() {
- lastTaskIds = append(lastTaskIds, st.LastTaskId)
- }
- }
- dictLastTask := map[primitive.ObjectID]models.Task{}
- dictLastTaskStat := map[primitive.ObjectID]models.TaskStat{}
- if len(lastTaskIds) > 0 {
- // task list
- queryTask := bson.M{
- "_id": bson.M{
- "$in": lastTaskIds,
- },
- }
- lastTasks, err = service.NewModelService[models.Task]().GetMany(queryTask, nil)
- if err != nil {
- return GetErrorListResponse[models.Spider](err)
- }
-
- // task stats list
- taskStats, err := service.NewModelService[models.TaskStat]().GetMany(queryTask, nil)
- if err != nil {
- return GetErrorListResponse[models.Spider](err)
- }
-
- for _, st := range taskStats {
- dictLastTaskStat[st.Id] = st
- }
-
- for _, t := range lastTasks {
- st, ok := dictLastTaskStat[t.Id]
- if ok {
- t.Stat = &st
- }
- dictLastTask[t.SpiderId] = t
- }
- }
-
- // iterate list again
- for i, s := range response.Data {
- // spider stat
- st, ok := dictSpiderStat[s.Id]
- if ok {
- s.Stat = &st
- }
-
- // last task and stat
- if !s.Stat.LastTaskId.IsZero() {
- t, ok := dictLastTask[s.Stat.LastTaskId]
- if ok {
- s.Stat.LastTask = &t
- }
- }
-
- response.Data[i] = s
- }
-
- return response, nil
+ return GetListResponse(spiders, total)
}
// PostSpider handles creating a new spider
diff --git a/core/controllers/task.go b/core/controllers/task.go
index d6646bba..6c85dae4 100644
--- a/core/controllers/task.go
+++ b/core/controllers/task.go
@@ -33,138 +33,62 @@ func GetTaskById(_ *gin.Context, params *GetTaskByIdParams) (response *Response[
return GetErrorResponse[models.Task](err)
}
- // task
- t, err := service.NewModelService[models.Task]().GetById(id)
- if errors.Is(err, mongo.ErrNoDocuments) {
- return GetErrorResponse[models.Task](err)
- }
+ // aggregation pipelines
+ pipelines := service.GetByIdPipeline(id)
+ pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_id", "_id", "_stat")...)
+ pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Node]()...)
+ pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...)
+ pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Schedule]()...)
+
+ // perform query
+ var tasks []models.Task
+ err = service.GetCollection[models.Task]().Aggregate(pipelines, nil).All(&tasks)
if err != nil {
return GetErrorResponse[models.Task](err)
}
- // skip if task status is pending
- if t.Status == constants.TaskStatusPending {
- return GetDataResponse(*t)
- }
-
- // spider
- if !t.SpiderId.IsZero() {
- t.Spider, _ = service.NewModelService[models.Spider]().GetById(t.SpiderId)
- }
-
- // schedule
- if !t.ScheduleId.IsZero() {
- t.Schedule, _ = service.NewModelService[models.Schedule]().GetById(t.ScheduleId)
- }
-
- // node
- if !t.NodeId.IsZero() {
- t.Node, _ = service.NewModelService[models.Node]().GetById(t.NodeId)
- }
-
- // task stat
- t.Stat, _ = service.NewModelService[models.TaskStat]().GetById(id)
-
- return GetDataResponse(*t)
-}
-
-type GetTaskListParams struct {
- *GetListParams
- Stats bool `query:"stats"`
-}
-
-func GetTaskList(c *gin.Context, params *GetTaskListParams) (response *ListResponse[models.Task], err error) {
- if params.Stats {
- return NewController[models.Task]().GetList(c, params.GetListParams)
- }
-
- // get query
- query := ConvertToBsonMFromListParams(params.GetListParams)
-
- sort, err := GetSortOptionFromString(params.GetListParams.Sort)
- if err != nil {
- return GetErrorListResponse[models.Task](err)
- }
-
- // get tasks
- tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo2.FindOptions{
- Sort: sort,
- Skip: params.Size * (params.Page - 1),
- Limit: params.Size,
- })
- if err != nil {
- if errors.Is(err, mongo.ErrNoDocuments) {
- return GetErrorListResponse[models.Task](err)
- }
- return GetErrorListResponse[models.Task](err)
- }
-
- // check empty list
+ // check results
if len(tasks) == 0 {
- return GetListResponse[models.Task](nil, 0)
+ return nil, errors.NotFoundf("task %s not found", params.Id)
}
- // ids
- var taskIds []primitive.ObjectID
- var spiderIds []primitive.ObjectID
- for _, t := range tasks {
- taskIds = append(taskIds, t.Id)
- spiderIds = append(spiderIds, t.SpiderId)
- }
+ return GetDataResponse(tasks[0])
+}
- // total count
+func GetTaskList(_ *gin.Context, params *GetListParams) (response *ListResponse[models.Task], err error) {
+ // query parameters
+ query := ConvertToBsonMFromListParams(params)
+ sort, err := GetSortOptionFromString(params.Sort)
+ if err != nil {
+ return GetErrorListResponse[models.Task](err)
+ }
+ skip, limit := GetSkipLimitFromListParams(params)
+
+ // total
total, err := service.NewModelService[models.Task]().Count(query)
if err != nil {
return GetErrorListResponse[models.Task](err)
}
- // stat list
- stats, err := service.NewModelService[models.TaskStat]().GetMany(bson.M{
- "_id": bson.M{
- "$in": taskIds,
- },
- }, nil)
+ // check total
+ if total == 0 {
+ return GetEmptyListResponse[models.Task]()
+ }
+
+ // aggregation pipelines
+ pipelines := service.GetPaginationPipeline(query, sort, skip, limit)
+ pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_id", "_id", "_stat")...)
+ pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Node]()...)
+ pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...)
+ pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Schedule]()...)
+
+ // perform query
+ var tasks []models.Task
+ err = service.GetCollection[models.Task]().Aggregate(pipelines, nil).All(&tasks)
if err != nil {
return GetErrorListResponse[models.Task](err)
}
- // cache stat list to dict
- statsDict := map[primitive.ObjectID]models.TaskStat{}
- for _, s := range stats {
- statsDict[s.Id] = s
- }
-
- // spider list
- spiders, err := service.NewModelService[models.Spider]().GetMany(bson.M{
- "_id": bson.M{
- "$in": spiderIds,
- },
- }, nil)
- if err != nil {
- return GetErrorListResponse[models.Task](err)
- }
-
- // cache spider list to dict
- spiderDict := map[primitive.ObjectID]models.Spider{}
- for _, s := range spiders {
- spiderDict[s.Id] = s
- }
-
- // iterate list again
- for i, t := range tasks {
- // task stat
- ts, ok := statsDict[t.Id]
- if ok {
- tasks[i].Stat = &ts
- }
-
- // spider
- s, ok := spiderDict[t.SpiderId]
- if ok {
- tasks[i].Spider = &s
- }
- }
-
return GetListResponse(tasks, total)
}
diff --git a/core/controllers/utils.go b/core/controllers/utils.go
index ddfa0d39..a0e09c6e 100644
--- a/core/controllers/utils.go
+++ b/core/controllers/utils.go
@@ -281,6 +281,10 @@ func SortsToOption(sorts []entity.Sort) (sort bson.D, err error) {
return sort, nil
}
+func GetSkipLimitFromListParams(params *GetListParams) (skip int, limit int) {
+ return params.Size * (params.Page - 1), params.Size
+}
+
type BaseResponse interface {
GetData() interface{}
GetDataString() string
diff --git a/core/models/models/spider.go b/core/models/models/spider.go
index 3ebbf7de..93f50635 100644
--- a/core/models/models/spider.go
+++ b/core/models/models/spider.go
@@ -21,18 +21,15 @@ type Spider struct {
Template string `json:"template,omitempty" bson:"template,omitempty" description:"Spider template"`
TemplateParams *SpiderTemplateParams `json:"template_params,omitempty" bson:"template_params,omitempty" description:"Spider template params"`
- // stats
- Stat *SpiderStat `json:"stat,omitempty" bson:"-"`
-
// execution
Cmd string `json:"cmd" bson:"cmd" description:"Execute command"`
Param string `json:"param" bson:"param" description:"Default task param"`
Priority int `json:"priority" bson:"priority" description:"Priority" default:"5" minimum:"1" maximum:"10"`
- // associated data
- Project *Project `json:"project,omitempty" bson:"-"`
- Git *Git `json:"git,omitempty" bson:"-"`
- DataSource *Database `json:"data_source,omitempty" bson:"-"`
+ Stat *SpiderStat `json:"stat,omitempty" bson:"_stat,omitempty"`
+ LastTask *Task `json:"last_task,omitempty" bson:"_last_task,omitempty"`
+ Project *Project `json:"project,omitempty" bson:"_project,omitempty"`
+ Git *Git `json:"git,omitempty" bson:"_git,omitempty"`
}
type SpiderTemplateParams struct {
diff --git a/core/models/models/spider_stat.go b/core/models/models/spider_stat.go
index 9fd71bbe..4a214e6b 100644
--- a/core/models/models/spider_stat.go
+++ b/core/models/models/spider_stat.go
@@ -8,7 +8,6 @@ type SpiderStat struct {
any `collection:"spider_stats"`
BaseModel `bson:",inline"`
LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty" description:"Last task ID"`
- LastTask *Task `json:"last_task,omitempty" bson:"-"`
Tasks int `json:"tasks" bson:"tasks" description:"Task count"`
Results int `json:"results" bson:"results" description:"Result count"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty" description:"Wait duration (in second)"`
diff --git a/core/models/models/task.go b/core/models/models/task.go
index 17e2e422..8dae2a26 100644
--- a/core/models/models/task.go
+++ b/core/models/models/task.go
@@ -20,8 +20,8 @@ type Task struct {
NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-"`
// associated data
- Stat *TaskStat `json:"stat,omitempty" bson:"-"`
- Spider *Spider `json:"spider,omitempty" bson:"-"`
- Schedule *Schedule `json:"schedule,omitempty" bson:"-"`
- Node *Node `json:"node,omitempty" bson:"-"`
+ Stat *TaskStat `json:"stat,omitempty" bson:"_stat,omitempty"`
+ Node *Node `json:"node,omitempty" bson:"_node,omitempty"`
+ Spider *Spider `json:"spider,omitempty" bson:"_spider,omitempty"`
+ Schedule *Schedule `json:"schedule,omitempty" bson:"_schedule,omitempty"`
}
diff --git a/core/models/service/base_service.go b/core/models/service/base_service.go
index dee6e5e2..aa9c7427 100644
--- a/core/models/service/base_service.go
+++ b/core/models/service/base_service.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/crawlab-team/crawlab/core/mongo"
+ "github.com/crawlab-team/crawlab/core/utils"
"reflect"
"sync"
@@ -303,13 +304,17 @@ func GetCollectionNameByInstance(v any) string {
return field.Tag.Get("collection")
}
-func getCollectionName[T any]() string {
+func GetCollectionName[T any]() string {
var instance T
t := reflect.TypeOf(instance)
field := t.Field(0)
return field.Tag.Get("collection")
}
+func GetCollection[T any]() *mongo.Col {
+ return mongo.GetMongoCol(GetCollectionName[T]())
+}
+
// NewModelService return singleton instance of ModelService
func NewModelService[T any]() *ModelService[T] {
typeName := fmt.Sprintf("%T", *new(T))
@@ -324,7 +329,7 @@ func NewModelService[T any]() *ModelService[T] {
var instance *ModelService[T]
onceMap[typeName].Do(func() {
- collectionName := getCollectionName[T]()
+ collectionName := GetCollectionName[T]()
collection := mongo.GetMongoCol(collectionName)
instance = &ModelService[T]{col: collection}
instanceMap[typeName] = instance
@@ -351,3 +356,80 @@ func NewModelServiceWithColName[T any](colName string) *ModelService[T] {
return instanceMap[colName].(*ModelService[T])
}
+
+func GetDefaultJoinPipeline[T any]() []bson.D {
+ return []bson.D{
+ GetDefaultLookupPipeline[T](),
+ GetDefaultUnwindPipeline[T](),
+ }
+}
+
+func GetJoinPipeline[T any](localField, foreignField, as string) []bson.D {
+ return []bson.D{
+ GetLookupPipeline[T](localField, foreignField, as),
+ GetUnwindPipeline(as),
+ }
+}
+
+func GetDefaultLookupPipeline[T any]() bson.D {
+ var model T
+ typ := reflect.TypeOf(model)
+ name := utils.ToSnakeCase(typ.Name())
+ return GetLookupByNamePipeline[T](name)
+}
+
+func GetLookupByNamePipeline[T any](name string) bson.D {
+ localField := fmt.Sprintf("%s_id", name)
+ foreignField := "_id"
+ as := fmt.Sprintf("_%s", name)
+ return GetLookupPipeline[T](localField, foreignField, as)
+}
+
+func GetLookupPipeline[T any](localField, foreignField, as string) bson.D {
+ return bson.D{{
+ Key: "$lookup",
+ Value: bson.M{
+ "from": GetCollectionName[T](),
+ "localField": localField,
+ "foreignField": foreignField,
+ "as": as,
+ }},
+ }
+}
+
+func GetDefaultUnwindPipeline[T any]() bson.D {
+ var model T
+ typ := reflect.TypeOf(model)
+ name := utils.ToSnakeCase(typ.Name())
+ as := fmt.Sprintf("_%s", name)
+ return GetUnwindPipeline(as)
+}
+
+func GetUnwindPipeline(as string) bson.D {
+ return bson.D{{
+ Key: "$unwind",
+ Value: bson.M{
+ "path": fmt.Sprintf("$%s", as),
+ "preserveNullAndEmptyArrays": true,
+ }},
+ }
+}
+
+func GetPaginationPipeline(query bson.M, sort bson.D, skip, limit int) []bson.D {
+ if query == nil {
+ query = bson.M{}
+ }
+ return []bson.D{
+ {{Key: "$match", Value: query}},
+ {{Key: "$sort", Value: sort}},
+ {{Key: "$skip", Value: skip}},
+ {{Key: "$limit", Value: limit}},
+ }
+}
+
+func GetByIdPipeline(id primitive.ObjectID) []bson.D {
+ return []bson.D{
+ {{Key: "$match", Value: bson.M{"_id": id}}},
+ {{Key: "$limit", Value: 1}},
+ }
+}
diff --git a/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue b/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue
index ba36663b..40089492 100644
--- a/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue
+++ b/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue
@@ -1,5 +1,5 @@
+
+
+ emit('change', value)"
+ @clear="() => emit('clear')"
+ >
+
+
+
+
+
+
diff --git a/frontend/crawlab-ui/src/components/ui/select/Select.vue b/frontend/crawlab-ui/src/components/ui/select/Select.vue
deleted file mode 100644
index 13fea16a..00000000
--- a/frontend/crawlab-ui/src/components/ui/select/Select.vue
+++ /dev/null
@@ -1,30 +0,0 @@
-
-
-
- emit('change', value)"
- @clear="() => emit('clear')"
- >
-
-
-
-
-
-
diff --git a/frontend/crawlab-ui/src/interfaces/models/spider.d.ts b/frontend/crawlab-ui/src/interfaces/models/spider.d.ts
index fcb99739..f369c510 100644
--- a/frontend/crawlab-ui/src/interfaces/models/spider.d.ts
+++ b/frontend/crawlab-ui/src/interfaces/models/spider.d.ts
@@ -18,20 +18,22 @@ export declare global {
description?: string;
update_ts?: string;
create_ts?: string;
- last_task?: Task;
- stat?: SpiderStat;
incremental_sync?: boolean;
auto_install?: boolean;
git_id?: string;
git_root_path?: string;
- git?: Git;
template?: SpiderTemplateName;
template_params?: SpiderTemplateParams;
+
+ // associated data
+ stat?: SpiderStat;
+ last_task?: Task;
+ project?: Project;
+ git?: Git;
}
interface SpiderStat {
_id: number;
- last_task?: Task;
tasks: number;
results: number;
wait_duration: number;
diff --git a/frontend/crawlab-ui/src/views/spider/list/useSpiderList.tsx b/frontend/crawlab-ui/src/views/spider/list/useSpiderList.tsx
index f46da81a..84bb1ef6 100644
--- a/frontend/crawlab-ui/src/views/spider/list/useSpiderList.tsx
+++ b/frontend/crawlab-ui/src/views/spider/list/useSpiderList.tsx
@@ -52,12 +52,6 @@ const useSpiderList = () => {
const { allListSelectOptions: allProjectListSelectOptions } =
useProject(store);
- // const allProjectList = computed(() => store.state.project.allList);
-
- // all project dict
- const allProjectDict = computed