From 79b7e074e1a21477836851e065f9b64ebc9e11db Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 12 Jun 2025 00:17:40 +0800 Subject: [PATCH] refactor: update filter parameter in API requests and improve component structure --- core/controllers/schedule.go | 35 +++--- core/controllers/spider.go | 75 ++++-------- core/controllers/task.go | 32 ++++-- core/models/models/schedule.go | 2 +- core/models/models/spider.go | 3 +- core/task/stats/service.go | 4 +- .../components/core/schedule/ScheduleForm.vue | 45 ++++---- .../components/core/schedule/useSchedule.ts | 7 +- .../src/components/core/spider/SpiderForm.vue | 1 - .../core/task/TaskResultDataWithDatabase.vue | 16 +-- .../src/components/core/task/useTask.ts | 14 +-- .../src/components/ui/filter/FilterInput.vue | 2 +- .../src/components/ui/form/useForm.ts | 21 ---- .../src/components/ui/select/RemoteSelect.vue | 27 ++++- .../ui/table/TableHeaderDialogFilter.vue | 5 +- .../src/interfaces/models/spider.d.ts | 3 +- .../src/interfaces/services/request.d.ts | 4 +- .../src/interfaces/views/schedule.d.ts | 3 + .../layouts/content/detail/DetailLayout.vue | 27 ++++- .../src/layouts/content/detail/useDetail.ts | 17 +-- frontend/crawlab-ui/src/services/index.ts | 5 +- frontend/crawlab-ui/src/services/request.ts | 5 - .../src/store/modules/dependency.ts | 2 +- frontend/crawlab-ui/src/store/modules/node.ts | 3 +- frontend/crawlab-ui/src/utils/store.ts | 4 +- .../src/views/git/detail/GitDetail.vue | 3 +- .../git/detail/tabs/GitDetailTabChanges.vue | 4 +- .../git/detail/tabs/GitDetailTabCommits.vue | 3 +- .../git/detail/tabs/GitDetailTabFiles.vue | 3 +- .../NotificationSettingDetailTabChannels.vue | 2 - .../views/schedule/list/useScheduleList.tsx | 108 +++++++++--------- .../actions/SpiderDetailActionsData.vue | 32 +++--- .../src/views/spider/list/useSpiderList.tsx | 7 -- .../detail/actions/TaskDetailActionsData.vue | 18 +-- 34 files changed, 258 insertions(+), 284 deletions(-) diff --git a/core/controllers/schedule.go b/core/controllers/schedule.go index 8656cf13..59a30286 100644 --- a/core/controllers/schedule.go +++ b/core/controllers/schedule.go @@ -2,8 +2,6 @@ package controllers import ( errors2 "errors" - "go.mongodb.org/mongo-driver/mongo" - "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" @@ -11,6 +9,7 @@ import ( "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/gin-gonic/gin" "github.com/juju/errors" + "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" ) @@ -20,25 +19,24 @@ func GetScheduleById(_ *gin.Context, params *GetByIdParams) (response *Response[ if err != nil { return GetErrorResponse[models.Schedule](errors.BadRequestf("invalid id format")) } - s, err := service.NewModelService[models.Schedule]().GetById(id) - if errors.Is(err, mongo.ErrNoDocuments) { - return GetErrorResponse[models.Schedule](errors.NotFoundf("spider not found")) - } + + // aggregation pipelines + pipelines := service.GetByIdPipeline(id) + pipelines = addSchedulePipelines(pipelines) + + // perform query + var schedules []models.Schedule + err = service.GetCollection[models.Schedule]().Aggregate(pipelines, nil).All(&schedules) if err != nil { return GetErrorResponse[models.Schedule](err) } - // spider - if !s.SpiderId.IsZero() { - s.Spider, err = service.NewModelService[models.Spider]().GetById(s.SpiderId) - if err != nil { - if !errors.Is(err, mongo.ErrNoDocuments) { - return GetErrorResponse[models.Schedule](err) - } - } + // check results + if len(schedules) == 0 { + return nil, errors.NotFoundf("schedule %s not found", params.Id) } - return GetDataResponse(*s) + return GetDataResponse(schedules[0]) } func GetScheduleList(_ *gin.Context, params *GetListParams) (response *ListResponse[models.Schedule], err error) { @@ -62,7 +60,7 @@ func GetScheduleList(_ *gin.Context, params *GetListParams) (response *ListRespo // aggregation pipelines pipelines := service.GetPaginationPipeline(query, sort, skip, limit) - pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...) + pipelines = addSchedulePipelines(pipelines) // perform query var schedules []models.Schedule @@ -240,3 +238,8 @@ func postScheduleRunFunc(params *PostScheduleRunParams, userId primitive.ObjectI return GetDataResponse(taskIds) } + +func addSchedulePipelines(pipelines []bson.D) []bson.D { + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...) + return pipelines +} diff --git a/core/controllers/spider.go b/core/controllers/spider.go index a6bafba8..7b08e6e0 100644 --- a/core/controllers/spider.go +++ b/core/controllers/spider.go @@ -28,55 +28,24 @@ func GetSpiderById(_ *gin.Context, params *GetByIdParams) (response *Response[mo if err != nil { return GetErrorResponse[models.Spider](errors.BadRequestf("invalid id format")) } - s, err := service.NewModelService[models.Spider]().GetById(id) - if errors.Is(err, mongo.ErrNoDocuments) { - return GetErrorResponse[models.Spider](errors.NotFoundf("spider not found")) - } + + // aggregation pipelines + pipelines := service.GetByIdPipeline(id) + pipelines = addSpiderPipelines(pipelines) + + // perform query + var spiders []models.Spider + err = service.GetCollection[models.Spider]().Aggregate(pipelines, nil).All(&spiders) if err != nil { return GetErrorResponse[models.Spider](err) } - // stat - s.Stat, err = service.NewModelService[models.SpiderStat]().GetById(s.Id) - if err != nil { - if !errors.Is(err, mongo.ErrNoDocuments) { - return GetErrorResponse[models.Spider](err) - } + // check results + if len(spiders) == 0 { + return nil, errors.NotFoundf("spider %s not found", params.Id) } - // project - if !s.ProjectId.IsZero() { - s.Project, err = service.NewModelService[models.Project]().GetById(s.ProjectId) - if err != nil { - if !errors.Is(err, mongo.ErrNoDocuments) { - return GetErrorResponse[models.Spider](err) - } - } - } - - // data collection (compatible to old version) - if s.ColName == "" && !s.ColId.IsZero() { - col, err := service.NewModelService[models.DataCollection]().GetById(s.ColId) - if err != nil { - if !errors.Is(err, mongo.ErrNoDocuments) { - return GetErrorResponse[models.Spider](err) - } - } else { - s.ColName = col.Name - } - } - - // git - if utils.IsPro() && !s.GitId.IsZero() { - s.Git, err = service.NewModelService[models.Git]().GetById(s.GitId) - if err != nil { - if !errors.Is(err, mongo.ErrNoDocuments) { - return GetErrorResponse[models.Spider](err) - } - } - } - - return GetDataResponse(*s) + return GetDataResponse(spiders[0]) } // GetSpiderList handles getting a list of spiders with optional stats @@ -102,13 +71,7 @@ func GetSpiderList(_ *gin.Context, params *GetListParams) (response *ListRespons // aggregation pipelines pipelines := service.GetPaginationPipeline(query, sort, skip, limit) - pipelines = append(pipelines, service.GetJoinPipeline[models.SpiderStat]("_id", "_id", "_stat")...) - pipelines = append(pipelines, service.GetJoinPipeline[models.Task]("_stat.last_task_id", "_id", "_last_task")...) - pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_last_task._id", "_id", "_last_task._stat")...) - pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Project]()...) - if utils.IsPro() { - pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Git]()...) - } + pipelines = addSpiderPipelines(pipelines) // perform query var spiders []models.Spider @@ -676,3 +639,15 @@ func getSpiderRootPathByContext(c *gin.Context) (rootPath string, err error) { } return utils.GetSpiderRootPath(s) } + +func addSpiderPipelines(pipelines []bson.D) []bson.D { + pipelines = append(pipelines, service.GetJoinPipeline[models.SpiderStat]("_id", "_id", "_stat")...) + pipelines = append(pipelines, service.GetJoinPipeline[models.Task]("_stat.last_task_id", "_id", "_last_task")...) + pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_last_task._id", "_id", "_last_task._stat")...) + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Project]()...) + if utils.IsPro() { + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Git]()...) + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Database]()...) + } + return pipelines +} diff --git a/core/controllers/task.go b/core/controllers/task.go index 6c85dae4..07f61fef 100644 --- a/core/controllers/task.go +++ b/core/controllers/task.go @@ -35,10 +35,7 @@ func GetTaskById(_ *gin.Context, params *GetTaskByIdParams) (response *Response[ // aggregation pipelines pipelines := service.GetByIdPipeline(id) - pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_id", "_id", "_stat")...) - pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Node]()...) - pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...) - pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Schedule]()...) + pipelines = addTaskPipelines(pipelines) // perform query var tasks []models.Task @@ -64,6 +61,20 @@ func GetTaskList(_ *gin.Context, params *GetListParams) (response *ListResponse[ } skip, limit := GetSkipLimitFromListParams(params) + // get spider ids if query is not nil + if query != nil { + spiders, err := service.NewModelService[models.Spider]().GetMany(query, &mongo2.FindOptions{Limit: 100}) + if err != nil { + query = nil // reset query if error occurs + } else { + spiderIds := make([]primitive.ObjectID, 0, len(spiders)) + for _, spider := range spiders { + spiderIds = append(spiderIds, spider.Id) + } + query = bson.M{"spider_id": bson.M{"$in": spiderIds}} // rewrite query to filter by spider ids + } + } + // total total, err := service.NewModelService[models.Task]().Count(query) if err != nil { @@ -77,10 +88,7 @@ func GetTaskList(_ *gin.Context, params *GetListParams) (response *ListResponse[ // aggregation pipelines pipelines := service.GetPaginationPipeline(query, sort, skip, limit) - pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_id", "_id", "_stat")...) - pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Node]()...) - pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...) - pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Schedule]()...) + pipelines = addTaskPipelines(pipelines) // perform query var tasks []models.Task @@ -425,3 +433,11 @@ func GetTaskResults(c *gin.Context, params *GetSpiderResultsParams) (response *L return GetListResponse(results, total) } + +func addTaskPipelines(pipelines []bson.D) []bson.D { + pipelines = append(pipelines, service.GetJoinPipeline[models.TaskStat]("_id", "_id", "_stat")...) + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Node]()...) + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Spider]()...) + pipelines = append(pipelines, service.GetDefaultJoinPipeline[models.Schedule]()...) + return pipelines +} diff --git a/core/models/models/schedule.go b/core/models/models/schedule.go index db0fdbc9..b3619e2a 100644 --- a/core/models/models/schedule.go +++ b/core/models/models/schedule.go @@ -21,5 +21,5 @@ type Schedule struct { Enabled bool `json:"enabled" bson:"enabled" description:"Enabled"` // associated data - Spider *Spider `json:"spider" bson:"-" description:"Spider"` + Spider *Spider `json:"spider" bson:"_spider" description:"Spider"` } diff --git a/core/models/models/spider.go b/core/models/models/spider.go index 93f50635..5ce6d978 100644 --- a/core/models/models/spider.go +++ b/core/models/models/spider.go @@ -11,8 +11,8 @@ type Spider struct { ColId primitive.ObjectID `json:"col_id" bson:"col_id" description:"Data collection id" deprecated:"true"` ColName string `json:"col_name,omitempty" bson:"col_name" description:"Data collection name"` DbName string `json:"db_name,omitempty" bson:"db_name" description:"Database name"` - DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id" description:"Data source id"` Description string `json:"description" bson:"description" description:"Description"` + DatabaseId primitive.ObjectID `json:"database_id" bson:"database_id" description:"Database Id"` ProjectId primitive.ObjectID `json:"project_id" bson:"project_id" description:"Project ID"` Mode string `json:"mode" bson:"mode" description:"Default task mode" enum:"random,all,selected-nodes"` NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids" description:"Default node ids, used in selected-nodes mode"` @@ -30,6 +30,7 @@ type Spider struct { LastTask *Task `json:"last_task,omitempty" bson:"_last_task,omitempty"` Project *Project `json:"project,omitempty" bson:"_project,omitempty"` Git *Git `json:"git,omitempty" bson:"_git,omitempty"` + Database *Database `json:"database,omitempty" bson:"_database,omitempty"` } type SpiderTemplateParams struct { diff --git a/core/task/stats/service.go b/core/task/stats/service.go index d06e1c5f..81c40eb7 100644 --- a/core/task/stats/service.go +++ b/core/task/stats/service.go @@ -112,7 +112,7 @@ func (svc *Service) getDatabaseServiceItem(taskId primitive.ObjectID) (item *dat var dbSvc interfaces2.DatabaseService if utils.IsPro() { if dbRegSvc := database.GetDatabaseRegistryService(); dbRegSvc != nil { - dbSvc, err = dbRegSvc.GetDatabaseService(s.DataSourceId) + dbSvc, err = dbRegSvc.GetDatabaseService(s.DatabaseId) if err != nil { return nil, err } @@ -123,7 +123,7 @@ func (svc *Service) getDatabaseServiceItem(taskId primitive.ObjectID) (item *dat item = &databaseServiceItem{ taskId: taskId, spiderId: s.Id, - dbId: s.DataSourceId, + dbId: s.DatabaseId, dbSvc: dbSvc, tableName: s.ColName, time: time.Now(), diff --git a/frontend/crawlab-ui/src/components/core/schedule/ScheduleForm.vue b/frontend/crawlab-ui/src/components/core/schedule/ScheduleForm.vue index ca23946f..f6854401 100644 --- a/frontend/crawlab-ui/src/components/core/schedule/ScheduleForm.vue +++ b/frontend/crawlab-ui/src/components/core/schedule/ScheduleForm.vue @@ -2,11 +2,9 @@ import { useStore } from 'vuex'; import { ElMessage } from 'element-plus'; import { TASK_MODE_SELECTED_NODES } from '@/constants/task'; -import useSchedule from '@/components/core/schedule/useSchedule'; -import useSpider from '@/components/core/spider/useSpider'; -import useNode from '@/components/core/node/useNode'; -import useTask from '@/components/core/task/useTask'; +import { useSchedule, useNode, ClRemoteSelect } from '@/components'; import { priorityOptions, translate } from '@/utils'; +import { ref } from 'vue'; const t = translate; @@ -24,10 +22,7 @@ const { } = useSchedule(store); // use node -const { activeNodesSorted: activeNodes } = useNode(store); - -// use spider -const { allListSelectOptions: allSpiderSelectOptions } = useSpider(store); +const { allNodesSorted: allNodes } = useNode(store); // on enabled change const onEnabledChange = async (value: boolean) => { @@ -40,6 +35,22 @@ const onEnabledChange = async (value: boolean) => { } await store.dispatch(`${ns}/getList`); }; + +const spiderRef = ref(); +const onSpiderChange = (spiderId: string) => { + if (!spiderId) return; + const payload = { ...form.value } as Schedule; + if (!spiderRef.value) return; + const spider: Spider = spiderRef.value.getSelectedItem(); + if (!spider) return; + if (spider.cmd) payload.cmd = spider.cmd; + if (spider.param) payload.param = spider.param; + if (spider.mode) payload.mode = spider.mode; + if (spider.node_ids?.length) payload.node_ids = spider.node_ids; + if (spider.node_tags?.length) payload.node_tags = spider.node_tags; + store.commit(`${ns}/setForm`, payload); +}; + defineOptions({ name: 'ClScheduleForm' }); @@ -71,18 +82,12 @@ defineOptions({ name: 'ClScheduleForm' }); prop="spider_id" required > - - - + endpoint="/spiders" + @change="onSpiderChange" + /> @@ -196,7 +201,7 @@ defineOptions({ name: 'ClScheduleForm' }); :placeholder="t('components.schedule.form.selectedNodes')" > ) => { const ns = 'schedule'; const state = store.state[ns]; - const { allDict: allSpiderDict } = useSpider(store); - // form const form = computed(() => state.form); @@ -56,9 +53,9 @@ const useSchedule = (store: Store) => { () => { if (activeId.value) return; if (!form.value?.spider_id) return; - const spider = allSpiderDict.value.get(form.value?.spider_id); - if (!spider) return; + if (!form.value?.spider) return; const payload = { ...form.value } as Schedule; + const spider = form.value.spider; if (spider.cmd) payload.cmd = spider.cmd; if (spider.param) payload.param = spider.param; if (spider.mode) payload.mode = spider.mode; diff --git a/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue b/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue index 1cf0b490..369c7ca8 100644 --- a/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue +++ b/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue @@ -180,7 +180,6 @@ defineOptions({ name: 'ClSpiderForm' }); (); const useTask = (store: Store) => { - const ns = 'task' as ListStoreNamespace; - const { task: state } = store.state as RootStoreState; - // options for default mode const modeOptions = getModeOptions(); const modeOptionsDict = computed(() => getModeOptionsDict()); - const { allDict: allSpiderDict } = useSpider(store); - // route const route = useRoute(); @@ -33,7 +26,6 @@ const useTask = (store: Store) => { return { ...useForm('task', store, useTaskService(store), formComponentData), - allSpiderDict, id, modeOptions, modeOptionsDict, diff --git a/frontend/crawlab-ui/src/components/ui/filter/FilterInput.vue b/frontend/crawlab-ui/src/components/ui/filter/FilterInput.vue index 0a0cb01d..4cb4cfa3 100644 --- a/frontend/crawlab-ui/src/components/ui/filter/FilterInput.vue +++ b/frontend/crawlab-ui/src/components/ui/filter/FilterInput.vue @@ -1,6 +1,6 @@