diff --git a/core/controllers/stats.go b/core/controllers/stats.go index 19f9499d..8a699bb6 100644 --- a/core/controllers/stats.go +++ b/core/controllers/stats.go @@ -1,13 +1,14 @@ package controllers import ( + "errors" + "github.com/crawlab-team/crawlab/core/models/service" "go.mongodb.org/mongo-driver/bson/primitive" "time" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/mongo" mongo2 "go.mongodb.org/mongo-driver/mongo" "github.com/gin-gonic/gin" @@ -44,7 +45,7 @@ func GetStatsOverview(_ *gin.Context, params *GetStatsOverviewParams) (response var data GetStatsOverviewResponse // nodes - data.Nodes, err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Node{})).Count(bson.M{"active": true}) + data.Nodes, err = service.GetCollection[models.Node]().Count(bson.M{"active": true}) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -53,7 +54,7 @@ func GetStatsOverview(_ *gin.Context, params *GetStatsOverviewParams) (response } // projects - data.Projects, err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Project{})).Count(nil) + data.Projects, err = service.GetCollection[models.Project]().Count(nil) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -62,7 +63,7 @@ func GetStatsOverview(_ *gin.Context, params *GetStatsOverviewParams) (response } // spiders - data.Spiders, err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Spider{})).Count(nil) + data.Spiders, err = service.GetCollection[models.Spider]().Count(nil) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -71,7 +72,7 @@ func GetStatsOverview(_ *gin.Context, params *GetStatsOverviewParams) (response } // schedules - data.Schedules, err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Schedule{})).Count(nil) + data.Schedules, err = service.GetCollection[models.Schedule]().Count(nil) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -80,7 +81,7 @@ func GetStatsOverview(_ *gin.Context, params *GetStatsOverviewParams) (response } // tasks - data.Tasks, err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Task{})).Count(nil) + data.Tasks, err = service.GetCollection[models.Task]().Count(nil) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -89,7 +90,7 @@ func GetStatsOverview(_ *gin.Context, params *GetStatsOverviewParams) (response } // error tasks - data.ErrorTasks, err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Task{})).Count(bson.M{"status": constants.TaskStatusError}) + data.ErrorTasks, err = service.GetCollection[models.Task]().Count(bson.M{"status": constants.TaskStatusError}) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -111,15 +112,18 @@ func GetStatsOverview(_ *gin.Context, params *GetStatsOverviewParams) (response var res struct { Results int `bson:"results"` } - if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskStat{})).Aggregate(pipeline, nil).One(&res); err != nil { - return nil, err + if err := service.GetCollection[models.TaskStat]().Aggregate(pipeline, nil).One(&res); err != nil { + if !errors.Is(err, mongo2.ErrNoDocuments) { + return nil, err + } + res.Results = 0 } data.Results = res.Results // users - data.Users, err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.User{})).Count(nil) + data.Users, err = service.GetCollection[models.User]().Count(nil) if err != nil { - if err.Error() != mongo2.ErrNoDocuments.Error() { + if !errors.Is(err, mongo2.ErrNoDocuments) { return nil, err } data.Users = 0 @@ -168,7 +172,7 @@ func GetStatsDaily(_ *gin.Context, params *GetStatsDailyParams) (response *Respo }}, } var results []entity.StatsDailyItem - if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskStat{})).Aggregate(pipeline, nil).All(&results); err != nil { + if err := service.GetCollection[models.TaskStat]().Aggregate(pipeline, nil).All(&results); err != nil { return nil, err } return GetDataResponse(results) @@ -248,7 +252,7 @@ func getTaskStatsByStatus(query bson.M) (data []GetStatsTaskResponseByStatusItem }, }}, } - if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Task{})).Aggregate(pipeline, nil).All(&data); err != nil { + if err := service.GetCollection[models.Task]().Aggregate(pipeline, nil).All(&data); err != nil { return nil, err } return data, nil @@ -283,7 +287,7 @@ func getTaskStatsByNode(query bson.M) (data []GetStatsTaskResponseByNodeItem, er }, }}, } - if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Task{})).Aggregate(pipeline, nil).All(&data); err != nil { + if err := service.GetCollection[models.Task]().Aggregate(pipeline, nil).All(&data); err != nil { return nil, err } return data, nil @@ -319,7 +323,7 @@ func getTaskStatsBySpider(query bson.M) (data []GetStatsTaskResponseBySpiderItem }}, {{"$limit", 10}}, } - if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.Task{})).Aggregate(pipeline, nil).All(&data); err != nil { + if err := service.GetCollection[models.Task]().Aggregate(pipeline, nil).All(&data); err != nil { return nil, err } return data, nil diff --git a/frontend/crawlab-ui/package.json b/frontend/crawlab-ui/package.json index b860044b..45ccc97d 100644 --- a/frontend/crawlab-ui/package.json +++ b/frontend/crawlab-ui/package.json @@ -172,5 +172,6 @@ "json-editor-vue": "^0.17.0", "semver": "^7.6.3", "uuid": "^10.0.0" - } + }, + "packageManager": "pnpm@9.9.0+sha512.60c18acd138bff695d339be6ad13f7e936eea6745660d4cc4a776d5247c540d0edee1a563695c183a66eb917ef88f2b4feb1fc25f32a7adcadc7aaf3438e99c1" } diff --git a/frontend/crawlab-ui/src/components/core/dependency/DependencyInstallDialog.vue b/frontend/crawlab-ui/src/components/core/dependency/DependencyInstallDialog.vue index c5cd988c..e24af111 100644 --- a/frontend/crawlab-ui/src/components/core/dependency/DependencyInstallDialog.vue +++ b/frontend/crawlab-ui/src/components/core/dependency/DependencyInstallDialog.vue @@ -9,7 +9,7 @@ const ns: ListStoreNamespace = 'dependency'; const store = useStore(); const { dependency: state, node: nodeState } = store.state as RootStoreState; -const activeNodes = computed(() => nodeState.activeNodes.filter(n => n.active)); +const activeNodes = computed(() => nodeState.allNodes.filter(n => n.active)); const toInstallNodes = computed(() => { const { mode, node_ids } = state.installForm; diff --git a/frontend/crawlab-ui/src/components/core/dependency/DependencyUninstallDialog.vue b/frontend/crawlab-ui/src/components/core/dependency/DependencyUninstallDialog.vue index a91d8805..f98b4b07 100644 --- a/frontend/crawlab-ui/src/components/core/dependency/DependencyUninstallDialog.vue +++ b/frontend/crawlab-ui/src/components/core/dependency/DependencyUninstallDialog.vue @@ -9,7 +9,7 @@ const ns: ListStoreNamespace = 'dependency'; const store = useStore(); const { dependency: state, node: nodeState } = store.state as RootStoreState; -const activeNodes = computed(() => nodeState.activeNodes.filter(n => n.active)); +const activeNodes = computed(() => nodeState.allNodes.filter(n => n.active)); const toUninstallNodes = computed(() => { const { mode, node_ids } = state.uninstallForm; diff --git a/frontend/crawlab-ui/src/components/core/node/useNode.ts b/frontend/crawlab-ui/src/components/core/node/useNode.ts index 4e5bcf08..5cb1c113 100644 --- a/frontend/crawlab-ui/src/components/core/node/useNode.ts +++ b/frontend/crawlab-ui/src/components/core/node/useNode.ts @@ -18,7 +18,7 @@ const useNode = (store: Store) => { const formRules: FormRules = {}; const activeNodesSorted = computed(() => { - return state.activeNodes + return state.allNodes .filter(n => n.active) .sort((a, b) => { if (a.is_master) return -1; diff --git a/frontend/crawlab-ui/src/components/core/notification/alert/useNotificationAlert.ts b/frontend/crawlab-ui/src/components/core/notification/alert/useNotificationAlert.ts index 48322342..df175c9e 100644 --- a/frontend/crawlab-ui/src/components/core/notification/alert/useNotificationAlert.ts +++ b/frontend/crawlab-ui/src/components/core/notification/alert/useNotificationAlert.ts @@ -31,7 +31,7 @@ const useNotificationAlert = (store: Store) => { () => [ { label: t('common.mode.all'), value: EMPTY_OBJECT_ID }, - ...nodeState.activeNodes + ...nodeState.allNodes .filter(node => node.active) .map(node => ({ label: node.name, diff --git a/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue b/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue index 40089492..fc0b493f 100644 --- a/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue +++ b/frontend/crawlab-ui/src/components/core/spider/SpiderForm.vue @@ -94,9 +94,10 @@ const activeTemplateOption = computed(() => { return getSpiderTemplates().find(d => d.name === form.value.template); }); -const getActiveNodes = async () => { - await store.dispatch('node/getActiveNodes'); +const getAllNodes = async () => { + await store.dispatch('node/getAllNodes'); }; +onBeforeMount(getAllNodes); defineExpose({ validate, diff --git a/frontend/crawlab-ui/src/interfaces/store/modules/node.d.ts b/frontend/crawlab-ui/src/interfaces/store/modules/node.d.ts index 8195776c..6ca912b9 100644 --- a/frontend/crawlab-ui/src/interfaces/store/modules/node.d.ts +++ b/frontend/crawlab-ui/src/interfaces/store/modules/node.d.ts @@ -7,17 +7,17 @@ type NodeStoreModule = BaseModule< interface NodeStoreState extends BaseStoreState { nodeMetricsMap: Record; - activeNodes: CNode[]; + allNodes: CNode[]; } type NodeStoreGetters = BaseStoreGetters; interface NodeStoreMutations extends BaseStoreMutations { setNodeMetricsMap: StoreMutation>; - setActiveNodes: StoreMutation; + setAllNodes: StoreMutation; } interface NodeStoreActions extends BaseStoreActions { getNodeMetrics: StoreAction; - getActiveNodes: StoreAction; + getAllNodes: StoreAction; } diff --git a/frontend/crawlab-ui/src/store/modules/node.ts b/frontend/crawlab-ui/src/store/modules/node.ts index 9ac63d22..dbbbc77a 100644 --- a/frontend/crawlab-ui/src/store/modules/node.ts +++ b/frontend/crawlab-ui/src/store/modules/node.ts @@ -21,7 +21,7 @@ const state = { { id: TAB_NAME_MONITORING, title: 'common.tabs.monitoring' }, ], nodeMetricsMap: {}, - activeNodes: [], + allNodes: [], } as NodeStoreState; const getters = { @@ -33,8 +33,8 @@ const mutations = { setNodeMetricsMap(state: NodeStoreState, metricsMap: Record) { state.nodeMetricsMap = metricsMap; }, - setActiveNodes: (state: NodeStoreState, activeNodes: CNode[]) => { - state.activeNodes = activeNodes; + setAllNodes: (state: NodeStoreState, allNodes: CNode[]) => { + state.allNodes = allNodes; }, } as NodeStoreMutations; @@ -51,9 +51,9 @@ const actions = { commit('setNodeMetricsMap', res.data); return res; }, - async getActiveNodes({ commit }: StoreActionContext) { + async getAllNodes({ commit }: StoreActionContext) { const res = await get('/nodes'); - commit('setActiveNodes', res.data || []); + commit('setAllNodes', res.data || []); return res; }, } as NodeStoreActions;