From b994cd49223ca629005582406453018244638864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 30 Sep 2019 16:35:36 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90=E7=88=AC=E8=99=AB=E7=9A=84?= =?UTF-8?q?=E5=88=86=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/entity/common.go | 17 +++++++++++++ backend/entity/spider.go | 4 ++-- backend/model/spider.go | 6 +++-- backend/routes/spider.go | 23 ++++++++++++------ frontend/src/components/Common/DialogView.vue | 2 +- .../components/InfoView/SpiderInfoView.vue | 24 +++++++------------ frontend/src/views/result/ResultDetail.vue | 2 +- frontend/src/views/result/ResultList.vue | 4 ++-- frontend/src/views/schedule/ScheduleList.vue | 2 +- frontend/src/views/spider/SpiderDetail.vue | 2 +- frontend/src/views/spider/SpiderList.vue | 19 +++++++++------ 11 files changed, 65 insertions(+), 40 deletions(-) create mode 100644 backend/entity/common.go diff --git a/backend/entity/common.go b/backend/entity/common.go new file mode 100644 index 00000000..332cc494 --- /dev/null +++ b/backend/entity/common.go @@ -0,0 +1,17 @@ +package entity + +import "strconv" + +type Page struct { + Skip int + Limit int + PageNum int + PageSize int +} + +func (p *Page)GetPage(pageNum string, pageSize string) { + p.PageNum, _ = strconv.Atoi(pageNum) + p.PageSize, _ = strconv.Atoi(pageSize) + p.Skip = p.PageSize * (p.PageNum - 1) + p.Limit = p.PageSize +} \ No newline at end of file diff --git a/backend/entity/spider.go b/backend/entity/spider.go index 8129ff40..7f5e02b4 100644 --- a/backend/entity/spider.go +++ b/backend/entity/spider.go @@ -1,6 +1,6 @@ package entity type SpiderType struct { - Type string `json:"type"` - Count int `json:"count"` + Type string `json:"type" bson:"_id"` + Count int `json:"count" bson:"count"` } diff --git a/backend/model/spider.go b/backend/model/spider.go index 6e7102d6..1f88acff 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -256,13 +256,15 @@ func GetSpiderTypes() ([]*entity.SpiderType, error) { s, c := database.GetCol("spiders") defer s.Close() + group := bson.M{ "$group": bson.M{ - "type": bson.M{"$sum": 1}, + "_id": "$type", + "count": bson.M{"$sum": 1}, }, } var types []*entity.SpiderType - if err := c.Pipe([]bson.M{group}).All(&types); err != nil { + if err := c.Pipe([]bson.M{ group}).All(&types); err != nil { log.Errorf("get spider types error: %s", err.Error()) debug.PrintStack() return nil, err diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 8f8fdd13..addddd99 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -3,6 +3,7 @@ package routes import ( "crawlab/constants" "crawlab/database" + "crawlab/entity" "crawlab/model" "crawlab/services" "crawlab/utils" @@ -24,14 +25,22 @@ import ( ) func GetSpiderList(c *gin.Context) { - pageNumStr, _ := c.GetQuery("pageNum") - pageSizeStr, _ := c.GetQuery("pageSize") + pageNum, _ := c.GetQuery("pageNum") + pageSize, _ := c.GetQuery("pageSize") keyword, _ := c.GetQuery("keyword") - pageNum, _ := strconv.Atoi(pageNumStr) - pageSize, _ := strconv.Atoi(pageSizeStr) - skip := pageSize * (pageNum - 1) - filter := bson.M{"name": bson.M{"$regex": bson.RegEx{Pattern: keyword, Options: "im"}}} - results, count, err := model.GetSpiderList(filter, skip, pageSize) + t, _ := c.GetQuery("type") + + filter := bson.M{ + "name": bson.M{"$regex": bson.RegEx{Pattern: keyword, Options: "im"}}, + } + + if t != "" { + filter["type"] = t + } + + page := &entity.Page{} + page.GetPage(pageNum, pageSize) + results, count, err := model.GetSpiderList(filter, page.Skip, page.Limit) if err != nil { HandleError(http.StatusInternalServerError, c, err) return diff --git a/frontend/src/components/Common/DialogView.vue b/frontend/src/components/Common/DialogView.vue index 7976171e..3c83d7ce 100644 --- a/frontend/src/components/Common/DialogView.vue +++ b/frontend/src/components/Common/DialogView.vue @@ -151,7 +151,7 @@ export default { } }, mounted () { - if (!this.spiderList || !this.spiderList.length) this.$store.dispatch('spider/getSpiderList') + // if (!this.spiderList || !this.spiderList.length) this.$store.dispatch('spider/getSpiderList') if (!this.nodeList || !this.nodeList.length) this.$store.dispatch('node/getNodeList') } } diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index 39702a5d..381b253c 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -18,10 +18,10 @@ - + - + @@ -39,10 +39,11 @@ - - - - + + + + + @@ -102,16 +103,7 @@ export default { 'spiderForm' ]), isShowRun () { - if (this.isCustomized) { - // customized spider - return !!this.spiderForm.cmd - } else { - // configurable spider - return !!this.spiderForm.fields - } - }, - isCustomized () { - return this.spiderForm.type === 'customized' + return !!this.spiderForm.cmd } }, methods: { diff --git a/frontend/src/views/result/ResultDetail.vue b/frontend/src/views/result/ResultDetail.vue index f42bee5c..df8487ef 100644 --- a/frontend/src/views/result/ResultDetail.vue +++ b/frontend/src/views/result/ResultDetail.vue @@ -59,7 +59,7 @@ export default { }, created () { // get the list of the spiders - this.$store.dispatch('spider/getSpiderList') + // this.$store.dispatch('spider/getSpiderList') // get spider basic info this.$store.dispatch('spider/getSpiderData', this.$route.params.id) diff --git a/frontend/src/views/result/ResultList.vue b/frontend/src/views/result/ResultList.vue index 85c70098..2f3e820b 100644 --- a/frontend/src/views/result/ResultList.vue +++ b/frontend/src/views/result/ResultList.vue @@ -195,7 +195,7 @@ export default { this.dialogVisible = true }, onRefresh () { - this.$store.dispatch('spider/getSpiderList') + // this.$store.dispatch('spider/getSpiderList') }, onSubmit () { const vm = this @@ -257,7 +257,7 @@ export default { } }, created () { - this.$store.dispatch('spider/getSpiderList') + // this.$store.dispatch('spider/getSpiderList') } } diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index 743a186e..c44d46e2 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -269,7 +269,7 @@ export default { }, created () { this.$store.dispatch('schedule/getScheduleList') - this.$store.dispatch('spider/getSpiderList') + // this.$store.dispatch('spider/getSpiderList') this.$store.dispatch('node/getNodeList') } } diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue index 69fdd770..916592f4 100644 --- a/frontend/src/views/spider/SpiderDetail.vue +++ b/frontend/src/views/spider/SpiderDetail.vue @@ -87,7 +87,7 @@ export default { }, created () { // get the list of the spiders - this.$store.dispatch('spider/getSpiderList') + // this.$store.dispatch('spider/getSpiderList') // get spider basic info this.$store.dispatch('spider/getSpiderData', this.$route.params.id) diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 5b93afb4..743aabbe 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -111,12 +111,13 @@
- - + + - + @@ -156,8 +157,7 @@ align="left" :width="col.width"> { - console.log('resp', resp) + this.types = resp.data.data }) } },