From 7835de2641a9fd58c16ca9114bb6553c22649ad4 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 3 Jan 2020 17:20:24 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E7=88=AC=E8=99=AB=E8=AF=A6=E6=83=85?= =?UTF-8?q?=E6=98=BE=E7=A4=BA=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/main.go | 2 +- backend/model/spider.go | 22 +------- backend/routes/spider.go | 36 +++++++----- frontend/src/store/modules/spider.js | 5 ++ frontend/src/views/result/ResultList.vue | 2 +- frontend/src/views/schedule/ScheduleList.vue | 29 +++++++++- frontend/src/views/spider/SpiderDetail.vue | 5 ++ frontend/src/views/spider/SpiderList.vue | 5 -- frontend/src/views/spider/SpiderSchedules.vue | 55 +++++++++++++++++++ 9 files changed, 117 insertions(+), 44 deletions(-) create mode 100644 frontend/src/views/spider/SpiderSchedules.vue diff --git a/backend/main.go b/backend/main.go index 7dd5046e..74fac484 100644 --- a/backend/main.go +++ b/backend/main.go @@ -168,7 +168,7 @@ func main() { authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名 authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 - authGroup.GET("/spider/types", routes.GetSpiderTypes) // 爬虫类型 + authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务 // 可配置爬虫 authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置 authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置 diff --git a/backend/model/spider.go b/backend/model/spider.go index 02c3aa8d..e08352e4 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -275,27 +275,7 @@ func GetSpiderCount() (int, error) { return count, nil } -// 获取爬虫类型 -func GetSpiderTypes() ([]*entity.SpiderType, error) { - s, c := database.GetCol("spiders") - defer s.Close() - - group := bson.M{ - "$group": bson.M{ - "_id": "$type", - "count": bson.M{"$sum": 1}, - }, - } - var types []*entity.SpiderType - if err := c.Pipe([]bson.M{group}).All(&types); err != nil { - log.Errorf("get spider types error: %s", err.Error()) - debug.PrintStack() - return nil, err - } - - return types, nil -} - +// 获取爬虫定时任务 func GetConfigSpiderData(spider Spider) (entity.ConfigSpiderData, error) { // 构造配置数据 configData := entity.ConfigSpiderData{} diff --git a/backend/routes/spider.go b/backend/routes/spider.go index a5623b67..ca9f74d9 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -693,20 +693,6 @@ func RenameSpiderFile(c *gin.Context) { }) } -// 爬虫类型 -func GetSpiderTypes(c *gin.Context) { - types, err := model.GetSpiderTypes() - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } - c.JSON(http.StatusOK, Response{ - Status: "ok", - Message: "success", - Data: types, - }) -} - func GetSpiderStats(c *gin.Context) { type Overview struct { TaskCount int `json:"task_count" bson:"task_count"` @@ -826,3 +812,25 @@ func GetSpiderStats(c *gin.Context) { }, }) } + +func GetSpiderSchedules(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid") + return + } + + // 获取定时任务 + list, err := model.GetScheduleList(bson.M{"spider_id": bson.ObjectIdHex(id)}) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: list, + }) +} diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js index 0b669a6d..4632483a 100644 --- a/frontend/src/store/modules/spider.js +++ b/frontend/src/store/modules/spider.js @@ -180,6 +180,11 @@ const actions = { async getTemplateList ({ state, commit }) { const res = await request.get(`/config_spiders_templates`) commit('SET_TEMPLATE_LIST', res.data.data) + }, + async getScheduleList ({ state, commit }, payload) { + const { id } = payload + const res = await request.get(`/spiders/${id}/schedules`) + commit('schedule/SET_SCHEDULE_LIST', res.data.data, { root: true }) } } diff --git a/frontend/src/views/result/ResultList.vue b/frontend/src/views/result/ResultList.vue index 2f3e820b..44d04c46 100644 --- a/frontend/src/views/result/ResultList.vue +++ b/frontend/src/views/result/ResultList.vue @@ -240,7 +240,7 @@ export default { message: 'Deleted successfully' }) }) - }) + })) }, onDeploy (row) { this.$store.dispatch('spider/getSpiderData', row._id) diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index 23d9fa97..0cfbb6d2 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -32,8 +32,30 @@ /> - - + + + + + + + + + + + @@ -38,10 +41,12 @@ import SpiderOverview from '../../components/Overview/SpiderOverview' import EnvironmentList from '../../components/Environment/EnvironmentList' import SpiderStats from '../../components/Stats/SpiderStats' import ConfigList from '../../components/Config/ConfigList' +import SpiderSchedules from './SpiderSchedules' export default { name: 'SpiderDetail', components: { + SpiderSchedules, ConfigList, SpiderStats, EnvironmentList, diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 80c3fa0b..fa6c6718 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -556,11 +556,6 @@ export default { } this.$store.dispatch('spider/getSpiderList', params) } - // getTypes () { - // request.get(`/spider/types`).then(resp => { - // this.types = resp.data.data - // }) - // } }, async created () { // fetch spider types diff --git a/frontend/src/views/spider/SpiderSchedules.vue b/frontend/src/views/spider/SpiderSchedules.vue new file mode 100644 index 00000000..0315e47d --- /dev/null +++ b/frontend/src/views/spider/SpiderSchedules.vue @@ -0,0 +1,55 @@ + From b98240fab6009d65fb11dc183876fbd3b7ffe5ac Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 3 Jan 2020 17:24:57 +0800 Subject: [PATCH 2/2] updated CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1275b9fa..81ea730d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Features / Enhancement - **Dependency Installation**. Allow users to install/uninstall dependencies and add programming languages (Node.js only for now) on the platform web interface. - **Pre-install Programming Languages in Docker**. Allow Docker users to set `CRAWLAB_SERVER_LANG_NODE` as `Y` to pre-install `Node.js` environments. +- **Add Schedule List in Spider Detail Page**. Allow users to view / add / edit schedule cron jobs in the spider detail page. [#360](https://github.com/crawlab-team/crawlab/issues/360) # 0.4.2 (2019-12-26)