Merge remote-tracking branch 'crawlab-team/develop' into develop

CHANGELOG.md

@@ -3,6 +3,7 @@
 ### Features / Enhancement
 - **Dependency Installation**. Allow users to install/uninstall dependencies and add programming languages (Node.js only for now) on the platform web interface.
 - **Pre-install Programming Languages in Docker**. Allow Docker users to set `CRAWLAB_SERVER_LANG_NODE` as `Y` to pre-install `Node.js` environments.
+- **Add Schedule List in Spider Detail Page**. Allow users to view / add / edit schedule cron jobs in the spider detail page. [#360](https://github.com/crawlab-team/crawlab/issues/360)
 
 
 # 0.4.2 (2019-12-26)

backend/main.go

@@ -168,7 +168,7 @@ func main() {
 			authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
 			authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir)              // 爬虫目录
 			authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats)          // 爬虫统计数据
-			authGroup.GET("/spider/types", routes.GetSpiderTypes)               // 爬虫类型
+			authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules)  // 爬虫定时任务
 			// 可配置爬虫
 			authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig)           // 获取可配置爬虫配置
 			authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig)         // 更改可配置爬虫配置

backend/model/spider.go

@@ -275,27 +275,7 @@ func GetSpiderCount() (int, error) {
 	return count, nil
 }
 
-// 获取爬虫类型
-func GetSpiderTypes() ([]*entity.SpiderType, error) {
-	s, c := database.GetCol("spiders")
-	defer s.Close()
-
-	group := bson.M{
-		"$group": bson.M{
-			"_id":   "$type",
-			"count": bson.M{"$sum": 1},
-		},
-	}
-	var types []*entity.SpiderType
-	if err := c.Pipe([]bson.M{group}).All(&types); err != nil {
-		log.Errorf("get spider types error: %s", err.Error())
-		debug.PrintStack()
-		return nil, err
-	}
-
-	return types, nil
-}
-
+// 获取爬虫定时任务
 func GetConfigSpiderData(spider Spider) (entity.ConfigSpiderData, error) {
 	// 构造配置数据
 	configData := entity.ConfigSpiderData{}

backend/routes/spider.go

@@ -693,20 +693,6 @@ func RenameSpiderFile(c *gin.Context) {
 	})
 }
 
-// 爬虫类型
-func GetSpiderTypes(c *gin.Context) {
-	types, err := model.GetSpiderTypes()
-	if err != nil {
-		HandleError(http.StatusInternalServerError, c, err)
-		return
-	}
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data:    types,
-	})
-}
-
 func GetSpiderStats(c *gin.Context) {
 	type Overview struct {
 		TaskCount            int     `json:"task_count" bson:"task_count"`
@@ -826,3 +812,25 @@ func GetSpiderStats(c *gin.Context) {
 		},
 	})
 }
+
+func GetSpiderSchedules(c *gin.Context) {
+	id := c.Param("id")
+
+	if !bson.IsObjectIdHex(id) {
+		HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid")
+		return
+	}
+
+	// 获取定时任务
+	list, err := model.GetScheduleList(bson.M{"spider_id": bson.ObjectIdHex(id)})
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    list,
+	})
+}

frontend/src/store/modules/spider.js

@@ -180,6 +180,11 @@ const actions = {
   async getTemplateList ({ state, commit }) {
     const res = await request.get(`/config_spiders_templates`)
     commit('SET_TEMPLATE_LIST', res.data.data)
+  },
+  async getScheduleList ({ state, commit }, payload) {
+    const { id } = payload
+    const res = await request.get(`/spiders/${id}/schedules`)
+    commit('schedule/SET_SCHEDULE_LIST', res.data.data, { root: true })
   }
 }
 

frontend/src/views/result/ResultList.vue

@@ -240,7 +240,7 @@ export default {
               message: 'Deleted successfully'
             })
           })
-      })
+      }))
     },
     onDeploy (row) {
       this.$store.dispatch('spider/getSpiderData', row._id)

frontend/src/views/schedule/ScheduleList.vue

@@ -32,8 +32,30 @@
             />
           </el-select>
         </el-form-item>
-        <el-form-item :label="$t('Spider')" prop="spider_id" required>
-          <el-select v-model="scheduleForm.spider_id" :placeholder="$t('Spider')" filterable>
+        <el-form-item v-if="!isDisabledSpiderSchedule" :label="$t('Spider')" prop="spider_id" required>
+          <el-select
+            v-model="scheduleForm.spider_id"
+            :placeholder="$t('Spider')"
+            filterable
+            :disabled="isDisabledSpiderSchedule"
+          >
+            <el-option
+              v-for="op in spiderList"
+              :key="op._id"
+              :value="op._id"
+              :label="`${op.display_name} (${op.name})`"
+              :disabled="isDisabledSpider(op)"
+            >
+            </el-option>
+          </el-select>
+        </el-form-item>
+        <el-form-item v-else :label="$t('Spider')" required>
+          <el-select
+            :value="spiderId"
+            :placeholder="$t('Spider')"
+            filterable
+            :disabled="isDisabledSpiderSchedule"
+          >
             <el-option
               v-for="op in spiderList"
               :key="op._id"
@@ -199,6 +221,9 @@ export default {
         }
       }
       return {}
+    },
+    isDisabledSpiderSchedule () {
+      return false
     }
   },
   methods: {

frontend/src/views/spider/SpiderDetail.vue

@@ -25,6 +25,9 @@
       <el-tab-pane :label="$t('Analytics')" name="analytics">
         <spider-stats ref="spider-stats"/>
       </el-tab-pane>
+      <el-tab-pane :label="$t('Schedules')" name="schedules">
+        <spider-schedules/>
+      </el-tab-pane>
     </el-tabs>
   </div>
 </template>
@@ -38,10 +41,12 @@ import SpiderOverview from '../../components/Overview/SpiderOverview'
 import EnvironmentList from '../../components/Environment/EnvironmentList'
 import SpiderStats from '../../components/Stats/SpiderStats'
 import ConfigList from '../../components/Config/ConfigList'
+import SpiderSchedules from './SpiderSchedules'
 
 export default {
   name: 'SpiderDetail',
   components: {
+    SpiderSchedules,
     ConfigList,
     SpiderStats,
     EnvironmentList,

frontend/src/views/spider/SpiderList.vue

@@ -556,11 +556,6 @@ export default {
       }
       this.$store.dispatch('spider/getSpiderList', params)
     }
-    // getTypes () {
-    //   request.get(`/spider/types`).then(resp => {
-    //     this.types = resp.data.data
-    //   })
-    // }
   },
   async created () {
     // fetch spider types

frontend/src/views/spider/SpiderSchedules.vue

@@ -0,0 +1,55 @@
+<script>
+import ScheduleList from '../schedule/ScheduleList'
+
+export default {
+  name: 'SpiderSchedules',
+  extends: ScheduleList,
+  computed: {
+    isDisabledSpiderSchedule () {
+      return true
+    },
+    spiderId () {
+      const arr = this.$route.path.split('/')
+      return arr[arr.length - 1]
+    }
+  },
+  methods: {
+    getNodeList () {
+      this.$request.get('/nodes', {}).then(response => {
+        this.nodeList = response.data.data.map(d => {
+          d.systemInfo = {
+            os: '',
+            arch: '',
+            num_cpu: '',
+            executables: []
+          }
+          return d
+        })
+      })
+    },
+    getSpiderList () {
+      this.$request.get('/spiders', {})
+        .then(response => {
+          this.spiderList = response.data.data.list || []
+        })
+    },
+    onAdd () {
+      this.isEdit = false
+      this.dialogVisible = true
+      this.$store.commit('schedule/SET_SCHEDULE_FORM', { node_ids: [], spider_id: this.spiderId })
+      this.$st.sendEv('定时任务', '添加定时任务')
+    }
+  },
+  created () {
+    const arr = this.$route.path.split('/')
+    const id = arr[arr.length - 1]
+    this.$store.dispatch(`spider/getScheduleList`, { id })
+
+    // 节点列表
+    this.getNodeList()
+
+    // 爬虫列表
+    this.getSpiderList()
+  }
+}
+</script>