added batch add schedules

frontend/src/components/Common/BatchAddScheduleDialog.vue

@@ -0,0 +1,315 @@
+<template>
+  <el-dialog
+    :visible="visible"
+    width="1200px"
+    :before-close="beforeClose"
+  >
+    <el-table
+      :data="batchScheduleList"
+    >
+      <el-table-column
+        :label="$t('Schedule Name')"
+        width="150px"
+      >
+        <template slot-scope="scope">
+          <el-input v-model="scope.row.name" size="mini" :placeholder="$t('Schedule Name')" />
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Cron')"
+        width="150px"
+      >
+        <template slot-scope="scope">
+          <el-input v-model="scope.row.cron" size="mini" :placeholder="$t('Cron')" />
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Spider')"
+        width="150px"
+      >
+        <template slot-scope="scope">
+          <el-select
+            v-model="scope.row.spider_id"
+            size="mini"
+            filterable
+            :placeholder="$t('Spider')"
+            @change="onSpiderChange(scope.row, $event)"
+          >
+            <!--            <el-option :label="$t('Same Above')" value="same-above" :placeholder="$t('Spider')"/>-->
+            <el-option
+              v-for="op in allSpiderList"
+              :key="op._id"
+              :label="`${op.display_name} (${op.name})`"
+              :value="op._id"
+            />
+          </el-select>
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Run Type')"
+        width="150px"
+      >
+        <template slot-scope="scope">
+          <el-select v-model="scope.row.run_type" size="mini">
+            <el-option value="all-nodes" :label="$t('All Nodes')" />
+            <el-option value="selected-nodes" :label="$t('Selected Nodes')" />
+            <el-option value="random" :label="$t('Random')" />
+          </el-select>
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Nodes')"
+        width="250px"
+      >
+        <template v-if="scope.row.run_type === 'selected-nodes'" slot-scope="scope">
+          <el-select
+            v-model="scope.row.node_ids"
+            size="mini"
+            multiple
+            :placeholder="$t('Nodes')"
+          >
+            <el-option
+              v-for="n in activeNodeList"
+              :key="n._id"
+              :label="n.name"
+              :value="n._id"
+            />
+          </el-select>
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Scrapy Spider')"
+        width="150px"
+      >
+        <template v-if="getSpiderById(scope.row.spider_id).is_scrapy" slot-scope="scope">
+          <el-select
+            v-model="scope.row.scrapy_spider_name"
+            size="mini"
+            :placeholder="$t('Scrapy Spider')"
+            :disabled="!scope.row.scrapy_spider_name"
+          >
+            <el-option
+              v-for="(n, index) in getScrapySpiderNames(scope.row.spider_id)"
+              :key="index"
+              :label="n"
+              :value="n"
+            />
+          </el-select>
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Scrapy Log Level')"
+        width="120px"
+      >
+        <template v-if="getSpiderById(scope.row.spider_id).is_scrapy" slot-scope="scope">
+          <el-select
+            v-model="scope.row.scrapy_log_level"
+            :placeholder="$t('Scrapy Log Level')"
+            size="mini"
+          >
+            <el-option value="INFO" label="INFO" />
+            <el-option value="DEBUG" label="DEBUG" />
+            <el-option value="WARN" label="WARN" />
+            <el-option value="ERROR" label="ERROR" />
+          </el-select>
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Parameters')"
+        min-width="150px"
+      >
+        <template slot-scope="scope">
+          <el-input v-model="scope.param" size="mini" :placeholder="$t('Parameters')" />
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Description')"
+        width="200px"
+      >
+        <template slot-scope="scope">
+          <el-input v-model="scope.row.description" size="mini" type="textarea" :placeholder="$t('Description')" />
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Action')"
+        fixed="right"
+        width="150px"
+      >
+        <template slot-scope="scope">
+          <el-button icon="el-icon-plus" size="mini" type="primary" @click="onAdd(scope.$index)" />
+          <el-button icon="el-icon-delete" size="mini" type="danger" @click="onRemove(scope.$index)" />
+        </template>
+      </el-table-column>
+    </el-table>
+    <template slot="footer">
+      <el-button type="plain" size="small" @click="$emit('close')">{{ $t('Cancel') }}</el-button>
+      <el-button type="plain" size="small" @click="reset">
+        {{ $t('Reset') }}
+      </el-button>
+      <el-button type="primary" size="small" :disabled="isConfirmDisabled" @click="onConfirm">
+        {{ $t('Confirm') }}
+      </el-button>
+    </template>
+  </el-dialog>
+</template>
+
+<script>
+  import {
+    mapState
+  } from 'vuex'
+
+  export default {
+    name: 'BatchAddScheduleDialog',
+    props: {
+      visible: {
+        type: Boolean,
+        default: false
+      }
+    },
+    data() {
+      return {
+        scrapySpidersNamesDict: {}
+      }
+    },
+    computed: {
+      ...mapState('schedule', [
+        'batchScheduleList'
+      ]),
+      ...mapState('spider', [
+        'allSpiderList'
+      ]),
+      ...mapState('node', [
+        'nodeList'
+      ]),
+      activeNodeList() {
+        return this.nodeList.filter(n => n.status === 'online')
+      },
+      validScheduleList() {
+        return this.batchScheduleList.filter(d => !!d.spider_id && !!d.name && !!d.cron)
+      },
+      isConfirmDisabled() {
+        if (this.validScheduleList.length === 0) {
+          return true
+        }
+        for (let i = 0; i < this.validScheduleList.length; i++) {
+          const row = this.validScheduleList[i]
+          const spider = this.getSpiderById(row.spider_id)
+          if (!spider) {
+            return true
+          }
+          if (spider.is_scrapy && !row.scrapy_spider_name) {
+            return true
+          }
+        }
+        return false
+      },
+      scrapySpidersIds() {
+        return Array.from(new Set(this.validScheduleList.filter(d => {
+          const spider = this.getSpiderById(d.spider_id)
+          return spider && spider.is_scrapy
+        }).map(d => d.spider_id)))
+      }
+    },
+    watch: {
+      visible() {
+        if (this.visible) {
+          this.fetchAllScrapySpiderNames()
+        }
+      }
+    },
+    methods: {
+      beforeClose() {
+        this.$emit('close')
+      },
+      reset() {
+        this.$store.commit('task/SET_BATCH_CRAWL_LIST', [])
+        for (let i = 0; i < 10; i++) {
+          this.batchScheduleList.push({
+            spider_id: '',
+            run_type: 'random',
+            param: '',
+            scrapy_log_level: 'INFO'
+          })
+        }
+        this.$st.sendEv('批量添加定时任务', '重置')
+      },
+      getSpiderById(id) {
+        return this.allSpiderList.filter(d => d._id === id)[0] || {}
+      },
+      async onSpiderChange(row, id) {
+        const spider = this.getSpiderById(id)
+        if (!spider) return
+        if (spider.is_scrapy) {
+          await this.fetchScrapySpiderNames(id)
+          if (this.scrapySpidersNamesDict[id] && this.scrapySpidersNamesDict[id].length > 0) {
+            this.$set(row, 'scrapy_spider_name', this.scrapySpidersNamesDict[id][0])
+          }
+        }
+        this.$st.sendEv('批量添加定时任务', '选择爬虫')
+      },
+      getScrapySpiderNames(id) {
+        if (!this.scrapySpidersNamesDict[id]) return []
+        return this.scrapySpidersNamesDict[id]
+      },
+      async onConfirm() {
+        const res = await this.$request.put('/schedules/batch', this.validScheduleList.map(d => {
+          const spider = this.getSpiderById(d.spider_id)
+          // Scrapy爬虫特殊处理
+          if (spider.type === 'customized' && spider.is_scrapy) {
+            d.param = `${this.scrapySpidersNamesDict[d.spider_id] ? this.scrapySpidersNamesDict[d.spider_id][0] : ''} --loglevel=${d.scrapy_log_level} ${d.param || ''}`
+          }
+          // cron特殊处理
+          d.cron = '0 ' + d.cron
+          return d
+        }))
+        if (res.status !== 200) {
+          this.$message.error(res.data.error)
+          return
+        }
+        this.reset()
+        this.$emit('close')
+        this.$emit('confirm')
+        this.$st.sendEv('批量添加定时任务', '确认添加')
+      },
+      async fetchScrapySpiderNames(id) {
+        if (!this.scrapySpidersNamesDict[id]) {
+          const res = await this.$request.get(`/spiders/${id}/scrapy/spiders`)
+          this.$set(this.scrapySpidersNamesDict, id, res.data.data)
+        }
+      },
+      async fetchAllScrapySpiderNames() {
+        await Promise.all(this.scrapySpidersIds.map(async id => {
+          await this.fetchScrapySpiderNames(id)
+        }))
+        this.validScheduleList.filter(d => {
+          const spider = this.getSpiderById(d.spider_id)
+          return spider && spider.is_scrapy
+        }).forEach(row => {
+          const id = row.spider_id
+          if (this.scrapySpidersNamesDict[id] && this.scrapySpidersNamesDict[id].length > 0) {
+            this.$set(row, 'scrapy_spider_name', this.scrapySpidersNamesDict[id][0])
+          }
+        })
+      },
+      onAdd(rowIndex) {
+        this.batchScheduleList.splice(rowIndex + 1, 0, {
+          spider_id: '',
+          run_type: 'random',
+          param: '',
+          scrapy_log_level: 'INFO'
+        })
+        this.$st.sendEv('批量添加定时任务', '添加')
+      },
+      onRemove(rowIndex) {
+        this.batchScheduleList.splice(rowIndex, 1)
+        this.$st.sendEv('批量添加定时任务', '删除')
+      }
+    }
+  }
+</script>
+
+<style scoped>
+  .el-table .el-button {
+    padding: 7px;
+  }
+</style>

frontend/src/components/Common/BatchCrawlDialog.vue

@@ -157,14 +157,6 @@
       ...mapState('node', [
         'nodeList'
       ]),
-      isBatchCrawlDialogVisible: {
-        get() {
-          return this.$store.state.task.isBatchCrawlDialogVisible
-        },
-        set(value) {
-          this.$store.commit('task/SET_IS_BATCH_CRAWL_DIALOG_VISIBLE', value)
-        }
-      },
       activeNodeList() {
         return this.nodeList.filter(n => n.status === 'online')
       },
@@ -172,6 +164,9 @@
         return this.batchCrawlList.filter(d => !!d.spider_id)
       },
       isConfirmDisabled() {
+        if (this.validCrawlList.length === 0) {
+          return true
+        }
         for (let i = 0; i < this.validCrawlList.length; i++) {
           const row = this.validCrawlList[i]
           const spider = this.getSpiderById(row.spider_id)
@@ -243,6 +238,7 @@
         }))
         this.reset()
         this.$emit('close')
+        this.$emit('confirm')
         this.$st.sendEv('批量运行', '确认批量运行')
       },
       async fetchScrapySpiderNames(id) {

frontend/src/i18n/zh.js

@@ -314,6 +314,7 @@ export default {
   'Cron Expression': 'Cron 表达式',
   'Cron expression is invalid': 'Cron 表达式不正确',
   'View Tasks': '查看任务',
+  'Batch Add': '批量添加',
 
   // 网站
   'Site': '网站',

frontend/src/store/modules/schedule.js

@@ -4,7 +4,8 @@ const state = {
   scheduleList: [],
   scheduleForm: {
     node_ids: []
-  }
+  },
+  batchScheduleList: []
 }
 
 const getters = {}
@@ -15,6 +16,9 @@ const mutations = {
   },
   SET_SCHEDULE_FORM(state, value) {
     state.scheduleForm = value
+  },
+  SET_BATCH_SCHEDULE_LIST(state, value) {
+    state.batchScheduleList = value
   }
 }
 

frontend/src/views/schedule/ScheduleList.vue

@@ -205,13 +205,30 @@
     />
     <!--./crawl confirm dialog-->
 
+    <!--batch add schedules dialog-->
+    <batch-add-schedule-dialog
+      :visible="batchAddDialogVisible"
+      @close="onBatchAddClose"
+      @confirm="onBatchAddConfirm"
+    />
+    <!--./batch add schedules dialog-->
+
     <el-card style="border-radius: 0" class="schedule-list">
       <!--filter-->
       <div class="filter">
         <div class="right">
           <el-button
             size="small"
-            type="primary"
+            type="success"
+            icon="el-icon-document-copy"
+            class="btn-add"
+            @click="onBatchAdd"
+          >
+            {{ $t('Batch Add') }}
+          </el-button>
+          <el-button
+            size="small"
+            type="success"
             icon="el-icon-plus"
             class="btn-add"
             @click="onAdd"
@@ -339,10 +356,12 @@
   import ParametersDialog from '../../components/Common/ParametersDialog'
   import ScheduleTaskList from '../../components/Schedule/ScheduleTaskList'
   import CrawlConfirmDialog from '../../components/Common/CrawlConfirmDialog'
+  import BatchAddScheduleDialog from '../../components/Common/BatchAddScheduleDialog'
 
   export default {
     name: 'ScheduleList',
     components: {
+      BatchAddScheduleDialog,
       CrawlConfirmDialog,
       ScheduleTaskList,
       VueCronLinux,
@@ -361,12 +380,13 @@
           { name: 'description', label: 'Description', width: '200px' },
           { name: 'enable', label: 'Enable/Disable', width: '120px' },
           { name: 'username', label: 'Owner', width: '100px' }
-        // { name: 'status', label: 'Status', width: '100px' }
+          // { name: 'status', label: 'Status', width: '100px' }
         ],
         isEdit: false,
         dialogTitle: '',
         dialogVisible: false,
         cronDialogVisible: false,
+        batchAddDialogVisible: false,
         expression: '',
         nodeList: [],
         isShowCron: false,
@@ -500,7 +520,8 @@
       ]),
       ...mapState('schedule', [
         'scheduleList',
-        'scheduleForm'
+        'scheduleForm',
+        'batchScheduleList'
       ]),
       lang() {
         const lang = this.$store.state.lang.lang || window.localStorage.getItem('lang')
@@ -541,6 +562,9 @@
 
       // 爬虫列表
       this.$store.dispatch('spider/getAllSpiderList')
+
+      // 节点列表
+      this.$store.dispatch('node/getNodeList')
     },
     mounted() {
       if (!this.isDisabledSpiderSchedule) {
@@ -598,6 +622,22 @@
         })
         this.$st.sendEv('定时任务', '提交定时任务')
       },
+      onBatchAdd() {
+        this.$store.commit('schedule/SET_BATCH_SCHEDULE_LIST', [])
+        for (let i = 0; i < 10; i++) {
+          this.batchScheduleList.push({
+            name: '',
+            cron: '',
+            spider_id: '',
+            run_type: 'random',
+            param: '',
+            scrapy_log_level: 'INFO',
+            description: ''
+          })
+        }
+        this.batchAddDialogVisible = true
+        this.$st.sendEv('定时任务', '点击批量添加定时任务')
+      },
       isShowRun(row) {
       },
       async onEdit(row) {
@@ -702,6 +742,14 @@
         this.crawlConfirmDialogVisible = true
         this.$store.commit('schedule/SET_SCHEDULE_FORM', row)
         this.$st.sendEv('定时任务', '点击运行任务')
+      },
+      onBatchAddClose() {
+        this.batchAddDialogVisible = false
+      },
+      onBatchAddConfirm() {
+        setTimeout(() => {
+          this.$store.dispatch('schedule/getScheduleList')
+        }, 1000)
       }
     }
   }