added batch run tasks

frontend/src/components/Common/BatchCrawlDialog.vue

@@ -0,0 +1,284 @@
+<template>
+  <el-dialog
+    :visible="visible"
+    width="1200px"
+    :before-close="beforeClose"
+  >
+    <el-table
+      :data="batchCrawlList"
+    >
+      <el-table-column
+        :label="$t('Spider')"
+        width="150px"
+      >
+        <template slot-scope="scope">
+          <el-select
+            v-model="scope.row.spider_id"
+            size="mini"
+            filterable
+            :placeholder="$t('Spider')"
+            @change="onSpiderChange(scope.row, $event)"
+          >
+<!--            <el-option :label="$t('Same Above')" value="same-above" :placeholder="$t('Spider')"/>-->
+            <el-option
+              v-for="op in allSpiderList"
+              :key="op._id"
+              :label="`${op.display_name} (${op.name})`"
+              :value="op._id"
+            />
+          </el-select>
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Run Type')"
+        width="150px"
+      >
+        <template slot-scope="scope">
+          <el-select v-model="scope.row.run_type" size="mini">
+            <el-option value="all-nodes" :label="$t('All Nodes')"/>
+            <el-option value="selected-nodes" :label="$t('Selected Nodes')"/>
+            <el-option value="random" :label="$t('Random')"/>
+          </el-select>
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Nodes')"
+        width="250px"
+      >
+        <template v-if="scope.row.run_type === 'selected-nodes'" slot-scope="scope">
+          <el-select
+            v-model="scope.row.node_ids"
+            size="mini"
+            multiple
+            :placeholder="$t('Nodes')"
+          >
+            <el-option
+              v-for="n in activeNodeList"
+              :key="n._id"
+              :label="n.name"
+              :value="n._id"
+            />
+          </el-select>
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Scrapy Spider')"
+        width="150px"
+      >
+        <template v-if="getSpiderById(scope.row.spider_id).is_scrapy" slot-scope="scope">
+          <el-select
+            v-model="scope.row.scrapy_spider_name"
+            size="mini"
+            :placeholder="$t('Scrapy Spider')"
+            :disabled="!scope.row.scrapy_spider_name"
+          >
+            <el-option
+              v-for="(n, index) in getScrapySpiderNames(scope.row.spider_id)"
+              :key="index"
+              :label="n"
+              :value="n"
+            />
+          </el-select>
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Scrapy Log Level')"
+        width="120px"
+      >
+        <template v-if="getSpiderById(scope.row.spider_id).is_scrapy" slot-scope="scope">
+          <el-select
+            v-model="scope.row.scrapy_log_level"
+            :placeholder="$t('Scrapy Log Level')"
+            size="mini"
+          >
+            <el-option value="INFO" label="INFO" />
+            <el-option value="DEBUG" label="DEBUG" />
+            <el-option value="WARN" label="WARN" />
+            <el-option value="ERROR" label="ERROR" />
+          </el-select>
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Parameters')"
+        min-width="150px"
+      >
+        <template slot-scope="scope">
+          <el-input v-model="scope.param" size="mini" :placeholder="$t('Parameters')"/>
+        </template>
+      </el-table-column>
+      <el-table-column
+        :label="$t('Action')"
+        fixed="right"
+        width="150px"
+      >
+        <template slot-scope="scope">
+          <el-button icon="el-icon-plus" size="mini" type="primary" @click="onAdd(scope.$index)"/>
+          <el-button icon="el-icon-delete" size="mini" type="danger" @click="onRemove(scope.$index)"/>
+        </template>
+      </el-table-column>
+    </el-table>
+    <template slot="footer">
+      <el-button type="plain" size="small" @click="$emit('close')">{{ $t('Cancel') }}</el-button>
+      <el-button type="plain" size="small" @click="reset">
+        {{ $t('Reset') }}
+      </el-button>
+      <el-button type="primary" size="small" :disabled="isConfirmDisabled" @click="onConfirm">
+        {{ $t('Confirm') }}
+      </el-button>
+    </template>
+  </el-dialog>
+</template>
+
+<script>
+  import {
+    mapState
+  } from 'vuex'
+
+  export default {
+    name: 'BatchCrawlDialog',
+    props: {
+      visible: {
+        type: Boolean,
+        default: false
+      }
+    },
+    data() {
+      return {
+        scrapySpidersNamesDict: {}
+      }
+    },
+    computed: {
+      ...mapState('task', [
+        'batchCrawlList'
+      ]),
+      ...mapState('spider', [
+        'allSpiderList'
+      ]),
+      ...mapState('node', [
+        'nodeList'
+      ]),
+      isBatchCrawlDialogVisible: {
+        get() {
+          return this.$store.state.task.isBatchCrawlDialogVisible
+        },
+        set(value) {
+          this.$store.commit('task/SET_IS_BATCH_CRAWL_DIALOG_VISIBLE', value)
+        }
+      },
+      activeNodeList() {
+        return this.nodeList.filter(n => n.status === 'online')
+      },
+      validCrawlList() {
+        return this.batchCrawlList.filter(d => !!d.spider_id)
+      },
+      isConfirmDisabled() {
+        for (let i = 0; i < this.validCrawlList.length; i++) {
+          const row = this.validCrawlList[i]
+          const spider = this.getSpiderById(row.spider_id)
+          if (!spider) {
+            return true
+          }
+          if (spider.is_scrapy && !row.scrapy_spider_name) {
+            return true
+          }
+        }
+        return false
+      },
+      scrapySpidersIds() {
+        return Array.from(new Set(this.validCrawlList.filter(d => {
+          const spider = this.getSpiderById(d.spider_id)
+          return spider && spider.is_scrapy
+        }).map(d => d.spider_id)))
+      }
+    },
+    watch: {
+      visible() {
+        if (this.visible) {
+          this.fetchAllScrapySpiderNames()
+        }
+      }
+    },
+    methods: {
+      beforeClose() {
+        this.$emit('close')
+      },
+      reset() {
+        this.$store.commit('task/SET_BATCH_CRAWL_LIST', [])
+        for (let i = 0; i < 10; i++) {
+          this.batchCrawlList.push({
+            spider_id: '',
+            run_type: 'random',
+            param: '',
+            scrapy_log_level: 'INFO'
+          })
+        }
+      },
+      getSpiderById(id) {
+        return this.allSpiderList.filter(d => d._id === id)[0] || {}
+      },
+      async onSpiderChange(row, id) {
+        const spider = this.getSpiderById(id)
+        if (!spider) return
+        if (spider.is_scrapy) {
+          await this.fetchScrapySpiderNames(id)
+          if (this.scrapySpidersNamesDict[id] && this.scrapySpidersNamesDict[id].length > 0) {
+            this.$set(row, 'scrapy_spider_name', this.scrapySpidersNamesDict[id][0])
+          }
+        }
+      },
+      getScrapySpiderNames(id) {
+        if (!this.scrapySpidersNamesDict[id]) return []
+        return this.scrapySpidersNamesDict[id]
+      },
+      async onConfirm() {
+        await this.$request.put('/tasks/batch', this.validCrawlList.map(d => {
+          const spider = this.getSpiderById(d.spider_id)
+          // Scrapy爬虫特殊处理
+          if (spider.type === 'customized' && spider.is_scrapy) {
+            d.param = `${this.scrapySpidersNamesDict[d.spider_id] ? this.scrapySpidersNamesDict[d.spider_id][0] : ''} --loglevel=${d.scrapy_log_level} ${d.param || ''}`
+          }
+          return d
+        }))
+        this.reset()
+        this.$emit('close')
+      },
+      async fetchScrapySpiderNames(id) {
+        if (!this.scrapySpidersNamesDict[id]) {
+          const res = await this.$request.get(`/spiders/${id}/scrapy/spiders`)
+          this.$set(this.scrapySpidersNamesDict, id, res.data.data)
+        }
+      },
+      async fetchAllScrapySpiderNames() {
+        await Promise.all(this.scrapySpidersIds.map(async id => {
+          await this.fetchScrapySpiderNames(id)
+        }))
+        this.validCrawlList.filter(d => {
+          const spider = this.getSpiderById(d.spider_id)
+          return spider && spider.is_scrapy
+        }).forEach(row => {
+          const id = row.spider_id
+          if (this.scrapySpidersNamesDict[id] && this.scrapySpidersNamesDict[id].length > 0) {
+            this.$set(row, 'scrapy_spider_name', this.scrapySpidersNamesDict[id][0])
+          }
+        })
+      },
+      onAdd(rowIndex) {
+        this.batchCrawlList.splice(rowIndex + 1, 0, {
+          spider_id: '',
+          run_type: 'random',
+          param: '',
+          scrapy_log_level: 'INFO'
+        })
+      },
+      onRemove(rowIndex) {
+        this.batchCrawlList.splice(rowIndex, 1)
+      }
+    }
+  }
+</script>
+
+<style scoped>
+  .el-table .el-button {
+    padding: 7px;
+  }
+</style>

frontend/src/i18n/zh.js

@@ -234,6 +234,8 @@ export default {
   'Overwrite': '覆盖',
   'Ignore': '忽略',
   'De-Duplication': '去重',
+  'Same Above': '同上',
+  'Batch Run': '批量运行',
 
   // 爬虫列表
   'Name': '名称',
@@ -659,6 +661,7 @@ export default {
   'Please enter Web Hook URL': '请输入 Web Hook URL',
   'Are you sure to download this spider?': '您确定要下载该爬虫?',
   'Downloaded successfully': '下载成功',
+  'Unable to submit because of some errors': '有错误，无法提交',
 
   // 其他
   'Star crawlab-team/crawlab on GitHub': '在 GitHub 上为 Crawlab 加星吧'

frontend/src/store/modules/spider.js

@@ -4,10 +4,13 @@ import request from '../../api/request'
 const state = {
   // list of spiders
   spiderList: [],
-  allSpiderList: [],
 
+  // total number of spiders
   spiderTotal: 0,
 
+  // list of all spiders
+  allSpiderList: [],
+
   // active spider data
   spiderForm: {},
 

frontend/src/store/modules/task.js

@@ -32,7 +32,9 @@ const state = {
   activeErrorLogItem: {},
   // results
   resultsPageNum: 1,
-  resultsPageSize: 10
+  resultsPageSize: 10,
+  // batch crawl
+  batchCrawlList: []
 }
 
 const getters = {
@@ -144,6 +146,12 @@ const mutations = {
   },
   SET_ACTIVE_ERROR_LOG_ITEM(state, value) {
     state.activeErrorLogItem = value
+  },
+  SET_BATCH_CRAWL_LIST(state, value) {
+    state.batchCrawlList = value
+  },
+  SET_IS_BATCH_CRAWL_DIALOG_VISIBLE(state, value) {
+    state.isBatchCrawlDialogVisible = value
   }
 }
 

frontend/src/views/spider/SpiderList.vue

@@ -315,6 +315,14 @@
     />
     <!--./copy dialog-->
 
+    <!--batch crawl dialog-->
+    <batch-crawl-dialog
+      :visible="batchCrawlDialogVisible"
+      @close="onBatchCrawlDialogClose"
+      @confirm="onBatchCrawlConfirm"
+    />
+    <!--./batch crawl dialog-->
+
     <el-card style="border-radius: 0">
       <!--filter-->
       <div class="filter">
@@ -379,15 +387,14 @@
         </div>
         <div class="right">
           <el-button
-            v-if="this.selectedSpiders.length"
             size="small"
             type="danger"
             icon="el-icon-video-play"
             class="btn add"
             style="font-weight: bolder"
-            @click="onCrawlSelectedSpiders"
+            @click="onBatchCrawl"
           >
-            {{ $t('Run') }}
+            {{ $t('Batch Run') }}
           </el-button>
           <el-button
             v-if="this.selectedSpiders.length"
@@ -686,6 +693,7 @@
   } from 'vuex'
   import dayjs from 'dayjs'
   import CrawlConfirmDialog from '../../components/Common/CrawlConfirmDialog'
+  import BatchCrawlDialog from '../../components/Common/BatchCrawlDialog'
   import StatusTag from '../../components/Status/StatusTag'
   import StatusLegend from '../../components/Status/StatusLegend'
   import CopySpiderDialog from '../../components/Spider/CopySpiderDialog'
@@ -696,7 +704,8 @@
       CopySpiderDialog,
       StatusLegend,
       CrawlConfirmDialog,
-      StatusTag
+      StatusTag,
+      BatchCrawlDialog
     },
     data() {
       return {
@@ -710,6 +719,7 @@
         dialogVisible: false,
         addDialogVisible: false,
         crawlConfirmDialogVisible: false,
+        batchCrawlDialogVisible: false,
         isRunningTasksDialogVisible: false,
         activeSpiderId: undefined,
         activeSpider: undefined,
@@ -868,6 +878,9 @@
         'spiderTotal',
         'templateList'
       ]),
+      ...mapState('task', [
+        'batchCrawlList'
+      ]),
       ...mapGetters('user', [
         'userInfo',
         'token'
@@ -928,6 +941,9 @@
       // fetch spider list
       await this.getList()
 
+      // fetch all spider list
+      await this.$store.dispatch('spider/getAllSpiderList')
+
       // fetch template list
       await this.$store.dispatch('spider/getTemplateList')
 
@@ -1076,6 +1092,11 @@
           this.getList()
         }, 1000)
       },
+      onBatchCrawlConfirm() {
+        setTimeout(() => {
+          this.getList()
+        }, 1000)
+      },
       onCopy(row, ev) {
         ev.stopPropagation()
         this.copyDialogVisible = true
@@ -1297,10 +1318,36 @@
         this.crawlConfirmDialogVisible = true
         this.isMultiple = true
       },
+      onBatchCrawl() {
+        this.$store.commit('task/SET_BATCH_CRAWL_LIST', this.selectedSpiders.map(d => {
+          return {
+            spider_id: d._id,
+            run_type: 'random',
+            param: '',
+            scrapy_log_level: 'INFO'
+          }
+        }))
+        if (this.batchCrawlList.length < 10) {
+          for (let i = this.batchCrawlList.length; i < 10; i++) {
+            this.batchCrawlList.push({
+              spider_id: '',
+              run_type: 'random',
+              param: '',
+              scrapy_log_level: 'INFO'
+            })
+          }
+        }
+        this.batchCrawlDialogVisible = true
+        this.isMultiple = true
+      },
       onCrawlConfirmDialogClose() {
         this.crawlConfirmDialogVisible = false
         this.isMultiple = false
       },
+      onBatchCrawlDialogClose() {
+        this.batchCrawlDialogVisible = false
+        this.isMultiple = false
+      },
       isDisabled(row) {
         return row.is_public && row.username !== this.userInfo.username && this.userInfo.role !== 'admin'
       }