加入长任务支持

CHANGELOG-zh.md

@@ -2,6 +2,7 @@
 ### 功能 / 优化
 - **更好的支持 Scrapy**. 爬虫识别，`settings.py` 配置，日志级别选择，爬虫选择. [#435](https://github.com/crawlab-team/crawlab/issues/435)
 - **Git 同步**. 允许用户将 Git 项目同步到 Crawlab.
+- **版本升级检测**. 检测最新版本，通知用户升级.
 
 ### Bug 修复
 

CHANGELOG.md

@@ -2,6 +2,7 @@
 ### Features / Enhancement
 - **Better Support for Scrapy**. Spiders identification, `settings.py` configuration, log level selection, spider selection. [#435](https://github.com/crawlab-team/crawlab/issues/435)
 - **Git Sync**. Allow users to sync git projects to Crawlab.
+- **Upgrade Check**. Check latest version and notifiy users to upgrade.
 
 ### Bug Fixes
 

frontend/src/components/Common/CrawlConfirmDialog.vue

@@ -68,7 +68,7 @@
             <span style="margin-left: 5px">我已阅读并同意 <a href="javascript:"
                                                       @click="onClickDisclaimer">《免责声明》</a> 所有内容</span>
           </div>
-          <div>
+          <div v-if="spiderForm.is_long_task">
             <el-checkbox v-model="isRedirect"/>
             <span style="margin-left: 5px">跳转到任务详情页</span>
           </div>
@@ -149,10 +149,15 @@ export default {
       this.$refs['form'].validate(async valid => {
         if (!valid) return
 
+        let param = this.form.param
+        if (this.spiderForm.type === 'customized' && this.spiderForm.is_scrapy) {
+          param = `${this.form.spider} --loglevel=${this.form.scrapy_log_level} ${this.form.param}`
+        }
+
         const res = await this.$store.dispatch('spider/crawlSpider', {
           spiderId: this.spiderId,
           nodeIds: this.form.nodeIds,
-          param: `${this.form.spider} --loglevel=${this.form.scrapy_log_level} ${this.form.param}`,
+          param,
           runType: this.form.runType
         })
 
@@ -163,7 +168,7 @@ export default {
         this.$emit('close')
         this.$st.sendEv('爬虫确认', '确认运行', this.form.runType)
 
-        if (this.isRedirect) {
+        if (this.isRedirect && !this.spiderForm.is_long_task) {
           this.$router.push('/tasks/' + id)
           this.$st.sendEv('爬虫确认', '跳转到任务详情')
         }

frontend/src/components/InfoView/SpiderInfoView.vue

@@ -76,6 +76,14 @@
               />
             </el-form-item>
           </el-col>
+          <el-col :span="6">
+            <el-form-item v-if="!isView" :label="$t('Is Long Task')" prop="is_long_task">
+              <el-switch
+                v-model="spiderForm.is_long_task"
+                active-color="#13ce66"
+              />
+            </el-form-item>
+          </el-col>
         </el-row>
       </el-form>
     </el-row>

frontend/src/components/Status/StatusTag.vue

@@ -51,8 +51,9 @@ export default {
         return 'el-icon-video-pause'
       } else if (this.status === 'abnormal') {
         return 'el-icon-question'
+      } else {
+        return 'el-icon-question'
       }
-      return ''
     }
   }
 }

frontend/src/i18n/zh.js

@@ -78,6 +78,8 @@ export default {
   'Sync Frequency': '同步频率',
   'Reset': '重置',
   'Copy': '复制',
+  'Upgrade': '版本升级',
+  'Ok': '确定',
 
   // 主页
   'Total Tasks': '总任务数',
@@ -210,6 +212,9 @@ export default {
   'Git Password': 'Git 密码',
   'Has Credential': '需要验证',
   'SSH Public Key': 'SSH 公钥',
+  'Is Long Task': '是否为长任务',
+  'Long Task': '长任务',
+  'Running Tasks': '运行任务数',
 
   // 爬虫列表
   'Name': '名称',
@@ -390,6 +395,9 @@ export default {
   'New directory name': '新目录名称',
   'Enter new file name': '输入新文件名称',
   'New file name': '新文件名称',
+  'Release Note': '发布记录',
+  'How to Upgrade': '升级方式',
+  'Release': '发布',
 
   // 登录
   'Sign in': '登录',

frontend/src/store/modules/version.js

@@ -1,5 +1,11 @@
+import request from '../../api/request'
+
 const state = {
-  version: ''
+  version: '',
+  latestRelease: {
+    name: '',
+    body: ''
+  }
 }
 
 const getters = {}
@@ -7,10 +13,20 @@ const getters = {}
 const mutations = {
   SET_VERSION: (state, value) => {
     state.version = value
+  },
+  SET_LATEST_RELEASE: (state, value) => {
+    state.latestRelease = value
   }
 }
 
-const actions = {}
+const actions = {
+  async getLatestRelease ({ commit }) {
+    const res = await request.get('/releases/latest')
+    if (!res.data.error) {
+      commit('SET_LATEST_RELEASE', res.data.data)
+    }
+  }
+}
 
 export default {
   namespaced: true,

frontend/src/utils/scrapy.js

@@ -198,6 +198,15 @@ export default {
     'TELNETCONSOLE_HOST',
     'TELNETCONSOLE_PASSWORD',
     'TELNETCONSOLE_PORT',
-    'TELNETCONSOLE_USERNAME'
+    'TELNETCONSOLE_USERNAME',
+    'REDIS_ITEMS_KEY',
+    'REDIS_ITEMS_SERIALIZER',
+    'REDIS_HOST',
+    'REDIS_PORT',
+    'REDIS_URL',
+    'REDIS_PARAMS',
+    'REDIS_START_URLS_AS_SET',
+    'REDIS_START_URLS_KEY',
+    'REDIS_ENCODING'
   ]
 }

frontend/src/views/spider/SpiderList.vue

@@ -20,7 +20,7 @@
                width="40%"
                :visible.sync="addDialogVisible"
                :before-close="onAddDialogClose">
-      <el-tabs :active-name="spiderType">
+      <el-tabs :active-name="activeTabName">
         <!-- customized -->
         <el-tab-pane name="customized" :label="$t('Customized')">
           <el-form :model="spiderForm" ref="addCustomizedForm" inline-message label-width="120px">
@@ -82,6 +82,16 @@
               <i class="fa fa-exclamation-triangle"></i> {{$t('NOTE: When uploading a zip file, please zip your' +
               ' spider files from the ROOT DIRECTORY.')}}
             </p>
+            <p>
+              <template v-if="lang === 'en'">
+                You can also upload spiders using <a href="https://docs.crawlab.cn/SDK/CLI.html" target="_blank"
+                                                     style="color: #409eff;font-weight: bolder">CLI Tool</a>.
+              </template>
+              <template v-else-if="lang === 'zh'">
+                您也可以利用 <a href="https://docs.crawlab.cn/SDK/CLI.html" target="_blank"
+                          style="color: #409eff;font-weight: bolder">CLI 工具</a> 上传爬虫。
+              </template>
+            </p>
           </el-alert>
           <div class="actions">
             <el-button size="small" type="primary" @click="onAddCustomized">{{$t('Add')}}</el-button>
@@ -211,7 +221,8 @@
       <el-tabs v-model="filter.type" @tab-click="onClickTab" class="tabs">
         <el-tab-pane :label="$t('All')" name="all" class="all"></el-tab-pane>
         <el-tab-pane :label="$t('Customized')" name="customized" class="customized"></el-tab-pane>
-        <el-tab-pane :label="$t('Configurable')" name="configurable" class="configuable"></el-tab-pane>
+        <el-tab-pane :label="$t('Configurable')" name="configurable" class="configurable"></el-tab-pane>
+        <el-tab-pane :label="$t('Long Task')" name="long-task" class="long-task"></el-tab-pane>
       </el-tabs>
       <!--./tabs-->
 
@@ -288,7 +299,7 @@
             </template>
           </el-table-column>
           <el-table-column
-            v-else-if="col.name === 'is_scrapy'"
+            v-else-if="['is_scrapy', 'is_long_task'].includes(col.name)"
             :key="col.name"
             :label="$t(col.label)"
             align="left"
@@ -298,12 +309,24 @@
             <template slot-scope="scope">
               <el-switch
                 v-if="scope.row.type === 'customized'"
-                v-model="scope.row.is_scrapy"
+                v-model="scope.row[col.name]"
                 active-color="#13ce66"
                 disabled
               />
             </template>
           </el-table-column>
+          <el-table-column
+            v-else-if="col.name === 'running_tasks'"
+            :key="col.name"
+            :label="$t(col.label)"
+            :width="col.width"
+          >
+            <template slot-scope="scope">
+              <el-tag :type="getRunningTasksStatusType(scope.row)" size="small">
+                {{scope.row[col.name] ? scope.row[col.name].length : '0'}} / {{activeNodeList.length}}
+              </el-tag>
+            </template>
+          </el-table-column>
           <el-table-column
             v-else
             :key="col.name"
@@ -389,21 +412,11 @@ export default {
         sortDirection: null
       },
       types: [],
-      columns: [
-        { name: 'display_name', label: 'Name', width: '160', align: 'left', sortable: true },
-        { name: 'type', label: 'Spider Type', width: '120', sortable: true },
-        { name: 'is_scrapy', label: 'Is Scrapy', width: '80' },
-        { name: 'last_status', label: 'Last Status', width: '120' },
-        { name: 'last_run_ts', label: 'Last Run', width: '140' },
-        { name: 'update_ts', label: 'Update Time', width: '140' },
-        { name: 'create_ts', label: 'Create Time', width: '140' },
-        { name: 'remark', label: 'Remark', width: '140' }
-      ],
       spiderFormRules: {
         name: [{ required: true, message: 'Required Field', trigger: 'change' }]
       },
       fileList: [],
-      spiderType: 'customized',
+      activeTabName: 'customized',
       tourSteps: [
         {
           target: '#tab-customized',
@@ -515,17 +528,18 @@ export default {
         },
         onPreviousStep: (currentStep) => {
           if (currentStep === 7) {
-            this.spiderType = 'customized'
+            this.activeTabName = 'customized'
           }
           this.$utils.tour.prevStep('spider-list-add', currentStep)
         },
         onNextStep: (currentStep) => {
           if (currentStep === 6) {
-            this.spiderType = 'configurable'
+            this.activeTabName = 'configurable'
           }
           this.$utils.tour.nextStep('spider-list-add', currentStep)
         }
-      }
+      },
+      handle: undefined
     }
   },
   computed: {
@@ -539,9 +553,15 @@ export default {
     ...mapGetters('user', [
       'token'
     ]),
+    ...mapState('lang', [
+      'lang'
+    ]),
     ...mapState('project', [
       'projectList'
     ]),
+    ...mapState('node', [
+      'nodeList'
+    ]),
     uploadForm () {
       return {
         name: this.spiderForm.name,
@@ -549,6 +569,25 @@ export default {
         col: this.spiderForm.col,
         cmd: this.spiderForm.cmd
       }
+    },
+    columns () {
+      const columns = []
+      columns.push({ name: 'display_name', label: 'Name', width: '160', align: 'left', sortable: true })
+      columns.push({ name: 'type', label: 'Spider Type', width: '120', sortable: true })
+      columns.push({ name: 'is_long_task', label: 'Is Long Task', width: '80' })
+      columns.push({ name: 'is_scrapy', label: 'Is Scrapy', width: '80' })
+      columns.push({ name: 'last_status', label: 'Last Status', width: '120' })
+      columns.push({ name: 'last_run_ts', label: 'Last Run', width: '140' })
+      columns.push({ name: 'update_ts', label: 'Update Time', width: '140' })
+      columns.push({ name: 'create_ts', label: 'Create Time', width: '140' })
+      columns.push({ name: 'running_tasks', label: 'Running Tasks', width: '120' })
+      columns.push({ name: 'remark', label: 'Remark', width: '140' })
+      return columns
+    },
+    activeNodeList () {
+      return this.nodeList.filter(d => {
+        return d.status === 'online'
+      })
     }
   },
   methods: {
@@ -797,6 +836,15 @@ export default {
         project_id: this.filter.project_id
       }
       await this.$store.dispatch('spider/getSpiderList', params)
+    },
+    getRunningTasksStatusType (row) {
+      if (!row.running_tasks || row.running_tasks.length === 0) {
+        return 'info'
+      } else if (this.activeNodeList && row.running_tasks.length === this.activeNodeList.length) {
+        return 'warning'
+      } else {
+        return 'warning'
+      }
     }
   },
   async created () {
@@ -813,6 +861,11 @@ export default {
 
     // fetch template list
     await this.$store.dispatch('spider/getTemplateList')
+
+    // periodically fetch spider list
+    this.handle = setInterval(() => {
+      this.getList()
+    }, 15000)
   },
   mounted () {
     const vm = this
@@ -824,6 +877,9 @@ export default {
       this.$tours['spider-list'].start()
       this.$st.sendEv('教程', '开始', 'spider-list')
     }
+  },
+  destroyed () {
+    clearInterval(this.handle)
   }
 }
 </script>