加入长任务支持

2026-01-21 17:21:09 +01:00 · 2020-02-19 13:17:01 +08:00
parent e4c012cf2f
commit 010f3ec1fa
9 changed files with 130 additions and 25 deletions
--- a/CHANGELOG-zh.md
+++ b/CHANGELOG-zh.md
@@ -2,6 +2,7 @@
 ### 功能 / 优化
 - **更好的支持 Scrapy**. 爬虫识别，`settings.py` 配置，日志级别选择，爬虫选择. [#435](https://github.com/crawlab-team/crawlab/issues/435)
 - **Git 同步**. 允许用户将 Git 项目同步到 Crawlab.
+- **版本升级检测**. 检测最新版本，通知用户升级.

 ### Bug 修复

--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
 ### Features / Enhancement
 - **Better Support for Scrapy**. Spiders identification, `settings.py` configuration, log level selection, spider selection. [#435](https://github.com/crawlab-team/crawlab/issues/435)
 - **Git Sync**. Allow users to sync git projects to Crawlab.
+- **Upgrade Check**. Check latest version and notifiy users to upgrade.

 ### Bug Fixes

--- a/frontend/src/components/Common/CrawlConfirmDialog.vue
+++ b/frontend/src/components/Common/CrawlConfirmDialog.vue
@@ -68,7 +68,7 @@
            <span style="margin-left: 5px">我已阅读并同意 <a href="javascript:"
                                                      @click="onClickDisclaimer">《免责声明》</a> 所有内容</span>
          </div>
-          <div>
+          <div v-if="spiderForm.is_long_task">
            <el-checkbox v-model="isRedirect"/>
            <span style="margin-left: 5px">跳转到任务详情页</span>
          </div>
@@ -149,10 +149,15 @@ export default {
      this.$refs['form'].validate(async valid => {
        if (!valid) return

+        let param = this.form.param
+        if (this.spiderForm.type === 'customized' && this.spiderForm.is_scrapy) {
+          param = `${this.form.spider} --loglevel=${this.form.scrapy_log_level} ${this.form.param}`
+        }
+
        const res = await this.$store.dispatch('spider/crawlSpider', {
          spiderId: this.spiderId,
          nodeIds: this.form.nodeIds,
-          param: `${this.form.spider} --loglevel=${this.form.scrapy_log_level} ${this.form.param}`,
+          param,
          runType: this.form.runType
        })

@@ -163,7 +168,7 @@ export default {
        this.$emit('close')
        this.$st.sendEv('爬虫确认', '确认运行', this.form.runType)

-        if (this.isRedirect) {
+        if (this.isRedirect && !this.spiderForm.is_long_task) {
          this.$router.push('/tasks/' + id)
          this.$st.sendEv('爬虫确认', '跳转到任务详情')
        }
--- a/frontend/src/components/InfoView/SpiderInfoView.vue
+++ b/frontend/src/components/InfoView/SpiderInfoView.vue
@@ -76,6 +76,14 @@
              />
            </el-form-item>
          </el-col>
+          <el-col :span="6">
+            <el-form-item v-if="!isView" :label="$t('Is Long Task')" prop="is_long_task">
+              <el-switch
+                v-model="spiderForm.is_long_task"
+                active-color="#13ce66"
+              />
+            </el-form-item>
+          </el-col>
        </el-row>
      </el-form>
    </el-row>
--- a/frontend/src/components/Status/StatusTag.vue
+++ b/frontend/src/components/Status/StatusTag.vue
@@ -51,8 +51,9 @@ export default {
        return 'el-icon-video-pause'
      } else if (this.status === 'abnormal') {
        return 'el-icon-question'
+      } else {
+        return 'el-icon-question'
      }
-      return ''
    }
  }
 }
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -78,6 +78,8 @@ export default {
  'Sync Frequency': '同步频率',
  'Reset': '重置',
  'Copy': '复制',
+  'Upgrade': '版本升级',
+  'Ok': '确定',

  // 主页
  'Total Tasks': '总任务数',
@@ -210,6 +212,9 @@ export default {
  'Git Password': 'Git 密码',
  'Has Credential': '需要验证',
  'SSH Public Key': 'SSH 公钥',
+  'Is Long Task': '是否为长任务',
+  'Long Task': '长任务',
+  'Running Tasks': '运行任务数',

  // 爬虫列表
  'Name': '名称',
@@ -390,6 +395,9 @@ export default {
  'New directory name': '新目录名称',
  'Enter new file name': '输入新文件名称',
  'New file name': '新文件名称',
+  'Release Note': '发布记录',
+  'How to Upgrade': '升级方式',
+  'Release': '发布',

  // 登录
  'Sign in': '登录',
--- a/frontend/src/store/modules/version.js
+++ b/frontend/src/store/modules/version.js
@@ -1,5 +1,11 @@
+import request from '../../api/request'
+
 const state = {
-  version: ''
+  version: '',
+  latestRelease: {
+    name: '',
+    body: ''
+  }
 }

 const getters = {}
@@ -7,10 +13,20 @@ const getters = {}
 const mutations = {
  SET_VERSION: (state, value) => {
    state.version = value
+  },
+  SET_LATEST_RELEASE: (state, value) => {
+    state.latestRelease = value
  }
 }

-const actions = {}
+const actions = {
+  async getLatestRelease ({ commit }) {
+    const res = await request.get('/releases/latest')
+    if (!res.data.error) {
+      commit('SET_LATEST_RELEASE', res.data.data)
+    }
+  }
+}

 export default {
  namespaced: true,
--- a/frontend/src/utils/scrapy.js
+++ b/frontend/src/utils/scrapy.js
@@ -198,6 +198,15 @@ export default {
    'TELNETCONSOLE_HOST',
    'TELNETCONSOLE_PASSWORD',
    'TELNETCONSOLE_PORT',
-    'TELNETCONSOLE_USERNAME'
+    'TELNETCONSOLE_USERNAME',
+    'REDIS_ITEMS_KEY',
+    'REDIS_ITEMS_SERIALIZER',
+    'REDIS_HOST',
+    'REDIS_PORT',
+    'REDIS_URL',
+    'REDIS_PARAMS',
+    'REDIS_START_URLS_AS_SET',
+    'REDIS_START_URLS_KEY',
+    'REDIS_ENCODING'
  ]
 }
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -20,7 +20,7 @@
               width="40%"
               :visible.sync="addDialogVisible"
               :before-close="onAddDialogClose">
-      <el-tabs :active-name="spiderType">
+      <el-tabs :active-name="activeTabName">
        <!-- customized -->
        <el-tab-pane name="customized" :label="$t('Customized')">
          <el-form :model="spiderForm" ref="addCustomizedForm" inline-message label-width="120px">
@@ -82,6 +82,16 @@
              <i class="fa fa-exclamation-triangle"></i> {{$t('NOTE: When uploading a zip file, please zip your' +
              ' spider files from the ROOT DIRECTORY.')}}
            </p>
+            <p>
+              <template v-if="lang === 'en'">
+                You can also upload spiders using <a href="https://docs.crawlab.cn/SDK/CLI.html" target="_blank"
+                                                     style="color: #409eff;font-weight: bolder">CLI Tool</a>.
+              </template>
+              <template v-else-if="lang === 'zh'">
+                您也可以利用 <a href="https://docs.crawlab.cn/SDK/CLI.html" target="_blank"
+                          style="color: #409eff;font-weight: bolder">CLI 工具</a> 上传爬虫。
+              </template>
+            </p>
          </el-alert>
          <div class="actions">
            <el-button size="small" type="primary" @click="onAddCustomized">{{$t('Add')}}</el-button>
@@ -211,7 +221,8 @@
      <el-tabs v-model="filter.type" @tab-click="onClickTab" class="tabs">
        <el-tab-pane :label="$t('All')" name="all" class="all"></el-tab-pane>
        <el-tab-pane :label="$t('Customized')" name="customized" class="customized"></el-tab-pane>
-        <el-tab-pane :label="$t('Configurable')" name="configurable" class="configuable"></el-tab-pane>
+        <el-tab-pane :label="$t('Configurable')" name="configurable" class="configurable"></el-tab-pane>
+        <el-tab-pane :label="$t('Long Task')" name="long-task" class="long-task"></el-tab-pane>
      </el-tabs>
      <!--./tabs-->

@@ -288,7 +299,7 @@
            </template>
          </el-table-column>
          <el-table-column
-            v-else-if="col.name === 'is_scrapy'"
+            v-else-if="['is_scrapy', 'is_long_task'].includes(col.name)"
            :key="col.name"
            :label="$t(col.label)"
            align="left"
@@ -298,12 +309,24 @@
            <template slot-scope="scope">
              <el-switch
                v-if="scope.row.type === 'customized'"
-                v-model="scope.row.is_scrapy"
+                v-model="scope.row[col.name]"
                active-color="#13ce66"
                disabled
              />
            </template>
          </el-table-column>
+          <el-table-column
+            v-else-if="col.name === 'running_tasks'"
+            :key="col.name"
+            :label="$t(col.label)"
+            :width="col.width"
+          >
+            <template slot-scope="scope">
+              <el-tag :type="getRunningTasksStatusType(scope.row)" size="small">
+                {{scope.row[col.name] ? scope.row[col.name].length : '0'}} / {{activeNodeList.length}}
+              </el-tag>
+            </template>
+          </el-table-column>
          <el-table-column
            v-else
            :key="col.name"
@@ -389,21 +412,11 @@ export default {
        sortDirection: null
      },
      types: [],
-      columns: [
-        { name: 'display_name', label: 'Name', width: '160', align: 'left', sortable: true },
-        { name: 'type', label: 'Spider Type', width: '120', sortable: true },
-        { name: 'is_scrapy', label: 'Is Scrapy', width: '80' },
-        { name: 'last_status', label: 'Last Status', width: '120' },
-        { name: 'last_run_ts', label: 'Last Run', width: '140' },
-        { name: 'update_ts', label: 'Update Time', width: '140' },
-        { name: 'create_ts', label: 'Create Time', width: '140' },
-        { name: 'remark', label: 'Remark', width: '140' }
-      ],
      spiderFormRules: {
        name: [{ required: true, message: 'Required Field', trigger: 'change' }]
      },
      fileList: [],
-      spiderType: 'customized',
+      activeTabName: 'customized',
      tourSteps: [
        {
          target: '#tab-customized',
@@ -515,17 +528,18 @@ export default {
        },
        onPreviousStep: (currentStep) => {
          if (currentStep === 7) {
-            this.spiderType = 'customized'
+            this.activeTabName = 'customized'
          }
          this.$utils.tour.prevStep('spider-list-add', currentStep)
        },
        onNextStep: (currentStep) => {
          if (currentStep === 6) {
-            this.spiderType = 'configurable'
+            this.activeTabName = 'configurable'
          }
          this.$utils.tour.nextStep('spider-list-add', currentStep)
        }
-      }
+      },
+      handle: undefined
    }
  },
  computed: {
@@ -539,9 +553,15 @@ export default {
    ...mapGetters('user', [
      'token'
    ]),
+    ...mapState('lang', [
+      'lang'
+    ]),
    ...mapState('project', [
      'projectList'
    ]),
+    ...mapState('node', [
+      'nodeList'
+    ]),
    uploadForm () {
      return {
        name: this.spiderForm.name,
@@ -549,6 +569,25 @@ export default {
        col: this.spiderForm.col,
        cmd: this.spiderForm.cmd
      }
+    },
+    columns () {
+      const columns = []
+      columns.push({ name: 'display_name', label: 'Name', width: '160', align: 'left', sortable: true })
+      columns.push({ name: 'type', label: 'Spider Type', width: '120', sortable: true })
+      columns.push({ name: 'is_long_task', label: 'Is Long Task', width: '80' })
+      columns.push({ name: 'is_scrapy', label: 'Is Scrapy', width: '80' })
+      columns.push({ name: 'last_status', label: 'Last Status', width: '120' })
+      columns.push({ name: 'last_run_ts', label: 'Last Run', width: '140' })
+      columns.push({ name: 'update_ts', label: 'Update Time', width: '140' })
+      columns.push({ name: 'create_ts', label: 'Create Time', width: '140' })
+      columns.push({ name: 'running_tasks', label: 'Running Tasks', width: '120' })
+      columns.push({ name: 'remark', label: 'Remark', width: '140' })
+      return columns
+    },
+    activeNodeList () {
+      return this.nodeList.filter(d => {
+        return d.status === 'online'
+      })
    }
  },
  methods: {
@@ -797,6 +836,15 @@ export default {
        project_id: this.filter.project_id
      }
      await this.$store.dispatch('spider/getSpiderList', params)
+    },
+    getRunningTasksStatusType (row) {
+      if (!row.running_tasks || row.running_tasks.length === 0) {
+        return 'info'
+      } else if (this.activeNodeList && row.running_tasks.length === this.activeNodeList.length) {
+        return 'warning'
+      } else {
+        return 'warning'
+      }
    }
  },
  async created () {
@@ -813,6 +861,11 @@ export default {

    // fetch template list
    await this.$store.dispatch('spider/getTemplateList')
+
+    // periodically fetch spider list
+    this.handle = setInterval(() => {
+      this.getList()
+    }, 15000)
  },
  mounted () {
    const vm = this
@@ -824,6 +877,9 @@ export default {
      this.$tours['spider-list'].start()
      this.$st.sendEv('教程', '开始', 'spider-list')
    }
+  },
+  destroyed () {
+    clearInterval(this.handle)
  }
 }
 </script>