Merge pull request #576 from crawlab-team/develop

Develop

.github/workflows/dockerpush.yml

@@ -0,0 +1,53 @@
+name: Docker
+
+on:
+  push:
+    # Publish `master` as Docker `latest` image.
+    branches:
+      - master
+      - release
+
+    # Publish `v1.2.3` tags as releases.
+    tags:
+      - v*
+
+  # Run tests for any PRs.
+  pull_request:
+
+env:
+  IMAGE_NAME: tikazyq/crawlab
+
+jobs:
+  # Push image to GitHub Package Registry.
+  # See also https://docs.docker.com/docker-hub/builds/
+  push:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build image
+        run: docker build . --file Dockerfile --tag tikazyq/crawlab
+
+      - name: Log into registry
+        run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+
+      - name: Push image
+        run: |
+          IMAGE_ID=tikazyq/$IMAGE_NAME
+
+          # Strip git ref prefix from version
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+          # Strip "v" prefix from tag name
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+          # Use Docker `latest` tag convention
+          [ "$VERSION" == "master" ] && VERSION=latest
+
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+
+          docker tag image $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION

CHANGELOG-zh.md

@@ -1,3 +1,13 @@
+# 0.4.7 (unknown)
+### 功能 / 优化
+- **更好的支持 Scrapy**. 爬虫识别，`settings.py` 配置，日志级别选择，爬虫选择. [#435](https://github.com/crawlab-team/crawlab/issues/435)
+- **Git 同步**. 允许用户将 Git 项目同步到 Crawlab.
+- **长任务支持**. 用户可以添加长任务爬虫，这些爬虫可以跑长期运行的任务. [425](https://github.com/crawlab-team/crawlab/issues/425)
+- **爬虫列表优化**. 分状态任务列数统计，任务列表详情弹出框，图例. [425](https://github.com/crawlab-team/crawlab/issues/425)
+- **版本升级检测**. 检测最新版本，通知用户升级.
+
+### Bug 修复
+
 # 0.4.6 (2020-02-13)
 ### 功能 / 优化
 - **Node.js SDK**. 用户可以将 SDK 应用到他们的 Node.js 爬虫中.

CHANGELOG.md

@@ -1,3 +1,13 @@
+# 0.4.7 (unknown)
+### Features / Enhancement
+- **Better Support for Scrapy**. Spiders identification, `settings.py` configuration, log level selection, spider selection. [#435](https://github.com/crawlab-team/crawlab/issues/435)
+- **Git Sync**. Allow users to sync git projects to Crawlab.
+- **Long Task Support**. Users can add long-task spiders which is supposed to run without finishing. [#425](https://github.com/crawlab-team/crawlab/issues/425)
+- **Spider List Optimization**. Tasks count by status, tasks detail popup, legend. [#425](https://github.com/crawlab-team/crawlab/issues/425)
+- **Upgrade Check**. Check latest version and notifiy users to upgrade.
+
+### Bug Fixes
+
 # 0.4.6 (2020-02-13)
 ### Features / Enhancement
 - **SDK for Node.js**. Users can apply SDK in their Node.js spiders.

backend/entity/version.go

@@ -0,0 +1,23 @@
+package entity
+
+type Release struct {
+	Name        string `json:"name"`
+	Draft       bool   `json:"draft"`
+	PreRelease  bool   `json:"pre_release"`
+	PublishedAt string `json:"published_at"`
+	Body        string `json:"body"`
+}
+
+type ReleaseSlices []Release
+
+func (r ReleaseSlices) Len() int {
+	return len(r)
+}
+
+func (r ReleaseSlices) Less(i, j int) bool {
+	return r[i].PublishedAt < r[j].PublishedAt
+}
+
+func (r ReleaseSlices) Swap(i, j int) {
+	r[i], r[j] = r[j], r[i]
+}

backend/main.go

@@ -133,7 +133,8 @@ func main() {
 			anonymousGroup.PUT("/users", routes.PutUser)      // 添加用户
 			anonymousGroup.GET("/setting", routes.GetSetting) // 获取配置信息
 			// release版本
-			anonymousGroup.GET("/version", routes.GetVersion) // 获取发布的版本
+			anonymousGroup.GET("/version", routes.GetVersion)               // 获取发布的版本
+			anonymousGroup.GET("/releases/latest", routes.GetLatestRelease) // 获取最近发布的版本
 		}
 		authGroup := app.Group("/", middlewares.AuthorizationMiddleware())
 		{

backend/model/spider.go

@@ -55,10 +55,14 @@ type Spider struct {
 	GitSyncFrequency string `json:"git_sync_frequency" bson:"git_sync_frequency"` // Git 同步频率
 	GitSyncError     string `json:"git_sync_error" bson:"git_sync_error"`         // Git 同步错误
 
+	// 长任务
+	IsLongTask bool `json:"is_long_task" bson:"is_long_task"` // 是否为长任务
+
 	// 前端展示
-	LastRunTs  time.Time               `json:"last_run_ts"` // 最后一次执行时间
+	LastRunTs   time.Time               `json:"last_run_ts"`  // 最后一次执行时间
-	LastStatus string                  `json:"last_status"` // 最后执行状态
+	LastStatus  string                  `json:"last_status"`  // 最后执行状态
-	Config     entity.ConfigSpiderData `json:"config"`      // 可配置爬虫配置
+	Config      entity.ConfigSpiderData `json:"config"`       // 可配置爬虫配置
+	LatestTasks []Task                  `json:"latest_tasks"` // 最近任务列表
 
 	// 时间
 	CreateTs time.Time `json:"create_ts" bson:"create_ts"`
@@ -124,6 +128,18 @@ func (spider *Spider) GetLastTask() (Task, error) {
 	return tasks[0], nil
 }
 
+// 爬虫正在运行的任务
+func (spider *Spider) GetLatestTasks(latestN int) (tasks []Task, err error) {
+	tasks, err = GetTaskList(bson.M{"spider_id": spider.Id}, 0, latestN, "-create_ts")
+	if err != nil {
+		return tasks, err
+	}
+	if tasks == nil {
+		return tasks, err
+	}
+	return tasks, nil
+}
+
 // 删除爬虫
 func (spider *Spider) Delete() error {
 	s, c := database.GetCol("spiders")
@@ -157,9 +173,18 @@ func GetSpiderList(filter interface{}, skip int, limit int, sortStr string) ([]S
 			continue
 		}
 
+		// 获取正在运行的爬虫
+		latestTasks, err := spider.GetLatestTasks(50) // TODO: latestN 暂时写死，后面加入数据库
+		if err != nil {
+			log.Errorf(err.Error())
+			debug.PrintStack()
+			continue
+		}
+
 		// 赋值
 		spiders[i].LastRunTs = task.CreateTs
 		spiders[i].LastStatus = task.Status
+		spiders[i].LatestTasks = latestTasks
 	}
 
 	count, _ := c.Find(filter).Count()

backend/routes/spider.go

@@ -35,13 +35,23 @@ func GetSpiderList(c *gin.Context) {
 	sortKey, _ := c.GetQuery("sort_key")
 	sortDirection, _ := c.GetQuery("sort_direction")
 
-	// 筛选
+	// 筛选-名称
 	filter := bson.M{
 		"name": bson.M{"$regex": bson.RegEx{Pattern: keyword, Options: "im"}},
 	}
+
+	// 筛选-类型
 	if t != "" && t != "all" {
 		filter["type"] = t
 	}
+
+	// 筛选-是否为长任务
+	if t == "long-task" {
+		delete(filter, "type")
+		filter["is_long_task"] = true
+	}
+
+	// 筛选-项目
 	if pid == "" {
 		// do nothing
 	} else if pid == constants.ObjectIdNull {

backend/routes/version.go

@@ -0,0 +1,20 @@
+package routes
+
+import (
+	"crawlab/services"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+func GetLatestRelease(c *gin.Context) {
+	latestRelease, err := services.GetLatestRelease()
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    latestRelease,
+	})
+}

backend/services/task.go

@@ -350,10 +350,9 @@ func SaveTaskResultCount(id string) func() {
 func ExecuteTask(id int) {
 	if flag, ok := LockList.Load(id); ok {
 		if flag.(bool) {
-			log.Debugf(GetWorkerPrefix(id) + "正在执行任务...")
+			log.Debugf(GetWorkerPrefix(id) + "running tasks...")
 			return
 		}
-
 	}
 
 	// 上锁
@@ -378,6 +377,7 @@ func ExecuteTask(id int) {
 
 	// 节点队列
 	queueCur := "tasks:node:" + node.Id.Hex()
+
 	// 节点队列任务
 	var msg string
 	if msg, err = database.RedisClient.LPop(queueCur); err != nil {
@@ -387,6 +387,7 @@ func ExecuteTask(id int) {
 		}
 	}
 
+	// 如果没有获取到任务，返回
 	if msg == "" {
 		return
 	}
@@ -504,6 +505,8 @@ func ExecuteTask(id int) {
 		log.Errorf(GetWorkerPrefix(id) + err.Error())
 		return
 	}
+
+	// 统计数据
 	t.Status = constants.StatusFinished                     // 任务状态: 已完成
 	t.FinishTs = time.Now()                                 // 结束时间
 	t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长
@@ -849,6 +852,14 @@ func SendNotifications(u model.User, t model.Task, s model.Spider) {
 	}
 }
 
+func UnlockLongTask(s model.Spider, n model.Node) {
+	if s.IsLongTask {
+		colName := "long-tasks"
+		key := fmt.Sprintf("%s:%s", s.Id.Hex(), n.Id.Hex())
+		_ = database.RedisClient.HDel(colName, key)
+	}
+}
+
 func InitTaskExecutor() error {
 	c := cron.New(cron.WithSeconds())
 	Exec = &Executor{

backend/services/version.go

@@ -0,0 +1,29 @@
+package services
+
+import (
+	"crawlab/entity"
+	"github.com/apex/log"
+	"github.com/imroc/req"
+	"runtime/debug"
+	"sort"
+)
+
+func GetLatestRelease() (release entity.Release, err error) {
+	res, err := req.Get("https://api.github.com/repos/crawlab-team/crawlab/releases")
+	if err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return release, err
+	}
+
+	var releaseDataList entity.ReleaseSlices
+	if err := res.ToJSON(&releaseDataList); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return release, err
+	}
+
+	sort.Sort(releaseDataList)
+
+	return releaseDataList[len(releaseDataList)-1], nil
+}

frontend/src/App.vue

@@ -35,33 +35,6 @@ export default {
   },
   methods: {},
   async mounted () {
-    // window.setUseStats = (value) => {
-    //   document.querySelector('.el-message__closeBtn').click()
-    //   if (value === 1) {
-    //     this.$st.sendPv('/allow_stats')
-    //     this.$st.sendEv('全局', '允许/禁止统计', '允许')
-    //   } else {
-    //     this.$st.sendPv('/disallow_stats')
-    //     this.$st.sendEv('全局', '允许/禁止统计', '禁止')
-    //   }
-    //   localStorage.setItem('useStats', value)
-    // }
-
-    // first-time user
-    // if (this.useStats === undefined || this.useStats === null) {
-    // this.$message({
-    //   type: 'info',
-    //   dangerouslyUseHTMLString: true,
-    //   showClose: true,
-    //   duration: 0,
-    //   message: '<p>' + this.$t('Do you allow us to collect some statistics to improve Crawlab?') + '</p>' +
-    //     '<div style="text-align: center;margin-top: 10px;">' +
-    //     '<button class="message-btn" onclick="setUseStats(1)">' + this.$t('Yes') + '</button>' +
-    //     '<button class="message-btn" onclick="setUseStats(0)">' + this.$t('No') + '</button>' +
-    //     '</div>'
-    // })
-    // }
-
     // set uid if first visit
     if (this.uid === undefined || this.uid === null) {
       localStorage.setItem('uid', this.$utils.encrypt.UUID())
@@ -71,6 +44,13 @@ export default {
     if (this.sid === undefined || this.sid === null) {
       sessionStorage.setItem('sid', this.$utils.encrypt.UUID())
     }
+
+    // get latest version
+    await this.$store.dispatch('version/getLatestRelease')
+
+    // remove loading-placeholder
+    const elLoading = document.querySelector('#loading-placeholder')
+    elLoading.remove()
   }
 }
 </script>

frontend/src/components/Common/CrawlConfirmDialog.vue

@@ -68,7 +68,7 @@
             <span style="margin-left: 5px">我已阅读并同意 <a href="javascript:"
                                                       @click="onClickDisclaimer">《免责声明》</a> 所有内容</span>
           </div>
-          <div>
+          <div v-if="!spiderForm.is_long_task">
             <el-checkbox v-model="isRedirect"/>
             <span style="margin-left: 5px">跳转到任务详情页</span>
           </div>
@@ -149,10 +149,15 @@ export default {
       this.$refs['form'].validate(async valid => {
         if (!valid) return
 
+        let param = this.form.param
+        if (this.spiderForm.type === 'customized' && this.spiderForm.is_scrapy) {
+          param = `${this.form.spider} --loglevel=${this.form.scrapy_log_level} ${this.form.param}`
+        }
+
         const res = await this.$store.dispatch('spider/crawlSpider', {
           spiderId: this.spiderId,
           nodeIds: this.form.nodeIds,
-          param: `${this.form.spider} --loglevel=${this.form.scrapy_log_level} ${this.form.param}`,
+          param,
           runType: this.form.runType
         })
 
@@ -163,10 +168,12 @@ export default {
         this.$emit('close')
         this.$st.sendEv('爬虫确认', '确认运行', this.form.runType)
 
-        if (this.isRedirect) {
+        if (this.isRedirect && !this.spiderForm.is_long_task) {
           this.$router.push('/tasks/' + id)
           this.$st.sendEv('爬虫确认', '跳转到任务详情')
         }
+
+        this.$emit('confirm')
       })
     },
     onClickDisclaimer () {

frontend/src/components/InfoView/SpiderInfoView.vue

@@ -76,6 +76,14 @@
               />
             </el-form-item>
           </el-col>
+          <el-col :span="6">
+            <el-form-item v-if="!isView" :label="$t('Is Long Task')" prop="is_long_task">
+              <el-switch
+                v-model="spiderForm.is_long_task"
+                active-color="#13ce66"
+              />
+            </el-form-item>
+          </el-col>
         </el-row>
       </el-form>
     </el-row>

frontend/src/components/Status/StatusLegend.vue

@@ -0,0 +1,40 @@
+<template>
+  <div class="legend">
+    <el-tag type="primary" size="small">
+      <i class="el-icon-loading"></i>
+      {{$t('Pending')}}
+    </el-tag>
+    <el-tag type="warning" size="small">
+      <i class="el-icon-loading"></i>
+      {{$t('Running')}}
+    </el-tag>
+    <el-tag type="success" size="small">
+      <i class="el-icon-check"></i>
+      {{$t('Finished')}}
+    </el-tag>
+    <el-tag type="danger" size="small">
+      <i class="el-icon-error"></i>
+      {{$t('Error')}}
+    </el-tag>
+    <el-tag type="info" size="small">
+      <i class="el-icon-video-pause"></i>
+      {{$t('Cancelled')}}
+    </el-tag>
+    <el-tag type="danger" size="small">
+      <i class="el-icon-warning"></i>
+      {{$t('Abnormal')}}
+    </el-tag>
+  </div>
+</template>
+
+<script>
+export default {
+  name: 'StatusLegend'
+}
+</script>
+
+<style scoped>
+  .legend .el-tag {
+    margin-right: 5px;
+  }
+</style>

frontend/src/components/Status/StatusTag.vue

@@ -21,7 +21,8 @@ export default {
         running: { label: 'Running', type: 'warning' },
         finished: { label: 'Finished', type: 'success' },
         error: { label: 'Error', type: 'danger' },
-        cancelled: { label: 'Cancelled', type: 'info' }
+        cancelled: { label: 'Cancelled', type: 'info' },
+        abnormal: { label: 'Abnormal', type: 'danger' }
       }
     }
   },
@@ -43,6 +44,8 @@ export default {
     icon () {
       if (this.status === 'finished') {
         return 'el-icon-check'
+      } else if (this.status === 'pending') {
+        return 'el-icon-loading'
       } else if (this.status === 'running') {
         return 'el-icon-loading'
       } else if (this.status === 'error') {
@@ -50,9 +53,10 @@ export default {
       } else if (this.status === 'cancelled') {
         return 'el-icon-video-pause'
       } else if (this.status === 'abnormal') {
+        return 'el-icon-warning'
+      } else {
         return 'el-icon-question'
       }
-      return ''
     }
   }
 }

frontend/src/i18n/zh.js

@@ -39,6 +39,7 @@ export default {
   Error: '错误',
   NA: '未知',
   Cancelled: '已取消',
+  Abnormal: '异常',
 
   // 操作
   Add: '添加',
@@ -78,6 +79,8 @@ export default {
   'Sync Frequency': '同步频率',
   'Reset': '重置',
   'Copy': '复制',
+  'Upgrade': '版本升级',
+  'Ok': '确定',
 
   // 主页
   'Total Tasks': '总任务数',
@@ -210,6 +213,10 @@ export default {
   'Git Password': 'Git 密码',
   'Has Credential': '需要验证',
   'SSH Public Key': 'SSH 公钥',
+  'Is Long Task': '是否为长任务',
+  'Long Task': '长任务',
+  'Running Task Count': '运行中的任务数',
+  'Running Tasks': '运行中的任务',
 
   // 爬虫列表
   'Name': '名称',
@@ -390,6 +397,9 @@ export default {
   'New directory name': '新目录名称',
   'Enter new file name': '输入新文件名称',
   'New file name': '新文件名称',
+  'Release Note': '发布记录',
+  'How to Upgrade': '升级方式',
+  'Release': '发布',
 
   // 登录
   'Sign in': '登录',

frontend/src/store/modules/task.js

@@ -174,10 +174,13 @@ const actions = {
     link.remove()
   },
   cancelTask ({ state, dispatch }, id) {
-    return request.post(`/tasks/${id}/cancel`)
+    return new Promise(resolve => {
-      .then(() => {
+      request.post(`/tasks/${id}/cancel`)
-        dispatch('getTaskData', id)
+        .then(res => {
-      })
+          dispatch('getTaskData', id)
+          resolve(res)
+        })
+    })
   }
 }
 

frontend/src/store/modules/version.js

@@ -1,5 +1,11 @@
+import request from '../../api/request'
+
 const state = {
-  version: ''
+  version: '',
+  latestRelease: {
+    name: '',
+    body: ''
+  }
 }
 
 const getters = {}
@@ -7,10 +13,20 @@ const getters = {}
 const mutations = {
   SET_VERSION: (state, value) => {
     state.version = value
+  },
+  SET_LATEST_RELEASE: (state, value) => {
+    state.latestRelease = value
   }
 }
 
-const actions = {}
+const actions = {
+  async getLatestRelease ({ commit }) {
+    const res = await request.get('/releases/latest')
+    if (!res.data.error) {
+      commit('SET_LATEST_RELEASE', res.data.data)
+    }
+  }
+}
 
 export default {
   namespaced: true,

frontend/src/utils/scrapy.js

@@ -198,6 +198,15 @@ export default {
     'TELNETCONSOLE_HOST',
     'TELNETCONSOLE_PASSWORD',
     'TELNETCONSOLE_PORT',
-    'TELNETCONSOLE_USERNAME'
+    'TELNETCONSOLE_USERNAME',
+    'REDIS_ITEMS_KEY',
+    'REDIS_ITEMS_SERIALIZER',
+    'REDIS_HOST',
+    'REDIS_PORT',
+    'REDIS_URL',
+    'REDIS_PARAMS',
+    'REDIS_START_URLS_AS_SET',
+    'REDIS_START_URLS_KEY',
+    'REDIS_ENCODING'
   ]
 }

frontend/src/views/layout/components/Navbar.vue

@@ -1,5 +1,27 @@
 <template>
   <div class="navbar">
+    <el-dialog
+      :visible.sync="isLatestReleaseNoteVisible"
+      :title="$t('Release') + ` ${latestRelease.name}`"
+    >
+      <el-tabs v-model="activeTabName">
+        <el-tab-pane :label="$t('Release Note')" name="release-note">
+          <div class="content markdown-body" v-html="latestReleaseNoteHtml">
+          </div>
+          <template slot="footer">
+            <el-button type="primary" size="small" @click="isLatestReleaseNoteVisible = false">{{$t('Ok')}}</el-button>
+          </template>
+        </el-tab-pane>
+        <el-tab-pane :label="$t('How to Upgrade')" name="how-to-upgrade">
+          <div class="content markdown-body" v-html="howToUpgradeHtml">
+          </div>
+        </el-tab-pane>
+      </el-tabs>
+      <template slot="footer">
+        <el-button type="primary" size="small" @click="isLatestReleaseNoteVisible = false">{{$t('Ok')}}</el-button>
+      </template>
+    </el-dialog>
+
     <hamburger :toggle-click="toggleSideBar" :is-active="sidebar.opened" class="hamburger-container"/>
     <breadcrumb class="breadcrumb"/>
     <el-dropdown class="avatar-container right" trigger="click">
@@ -33,6 +55,12 @@
         <span style="margin-left: 5px;">{{$t('Documentation')}}</span>
       </a>
     </div>
+    <div v-if="isUpgradable" class="upgrade right" @click="onClickUpgrade">
+      <font-awesome-icon :icon="['fas', 'arrow-up']"/>
+      <el-badge is-dot>
+        <span style="margin-left: 5px;">{{$t('Upgrade')}}</span>
+      </el-badge>
+    </div>
     <div class="github right">
       <!-- Place this tag where you want the button to render. -->
       <github-button
@@ -50,10 +78,15 @@
 </template>
 
 <script>
-import { mapGetters } from 'vuex'
+import {
+  mapState,
+  mapGetters
+} from 'vuex'
 import Breadcrumb from '@/components/Breadcrumb'
 import Hamburger from '@/components/Hamburger'
 import GithubButton from 'vue-github-button'
+import showdown from 'showdown'
+import 'github-markdown-css/github-markdown.css'
 
 export default {
   components: {
@@ -61,7 +94,58 @@ export default {
     Hamburger,
     GithubButton
   },
+  data () {
+    const converter = new showdown.Converter()
+    return {
+      isLatestReleaseNoteVisible: false,
+      converter,
+      activeTabName: 'release-note',
+      howToUpgradeHtmlZh: `
+### Docker 部署
+\`\`\`bash
+# 拉取最新镜像
+docker pull tikazyq/crawlab:latest
+
+# 删除容器
+docker-compose down | true
+
+# 启动容器
+docker-compose up -d
+\`\`\`
+
+### 直接部署
+
+1. 拉取最新 Github 代码
+2. 重新构建前后端应用
+3. 启动前后端应用
+`,
+      howToUpgradeHtmlEn: `
+### Docker Deployment
+\`\`\`bash
+# pull the latest image
+docker pull tikazyq/crawlab:latest
+
+# delete containers
+docker-compose down | true
+
+# start containers
+docker-compose up -d
+\`\`\`
+
+### Direct Deployment
+1. Pull the latest Github repository
+2. Build frontend and backend applications
+3. Start frontend and backend applications
+`
+    }
+  },
   computed: {
+    ...mapState('version', [
+      'latestRelease'
+    ]),
+    ...mapState('lang', [
+      'lang'
+    ]),
     ...mapGetters([
       'sidebar',
       'avatar'
@@ -70,6 +154,39 @@ export default {
       if (!this.$store.getters['user/userInfo']) return this.$t('User')
       if (!this.$store.getters['user/userInfo'].username) return this.$t('User')
       return this.$store.getters['user/userInfo'].username
+    },
+    isUpgradable () {
+      if (!this.latestRelease.name) return false
+
+      const currentVersion = sessionStorage.getItem('v')
+      const latestVersion = this.latestRelease.name.replace('v', '')
+
+      if (!latestVersion || !currentVersion) return false
+
+      const currentVersionList = currentVersion.split('.')
+      const latestVersionList = latestVersion.split('.')
+      for (let i = 0; i < currentVersionList.length; i++) {
+        let nc = Number(currentVersionList[i])
+        let nl = Number(latestVersionList[i])
+        if (isNaN(nl)) nl = 0
+        if (nc < nl) return true
+      }
+      return false
+    },
+    latestReleaseNoteHtml () {
+      if (!this.latestRelease.body) return ''
+      const body = this.latestRelease.body
+      return this.converter.makeHtml(body)
+    },
+    howToUpgradeHtml () {
+      if (this.lang === 'zh') {
+        console.log(this.howToUpgradeHtmlZh)
+        return this.converter.makeHtml(this.howToUpgradeHtmlZh)
+      } else if (this.lang === 'en') {
+        return this.converter.makeHtml(this.howToUpgradeHtmlEn)
+      } else {
+        return ''
+      }
     }
   },
   methods: {
@@ -86,6 +203,10 @@ export default {
       this.$store.commit('lang/SET_LANG', lang)
 
       this.$st.sendEv('全局', '切换中英文', lang)
+    },
+    onClickUpgrade () {
+      this.isLatestReleaseNoteVisible = true
+      this.$st.sendEv('全局', '点击版本升级')
     }
   }
 }
@@ -132,6 +253,7 @@ export default {
       margin-right: 35px;
       color: #606266;
       font-size: 14px;
+
       .span {
         margin-left: 5px;
       }
@@ -143,8 +265,26 @@ export default {
       margin-top: -10px;
     }
 
+    .upgrade {
+      margin-top: 12.5px;
+      line-height: 25px;
+      cursor: pointer;
+      font-size: 14px;
+      color: #606266;
+      margin-right: 35px;
+
+      .span {
+        margin-left: 5px;
+      }
+    }
+
     .right {
       float: right
     }
   }
 </style>
+<style scoped>
+  .navbar >>> .el-dialog__body {
+    padding-top: 0;
+  }
+</style>

frontend/src/views/spider/SpiderList.vue

@@ -20,7 +20,7 @@
                width="40%"
                :visible.sync="addDialogVisible"
                :before-close="onAddDialogClose">
-      <el-tabs :active-name="spiderType">
+      <el-tabs :active-name="activeTabName">
         <!-- customized -->
         <el-tab-pane name="customized" :label="$t('Customized')">
           <el-form :model="spiderForm" ref="addCustomizedForm" inline-message label-width="120px">
@@ -70,6 +70,12 @@
                 active-color="#13ce66"
               />
             </el-form-item>
+            <el-form-item :label="$t('Is Long Task')" prop="is_long_task">
+              <el-switch
+                v-model="spiderForm.is_long_task"
+                active-color="#13ce66"
+              />
+            </el-form-item>
           </el-form>
           <el-alert
             type="warning"
@@ -82,6 +88,16 @@
               <i class="fa fa-exclamation-triangle"></i> {{$t('NOTE: When uploading a zip file, please zip your' +
               ' spider files from the ROOT DIRECTORY.')}}
             </p>
+            <p>
+              <template v-if="lang === 'en'">
+                You can also upload spiders using <a href="https://docs.crawlab.cn/SDK/CLI.html" target="_blank"
+                                                     style="color: #409eff;font-weight: bolder">CLI Tool</a>.
+              </template>
+              <template v-else-if="lang === 'zh'">
+                您也可以利用 <a href="https://docs.crawlab.cn/SDK/CLI.html" target="_blank"
+                          style="color: #409eff;font-weight: bolder">CLI 工具</a> 上传爬虫。
+              </template>
+            </p>
           </el-alert>
           <div class="actions">
             <el-button size="small" type="primary" @click="onAddCustomized">{{$t('Add')}}</el-button>
@@ -133,11 +149,122 @@
     </el-dialog>
     <!--./add dialog-->
 
+    <!--running tasks dialog-->
+    <el-dialog
+      :visible.sync="isRunningTasksDialogVisible"
+      :title="`${$t('Latest Tasks')} (${$t('Spider')}: ${activeSpider ? activeSpider.name : ''})`"
+      width="920px"
+    >
+      <el-tabs v-model="activeSpiderTaskStatus">
+        <el-tab-pane name="pending" :label="$t('Pending')"/>
+        <el-tab-pane name="running" :label="$t('Running')"/>
+        <el-tab-pane name="finished" :label="$t('Finished')"/>
+        <el-tab-pane name="error" :label="$t('Error')"/>
+        <el-tab-pane name="cancelled" :label="$t('Cancelled')"/>
+        <el-tab-pane name="abnormal" :label="$t('Abnormal')"/>
+      </el-tabs>
+      <el-table
+        :data="activeNodeList"
+        class="table"
+        :header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
+        border
+        default-expand-all
+      >
+        <el-table-column type="expand">
+          <template slot-scope="scope">
+            <h4 style="margin: 5px 10px">{{$t('Tasks')}}</h4>
+            <el-table
+              :data="getTasksByNode(scope.row)"
+              class="table"
+              border
+              style="margin: 5px 10px"
+              max-height="240px"
+              @row-click="onViewTask"
+            >
+              <el-table-column
+                :label="$t('Create Time')"
+                prop="create_ts"
+                width="140px"
+              />
+              <el-table-column
+                :label="$t('Start Time')"
+                prop="start_ts"
+                width="140px"
+              />
+              <el-table-column
+                :label="$t('Finish Time')"
+                prop="finish_ts"
+                width="140px"
+              />
+              <el-table-column
+                :label="$t('Parameters')"
+                prop="param"
+                width="120px"
+              />
+              <el-table-column
+                :label="$t('Status')"
+                width="120px"
+              >
+                <template slot-scope="scope">
+                  <status-tag :status="scope.row.status"/>
+                </template>
+              </el-table-column>
+              <el-table-column
+                :label="$t('Results Count')"
+                prop="result_count"
+                width="80px"
+              />
+              <el-table-column
+                :label="$t('Action')"
+                width="auto"
+              >
+                <template slot-scope="scope">
+                  <el-button
+                    v-if="['pending', 'running'].includes(scope.row.status)"
+                    type="danger"
+                    size="mini"
+                    icon="el-icon-video-pause"
+                    @click="onStop(scope.row, $event)"
+                  />
+                </template>
+              </el-table-column>
+            </el-table>
+          </template>
+        </el-table-column>
+        <el-table-column
+          :label="$t('Node')"
+          width="150px"
+          prop="name"
+        />
+        <el-table-column
+          :label="$t('Status')"
+          width="120px"
+          prop="status"
+        >
+          <template slot-scope="scope">
+            <el-tag type="info" v-if="scope.row.status === 'offline'">{{$t('Offline')}}</el-tag>
+            <el-tag type="success" v-else-if="scope.row.status === 'online'">{{$t('Online')}}</el-tag>
+            <el-tag type="danger" v-else>{{$t('Unavailable')}}</el-tag>
+          </template>
+        </el-table-column>
+        <el-table-column
+          :label="$t('Description')"
+          width="auto"
+          prop="description"
+        />
+      </el-table>
+      <template slot="footer">
+        <el-button type="primary" size="small" @click="isRunningTasksDialogVisible = false">{{$t('Ok')}}</el-button>
+      </template>
+    </el-dialog>
+    <!--./running tasks dialog-->
+
     <!--crawl confirm dialog-->
     <crawl-confirm-dialog
       :visible="crawlConfirmDialogVisible"
       :spider-id="activeSpiderId"
       @close="crawlConfirmDialogVisible = false"
+      @confirm="onCrawlConfirm"
     />
     <!--./crawl confirm dialog-->
 
@@ -211,10 +338,15 @@
       <el-tabs v-model="filter.type" @tab-click="onClickTab" class="tabs">
         <el-tab-pane :label="$t('All')" name="all" class="all"></el-tab-pane>
         <el-tab-pane :label="$t('Customized')" name="customized" class="customized"></el-tab-pane>
-        <el-tab-pane :label="$t('Configurable')" name="configurable" class="configuable"></el-tab-pane>
+        <el-tab-pane :label="$t('Configurable')" name="configurable" class="configurable"></el-tab-pane>
+        <el-tab-pane :label="$t('Long Task')" name="long-task" class="long-task"></el-tab-pane>
       </el-tabs>
       <!--./tabs-->
 
+      <!--legend-->
+      <status-legend/>
+      <!--./legend-->
+
       <!--table list-->
       <el-table
         :data="spiderList"
@@ -288,7 +420,7 @@
             </template>
           </el-table-column>
           <el-table-column
-            v-else-if="col.name === 'is_scrapy'"
+            v-else-if="['is_scrapy', 'is_long_task'].includes(col.name)"
             :key="col.name"
             :label="$t(col.label)"
             align="left"
@@ -298,12 +430,71 @@
             <template slot-scope="scope">
               <el-switch
                 v-if="scope.row.type === 'customized'"
-                v-model="scope.row.is_scrapy"
+                v-model="scope.row[col.name]"
                 active-color="#13ce66"
                 disabled
               />
             </template>
           </el-table-column>
+          <el-table-column
+            v-else-if="col.name === 'latest_tasks'"
+            :key="col.name"
+            :label="$t(col.label)"
+            :width="col.width"
+            :align="col.align"
+            class-name="latest-tasks"
+          >
+            <template slot-scope="scope">
+              <el-tag
+                v-if="getTaskCountByStatus(scope.row, 'pending') > 0"
+                type="primary"
+                size="small"
+              >
+                <i class="el-icon-loading"></i>
+                {{getTaskCountByStatus(scope.row, 'pending')}}
+              </el-tag>
+              <el-tag
+                v-if="getTaskCountByStatus(scope.row, 'running') > 0"
+                type="warning"
+                size="small"
+              >
+                <i class="el-icon-loading"></i>
+                {{getTaskCountByStatus(scope.row, 'running')}}
+              </el-tag>
+              <el-tag
+                v-if="getTaskCountByStatus(scope.row, 'finished') > 0"
+                type="success"
+                size="small"
+              >
+                <i class="el-icon-check"></i>
+                {{getTaskCountByStatus(scope.row, 'finished')}}
+              </el-tag>
+              <el-tag
+                v-if="getTaskCountByStatus(scope.row, 'error') > 0"
+                type="danger"
+                size="small"
+              >
+                <i class="el-icon-error"></i>
+                {{getTaskCountByStatus(scope.row, 'error')}}
+              </el-tag>
+              <el-tag
+                v-if="getTaskCountByStatus(scope.row, 'cancelled') > 0"
+                type="info"
+                size="small"
+              >
+                <i class="el-icon-video-pause"></i>
+                {{getTaskCountByStatus(scope.row, 'cancelled')}}
+              </el-tag>
+              <el-tag
+                v-if="getTaskCountByStatus(scope.row, 'abnormal') > 0"
+                type="danger"
+                size="small"
+              >
+                <i class="el-icon-warning"></i>
+                {{getTaskCountByStatus(scope.row, 'abnormal')}}
+              </el-tag>
+            </template>
+          </el-table-column>
           <el-table-column
             v-else
             :key="col.name"
@@ -315,7 +506,7 @@
           >
           </el-table-column>
         </template>
-        <el-table-column :label="$t('Action')" align="left" fixed="right">
+        <el-table-column :label="$t('Action')" align="left" fixed="right" min-width="170px">
           <template slot-scope="scope">
             <el-tooltip :content="$t('View')" placement="top">
               <el-button type="primary" icon="el-icon-search" size="mini"
@@ -332,6 +523,14 @@
             <el-tooltip v-else :content="$t('Run')" placement="top">
               <el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row, $event)"></el-button>
             </el-tooltip>
+            <el-tooltip :content="$t('Latest Tasks')" placement="top">
+              <el-button
+                type="warning"
+                icon="fa fa-tasks"
+                size="mini"
+                @click="onViewRunningTasks(scope.row, $event)"
+              />
+            </el-tooltip>
           </template>
         </el-table-column>
       </el-table>
@@ -359,10 +558,12 @@ import {
 import dayjs from 'dayjs'
 import CrawlConfirmDialog from '../../components/Common/CrawlConfirmDialog'
 import StatusTag from '../../components/Status/StatusTag'
+import StatusLegend from '../../components/Status/StatusLegend'
 
 export default {
   name: 'SpiderList',
   components: {
+    StatusLegend,
     CrawlConfirmDialog,
     StatusTag
   },
@@ -378,7 +579,9 @@ export default {
       dialogVisible: false,
       addDialogVisible: false,
       crawlConfirmDialogVisible: false,
+      isRunningTasksDialogVisible: false,
       activeSpiderId: undefined,
+      activeSpider: undefined,
       filter: {
         project_id: '',
         keyword: '',
@@ -389,21 +592,11 @@ export default {
         sortDirection: null
       },
       types: [],
-      columns: [
-        { name: 'display_name', label: 'Name', width: '160', align: 'left', sortable: true },
-        { name: 'type', label: 'Spider Type', width: '120', sortable: true },
-        { name: 'is_scrapy', label: 'Is Scrapy', width: '80' },
-        { name: 'last_status', label: 'Last Status', width: '120' },
-        { name: 'last_run_ts', label: 'Last Run', width: '140' },
-        { name: 'update_ts', label: 'Update Time', width: '140' },
-        { name: 'create_ts', label: 'Create Time', width: '140' },
-        { name: 'remark', label: 'Remark', width: '140' }
-      ],
       spiderFormRules: {
         name: [{ required: true, message: 'Required Field', trigger: 'change' }]
       },
       fileList: [],
-      spiderType: 'customized',
+      activeTabName: 'customized',
       tourSteps: [
         {
           target: '#tab-customized',
@@ -515,17 +708,19 @@ export default {
         },
         onPreviousStep: (currentStep) => {
           if (currentStep === 7) {
-            this.spiderType = 'customized'
+            this.activeTabName = 'customized'
           }
           this.$utils.tour.prevStep('spider-list-add', currentStep)
         },
         onNextStep: (currentStep) => {
           if (currentStep === 6) {
-            this.spiderType = 'configurable'
+            this.activeTabName = 'configurable'
           }
           this.$utils.tour.nextStep('spider-list-add', currentStep)
         }
-      }
+      },
+      handle: undefined,
+      activeSpiderTaskStatus: 'running'
     }
   },
   computed: {
@@ -539,9 +734,15 @@ export default {
     ...mapGetters('user', [
       'token'
     ]),
+    ...mapState('lang', [
+      'lang'
+    ]),
     ...mapState('project', [
       'projectList'
     ]),
+    ...mapState('node', [
+      'nodeList'
+    ]),
     uploadForm () {
       return {
         name: this.spiderForm.name,
@@ -549,6 +750,25 @@ export default {
         col: this.spiderForm.col,
         cmd: this.spiderForm.cmd
       }
+    },
+    columns () {
+      const columns = []
+      columns.push({ name: 'display_name', label: 'Name', width: '160', align: 'left', sortable: true })
+      columns.push({ name: 'type', label: 'Spider Type', width: '120', sortable: true })
+      columns.push({ name: 'is_long_task', label: 'Is Long Task', width: '80' })
+      columns.push({ name: 'is_scrapy', label: 'Is Scrapy', width: '80' })
+      columns.push({ name: 'latest_tasks', label: 'Latest Tasks', width: '180' })
+      columns.push({ name: 'last_status', label: 'Last Status', width: '120' })
+      columns.push({ name: 'last_run_ts', label: 'Last Run', width: '140' })
+      columns.push({ name: 'update_ts', label: 'Update Time', width: '140' })
+      columns.push({ name: 'create_ts', label: 'Create Time', width: '140' })
+      columns.push({ name: 'remark', label: 'Remark', width: '140' })
+      return columns
+    },
+    activeNodeList () {
+      return this.nodeList.filter(d => {
+        return d.status === 'online'
+      })
     }
   },
   methods: {
@@ -647,12 +867,6 @@ export default {
     onAddDialogClose () {
       this.addDialogVisible = false
     },
-    onAddCustomizedDialogClose () {
-      this.addCustomizedDialogVisible = false
-    },
-    onAddConfigurableDialogClose () {
-      this.addConfigurableDialogVisible = false
-    },
     onEdit (row) {
       this.isEditMode = true
       this.$store.commit('spider/SET_SPIDER_FORM', row)
@@ -680,6 +894,11 @@ export default {
       this.activeSpiderId = row._id
       this.$st.sendEv('爬虫列表', '点击运行')
     },
+    onCrawlConfirm () {
+      setTimeout(() => {
+        this.getList()
+      }, 1000)
+    },
     onView (row, ev) {
       ev.stopPropagation()
       this.$router.push('/spiders/' + row._id)
@@ -729,21 +948,6 @@ export default {
         callback(data)
       })
     },
-    onAddConfigurableSiteSelect (item) {
-      this.spiderForm.site = item._id
-    },
-    onAddConfigurableSpider () {
-      this.$refs['addConfigurableForm'].validate(res => {
-        if (res) {
-          this.addConfigurableLoading = true
-          this.$store.dispatch('spider/addSpider')
-            .finally(() => {
-              this.addConfigurableLoading = false
-              this.addConfigurableDialogVisible = false
-            })
-        }
-      })
-    },
     onUploadSuccess (res) {
       // clear fileList
       this.fileList = []
@@ -797,6 +1001,57 @@ export default {
         project_id: this.filter.project_id
       }
       await this.$store.dispatch('spider/getSpiderList', params)
+
+      // 更新当前爬虫（任务列表）
+      this.updateActiveSpider()
+    },
+    getTasksByStatus (row, status) {
+      if (!row.latest_tasks) return []
+      return row.latest_tasks.filter(d => d.status === status)
+    },
+    getTaskCountByStatus (row, status) {
+      return this.getTasksByStatus(row, status).length
+    },
+    updateActiveSpider () {
+      if (this.activeSpider) {
+        for (let i = 0; i < this.spiderList.length; i++) {
+          const spider = this.spiderList[i]
+          if (this.activeSpider._id === spider._id) {
+            this.activeSpider = spider
+          }
+        }
+      }
+    },
+    onViewRunningTasks (row, ev) {
+      ev.stopPropagation()
+      this.activeSpider = row
+      this.isRunningTasksDialogVisible = true
+    },
+    getTasksByNode (row) {
+      if (!this.activeSpider.latest_tasks) {
+        return []
+      }
+      return this.activeSpider.latest_tasks
+        .filter(d => d.node_id === row._id && d.status === this.activeSpiderTaskStatus)
+        .map(d => {
+          d = JSON.parse(JSON.stringify(d))
+          d.create_ts = d.create_ts.match('^0001') ? 'NA' : dayjs(d.create_ts).format('YYYY-MM-DD HH:mm:ss')
+          d.start_ts = d.start_ts.match('^0001') ? 'NA' : dayjs(d.start_ts).format('YYYY-MM-DD HH:mm:ss')
+          d.finish_ts = d.finish_ts.match('^0001') ? 'NA' : dayjs(d.finish_ts).format('YYYY-MM-DD HH:mm:ss')
+          return d
+        })
+    },
+    onViewTask (row) {
+      this.$router.push(`/tasks/${row._id}`)
+      this.$st.sendEv('爬虫列表', '任务列表', '查看任务')
+    },
+    async onStop (row, ev) {
+      ev.stopPropagation()
+      const res = await this.$store.dispatch('task/cancelTask', row._id)
+      if (!res.data.error) {
+        this.$message.success(`Task "${row._id}" has been sent signal to stop`)
+        this.getList()
+      }
     }
   },
   async created () {
@@ -808,11 +1063,19 @@ export default {
       this.filter.project_id = this.$route.params.project_id
     }
 
+    // fetch node list
+    await this.$store.dispatch('node/getNodeList')
+
     // fetch spider list
     await this.getList()
 
     // fetch template list
     await this.$store.dispatch('spider/getTemplateList')
+
+    // periodically fetch spider list
+    this.handle = setInterval(() => {
+      this.getList()
+    }, 15000)
   },
   mounted () {
     const vm = this
@@ -824,6 +1087,9 @@ export default {
       this.$tours['spider-list'].start()
       this.$st.sendEv('教程', '开始', 'spider-list')
     }
+  },
+  destroyed () {
+    clearInterval(this.handle)
   }
 }
 </script>
@@ -913,4 +1179,8 @@ export default {
   .actions {
     text-align: right;
   }
+
+  .el-table >>> .latest-tasks .el-tag {
+    margin: 3px 3px 0 0;
+  }
 </style>

frontend/src/views/task/TaskList.vue

@@ -45,6 +45,10 @@
       </div>
       <!--./filter-->
 
+      <!--legend-->
+      <status-legend/>
+      <!--./legend-->
+
       <!--table list-->
       <el-table :data="filteredTableData"
                 ref="table"
@@ -169,10 +173,11 @@ import {
 } from 'vuex'
 import dayjs from 'dayjs'
 import StatusTag from '../../components/Status/StatusTag'
+import StatusLegend from '../../components/Status/StatusLegend'
 
 export default {
   name: 'TaskList',
-  components: { StatusTag },
+  components: { StatusLegend, StatusTag },
   data () {
     return {
       // setInterval handle