优化爬虫列表页面

2026-01-21 17:21:09 +01:00 · 2020-02-20 11:52:28 +08:00
parent 7de40f415b
commit e3883faf1c
9 changed files with 344 additions and 48 deletions
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -55,10 +55,14 @@ type Spider struct {
 	GitSyncFrequency string `json:"git_sync_frequency" bson:"git_sync_frequency"` // Git 同步频率
 	GitSyncError     string `json:"git_sync_error" bson:"git_sync_error"`         // Git 同步错误

+	// 长任务
+	IsLongTask bool `json:"is_long_task" bson:"is_long_task"` // 是否为长任务
+
 	// 前端展示
-	LastRunTs  time.Time               `json:"last_run_ts"` // 最后一次执行时间
-	LastStatus string                  `json:"last_status"` // 最后执行状态
-	Config     entity.ConfigSpiderData `json:"config"`      // 可配置爬虫配置
+	LastRunTs   time.Time               `json:"last_run_ts"`  // 最后一次执行时间
+	LastStatus  string                  `json:"last_status"`  // 最后执行状态
+	Config      entity.ConfigSpiderData `json:"config"`       // 可配置爬虫配置
+	LatestTasks []Task                  `json:"latest_tasks"` // 最近任务列表

 	// 时间
 	CreateTs time.Time `json:"create_ts" bson:"create_ts"`
@@ -124,6 +128,18 @@ func (spider *Spider) GetLastTask() (Task, error) {
 	return tasks[0], nil
 }

+// 爬虫正在运行的任务
+func (spider *Spider) GetLatestTasks(latestN int) (tasks []Task, err error) {
+	tasks, err = GetTaskList(bson.M{"spider_id": spider.Id}, 0, latestN, "-create_ts")
+	if err != nil {
+		return tasks, err
+	}
+	if tasks == nil {
+		return tasks, err
+	}
+	return tasks, nil
+}
+
 // 删除爬虫
 func (spider *Spider) Delete() error {
 	s, c := database.GetCol("spiders")
@@ -157,9 +173,18 @@ func GetSpiderList(filter interface{}, skip int, limit int, sortStr string) ([]S
 			continue
 		}

+		// 获取正在运行的爬虫
+		latestTasks, err := spider.GetLatestTasks(50) // TODO: latestN 暂时写死，后面加入数据库
+		if err != nil {
+			log.Errorf(err.Error())
+			debug.PrintStack()
+			continue
+		}
+
 		// 赋值
 		spiders[i].LastRunTs = task.CreateTs
 		spiders[i].LastStatus = task.Status
+		spiders[i].LatestTasks = latestTasks
 	}

 	count, _ := c.Find(filter).Count()
--- a/backend/routes/spider.go
+++ b/backend/routes/spider.go
@@ -35,13 +35,23 @@ func GetSpiderList(c *gin.Context) {
 	sortKey, _ := c.GetQuery("sort_key")
 	sortDirection, _ := c.GetQuery("sort_direction")

-	// 筛选
+	// 筛选-名称
 	filter := bson.M{
 		"name": bson.M{"$regex": bson.RegEx{Pattern: keyword, Options: "im"}},
 	}
+
+	// 筛选-类型
 	if t != "" && t != "all" {
 		filter["type"] = t
 	}
+
+	// 筛选-是否为长任务
+	if t == "long-task" {
+		delete(filter, "type")
+		filter["is_long_task"] = true
+	}
+
+	// 筛选-项目
 	if pid == "" {
 		// do nothing
 	} else if pid == constants.ObjectIdNull {
--- a/backend/services/task.go
+++ b/backend/services/task.go
@@ -350,10 +350,9 @@ func SaveTaskResultCount(id string) func() {
 func ExecuteTask(id int) {
 	if flag, ok := LockList.Load(id); ok {
 		if flag.(bool) {
-			log.Debugf(GetWorkerPrefix(id) + "正在执行任务...")
+			log.Debugf(GetWorkerPrefix(id) + "running tasks...")
 			return
 		}
-
 	}

 	// 上锁
@@ -378,6 +377,7 @@ func ExecuteTask(id int) {

 	// 节点队列
 	queueCur := "tasks:node:" + node.Id.Hex()
+
 	// 节点队列任务
 	var msg string
 	if msg, err = database.RedisClient.LPop(queueCur); err != nil {
@@ -387,6 +387,7 @@ func ExecuteTask(id int) {
 		}
 	}

+	// 如果没有获取到任务，返回
 	if msg == "" {
 		return
 	}
@@ -504,6 +505,8 @@ func ExecuteTask(id int) {
 		log.Errorf(GetWorkerPrefix(id) + err.Error())
 		return
 	}
+
+	// 统计数据
 	t.Status = constants.StatusFinished                     // 任务状态: 已完成
 	t.FinishTs = time.Now()                                 // 结束时间
 	t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长
@@ -849,6 +852,14 @@ func SendNotifications(u model.User, t model.Task, s model.Spider) {
 	}
 }

+func UnlockLongTask(s model.Spider, n model.Node) {
+	if s.IsLongTask {
+		colName := "long-tasks"
+		key := fmt.Sprintf("%s:%s", s.Id.Hex(), n.Id.Hex())
+		_ = database.RedisClient.HDel(colName, key)
+	}
+}
+
 func InitTaskExecutor() error {
 	c := cron.New(cron.WithSeconds())
 	Exec = &Executor{
--- a/frontend/src/App.vue
+++ b/frontend/src/App.vue
@@ -47,6 +47,10 @@ export default {

    // get latest version
    await this.$store.dispatch('version/getLatestRelease')
+
+    // remove loading-placeholder
+    const elLoading = document.querySelector('#loading-placeholder')
+    elLoading.remove()
  }
 }
 </script>
--- a/frontend/src/components/Common/CrawlConfirmDialog.vue
+++ b/frontend/src/components/Common/CrawlConfirmDialog.vue
@@ -68,7 +68,7 @@
            <span style="margin-left: 5px">我已阅读并同意 <a href="javascript:"
                                                      @click="onClickDisclaimer">《免责声明》</a> 所有内容</span>
          </div>
-          <div v-if="spiderForm.is_long_task">
+          <div v-if="!spiderForm.is_long_task">
            <el-checkbox v-model="isRedirect"/>
            <span style="margin-left: 5px">跳转到任务详情页</span>
          </div>
@@ -172,6 +172,8 @@ export default {
          this.$router.push('/tasks/' + id)
          this.$st.sendEv('爬虫确认', '跳转到任务详情')
        }
+
+        this.$emit('confirm')
      })
    },
    onClickDisclaimer () {
--- a/frontend/src/components/Status/StatusTag.vue
+++ b/frontend/src/components/Status/StatusTag.vue
@@ -21,7 +21,8 @@ export default {
        running: { label: 'Running', type: 'warning' },
        finished: { label: 'Finished', type: 'success' },
        error: { label: 'Error', type: 'danger' },
-        cancelled: { label: 'Cancelled', type: 'info' }
+        cancelled: { label: 'Cancelled', type: 'info' },
+        abnormal: { label: 'Abnormal', type: 'danger' }
      }
    }
  },
@@ -43,6 +44,8 @@ export default {
    icon () {
      if (this.status === 'finished') {
        return 'el-icon-check'
+      } else if (this.status === 'pending') {
+        return 'el-icon-loading'
      } else if (this.status === 'running') {
        return 'el-icon-loading'
      } else if (this.status === 'error') {
@@ -50,7 +53,7 @@ export default {
      } else if (this.status === 'cancelled') {
        return 'el-icon-video-pause'
      } else if (this.status === 'abnormal') {
-        return 'el-icon-question'
+        return 'el-icon-warning'
      } else {
        return 'el-icon-question'
      }
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -39,6 +39,7 @@ export default {
  Error: '错误',
  NA: '未知',
  Cancelled: '已取消',
+  Abnormal: '异常',

  // 操作
  Add: '添加',
@@ -214,7 +215,8 @@ export default {
  'SSH Public Key': 'SSH 公钥',
  'Is Long Task': '是否为长任务',
  'Long Task': '长任务',
-  'Running Tasks': '运行任务数',
+  'Running Task Count': '运行中的任务数',
+  'Running Tasks': '运行中的任务',

  // 爬虫列表
  'Name': '名称',
--- a/frontend/src/store/modules/task.js
+++ b/frontend/src/store/modules/task.js
@@ -174,10 +174,13 @@ const actions = {
    link.remove()
  },
  cancelTask ({ state, dispatch }, id) {
-    return request.post(`/tasks/${id}/cancel`)
-      .then(() => {
-        dispatch('getTaskData', id)
-      })
+    return new Promise(resolve => {
+      request.post(`/tasks/${id}/cancel`)
+        .then(res => {
+          dispatch('getTaskData', id)
+          resolve(res)
+        })
+    })
  }
 }

--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -143,11 +143,123 @@
    </el-dialog>
    <!--./add dialog-->

+    <!--running tasks dialog-->
+    <el-dialog
+      :visible.sync="isRunningTasksDialogVisible"
+      :title="`${$t('Latest Tasks')} (${$t('Spider')}: ${activeSpider ? activeSpider.name : ''})`"
+      width="920px"
+    >
+      <el-tabs v-model="activeSpiderTaskStatus">
+        <el-tab-pane name="pending" :label="$t('Pending')"/>
+        <el-tab-pane name="running" :label="$t('Running')"/>
+        <el-tab-pane name="finished" :label="$t('Finished')"/>
+        <el-tab-pane name="error" :label="$t('Error')"/>
+        <el-tab-pane name="cancelled" :label="$t('Cancelled')"/>
+        <el-tab-pane name="abnormal" :label="$t('Abnormal')"/>
+      </el-tabs>
+      <el-table
+        :data="activeNodeList"
+        class="table"
+        :header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
+        border
+        default-expand-all
+      >
+        <el-table-column type="expand">
+          <template slot-scope="scope">
+            <h4 style="margin: 5px 10px">{{$t('Tasks')}}</h4>
+            <el-table
+              :data="getTasksByNode(scope.row)"
+              class="table"
+              :header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
+              border
+              style="margin: 5px 10px"
+              max-height="240px"
+              @row-click="onViewTask"
+            >
+              <el-table-column
+                :label="$t('Create Time')"
+                prop="create_ts"
+                width="140px"
+              />
+              <el-table-column
+                :label="$t('Start Time')"
+                prop="start_ts"
+                width="140px"
+              />
+              <el-table-column
+                :label="$t('Finish Time')"
+                prop="finish_ts"
+                width="140px"
+              />
+              <el-table-column
+                :label="$t('Parameters')"
+                prop="param"
+                width="120px"
+              />
+              <el-table-column
+                :label="$t('Status')"
+                width="120px"
+              >
+                <template slot-scope="scope">
+                  <status-tag :status="scope.row.status"/>
+                </template>
+              </el-table-column>
+              <el-table-column
+                :label="$t('Results Count')"
+                prop="result_count"
+                width="80px"
+              />
+              <el-table-column
+                :label="$t('Action')"
+                width="auto"
+              >
+                <template slot-scope="scope">
+                  <el-button
+                    v-if="['pending', 'running'].includes(scope.row.status)"
+                    type="danger"
+                    size="mini"
+                    icon="el-icon-video-pause"
+                    @click="onStop(scope.row, $event)"
+                  />
+                </template>
+              </el-table-column>
+            </el-table>
+          </template>
+        </el-table-column>
+        <el-table-column
+          :label="$t('Node')"
+          width="150px"
+          prop="name"
+        />
+        <el-table-column
+          :label="$t('Status')"
+          width="120px"
+          prop="status"
+        >
+          <template slot-scope="scope">
+            <el-tag type="info" v-if="scope.row.status === 'offline'">{{$t('Offline')}}</el-tag>
+            <el-tag type="success" v-else-if="scope.row.status === 'online'">{{$t('Online')}}</el-tag>
+            <el-tag type="danger" v-else>{{$t('Unavailable')}}</el-tag>
+          </template>
+        </el-table-column>
+        <el-table-column
+          :label="$t('Description')"
+          width="auto"
+          prop="description"
+        />
+      </el-table>
+      <template slot="footer">
+        <el-button type="primary" size="small" @click="isRunningTasksDialogVisible = false">{{$t('Ok')}}</el-button>
+      </template>
+    </el-dialog>
+    <!--./running tasks dialog-->
+
    <!--crawl confirm dialog-->
    <crawl-confirm-dialog
      :visible="crawlConfirmDialogVisible"
      :spider-id="activeSpiderId"
      @close="crawlConfirmDialogVisible = false"
+      @confirm="onCrawlConfirm"
    />
    <!--./crawl confirm dialog-->

@@ -226,6 +338,35 @@
      </el-tabs>
      <!--./tabs-->

+      <!--legend-->
+      <div class="legend">
+        <el-tag type="primary" size="small">
+          <i class="el-icon-loading"></i>
+          {{$t('Pending')}}
+        </el-tag>
+        <el-tag type="warning" size="small">
+          <i class="el-icon-loading"></i>
+          {{$t('Running')}}
+        </el-tag>
+        <el-tag type="success" size="small">
+          <i class="el-icon-check"></i>
+          {{$t('Finished')}}
+        </el-tag>
+        <el-tag type="danger" size="small">
+          <i class="el-icon-error"></i>
+          {{$t('Error')}}
+        </el-tag>
+        <el-tag type="info" size="small">
+          <i class="el-icon-video-pause"></i>
+          {{$t('Cancelled')}}
+        </el-tag>
+        <el-tag type="danger" size="small">
+          <i class="el-icon-warning"></i>
+          {{$t('Abnormal')}}
+        </el-tag>
+      </div>
+      <!--./legend-->
+
      <!--table list-->
      <el-table
        :data="spiderList"
@@ -316,14 +457,61 @@
            </template>
          </el-table-column>
          <el-table-column
-            v-else-if="col.name === 'running_tasks'"
+            v-else-if="col.name === 'latest_tasks'"
            :key="col.name"
            :label="$t(col.label)"
            :width="col.width"
+            :align="col.align"
+            class-name="latest-tasks"
          >
            <template slot-scope="scope">
-              <el-tag :type="getRunningTasksStatusType(scope.row)" size="small">
-                {{scope.row[col.name] ? scope.row[col.name].length : '0'}} / {{activeNodeList.length}}
+              <el-tag
+                v-if="getTaskCountByStatus(scope.row, 'pending') > 0"
+                type="primary"
+                size="small"
+              >
+                <i class="el-icon-loading"></i>
+                {{getTaskCountByStatus(scope.row, 'pending')}}
+              </el-tag>
+              <el-tag
+                v-if="getTaskCountByStatus(scope.row, 'running') > 0"
+                type="warning"
+                size="small"
+              >
+                <i class="el-icon-loading"></i>
+                {{getTaskCountByStatus(scope.row, 'running')}}
+              </el-tag>
+              <el-tag
+                v-if="getTaskCountByStatus(scope.row, 'finished') > 0"
+                type="success"
+                size="small"
+              >
+                <i class="el-icon-check"></i>
+                {{getTaskCountByStatus(scope.row, 'finished')}}
+              </el-tag>
+              <el-tag
+                v-if="getTaskCountByStatus(scope.row, 'error') > 0"
+                type="danger"
+                size="small"
+              >
+                <i class="el-icon-error"></i>
+                {{getTaskCountByStatus(scope.row, 'error')}}
+              </el-tag>
+              <el-tag
+                v-if="getTaskCountByStatus(scope.row, 'cancelled') > 0"
+                type="info"
+                size="small"
+              >
+                <i class="el-icon-video-pause"></i>
+                {{getTaskCountByStatus(scope.row, 'cancelled')}}
+              </el-tag>
+              <el-tag
+                v-if="getTaskCountByStatus(scope.row, 'abnormal') > 0"
+                type="danger"
+                size="small"
+              >
+                <i class="el-icon-warning"></i>
+                {{getTaskCountByStatus(scope.row, 'abnormal')}}
              </el-tag>
            </template>
          </el-table-column>
@@ -338,7 +526,7 @@
          >
          </el-table-column>
        </template>
-        <el-table-column :label="$t('Action')" align="left" fixed="right">
+        <el-table-column :label="$t('Action')" align="left" fixed="right" min-width="170px">
          <template slot-scope="scope">
            <el-tooltip :content="$t('View')" placement="top">
              <el-button type="primary" icon="el-icon-search" size="mini"
@@ -355,6 +543,14 @@
            <el-tooltip v-else :content="$t('Run')" placement="top">
              <el-button type="success" icon="fa fa-bug" size="mini" @click="onCrawl(scope.row, $event)"></el-button>
            </el-tooltip>
+            <el-tooltip :content="$t('Latest Tasks')" placement="top">
+              <el-button
+                type="warning"
+                icon="fa fa-tasks"
+                size="mini"
+                @click="onViewRunningTasks(scope.row, $event)"
+              />
+            </el-tooltip>
          </template>
        </el-table-column>
      </el-table>
@@ -401,7 +597,9 @@ export default {
      dialogVisible: false,
      addDialogVisible: false,
      crawlConfirmDialogVisible: false,
+      isRunningTasksDialogVisible: false,
      activeSpiderId: undefined,
+      activeSpider: undefined,
      filter: {
        project_id: '',
        keyword: '',
@@ -539,7 +737,8 @@ export default {
          this.$utils.tour.nextStep('spider-list-add', currentStep)
        }
      },
-      handle: undefined
+      handle: undefined,
+      activeSpiderTaskStatus: 'running'
    }
  },
  computed: {
@@ -576,11 +775,11 @@ export default {
      columns.push({ name: 'type', label: 'Spider Type', width: '120', sortable: true })
      columns.push({ name: 'is_long_task', label: 'Is Long Task', width: '80' })
      columns.push({ name: 'is_scrapy', label: 'Is Scrapy', width: '80' })
+      columns.push({ name: 'latest_tasks', label: 'Latest Tasks', width: '180' })
      columns.push({ name: 'last_status', label: 'Last Status', width: '120' })
      columns.push({ name: 'last_run_ts', label: 'Last Run', width: '140' })
      columns.push({ name: 'update_ts', label: 'Update Time', width: '140' })
      columns.push({ name: 'create_ts', label: 'Create Time', width: '140' })
-      columns.push({ name: 'running_tasks', label: 'Running Tasks', width: '120' })
      columns.push({ name: 'remark', label: 'Remark', width: '140' })
      return columns
    },
@@ -686,12 +885,6 @@ export default {
    onAddDialogClose () {
      this.addDialogVisible = false
    },
-    onAddCustomizedDialogClose () {
-      this.addCustomizedDialogVisible = false
-    },
-    onAddConfigurableDialogClose () {
-      this.addConfigurableDialogVisible = false
-    },
    onEdit (row) {
      this.isEditMode = true
      this.$store.commit('spider/SET_SPIDER_FORM', row)
@@ -719,6 +912,11 @@ export default {
      this.activeSpiderId = row._id
      this.$st.sendEv('爬虫列表', '点击运行')
    },
+    onCrawlConfirm () {
+      setTimeout(() => {
+        this.getList()
+      }, 1000)
+    },
    onView (row, ev) {
      ev.stopPropagation()
      this.$router.push('/spiders/' + row._id)
@@ -768,21 +966,6 @@ export default {
        callback(data)
      })
    },
-    onAddConfigurableSiteSelect (item) {
-      this.spiderForm.site = item._id
-    },
-    onAddConfigurableSpider () {
-      this.$refs['addConfigurableForm'].validate(res => {
-        if (res) {
-          this.addConfigurableLoading = true
-          this.$store.dispatch('spider/addSpider')
-            .finally(() => {
-              this.addConfigurableLoading = false
-              this.addConfigurableDialogVisible = false
-            })
-        }
-      })
-    },
    onUploadSuccess (res) {
      // clear fileList
      this.fileList = []
@@ -836,14 +1019,56 @@ export default {
        project_id: this.filter.project_id
      }
      await this.$store.dispatch('spider/getSpiderList', params)
+
+      // 更新当前爬虫（任务列表）
+      this.updateActiveSpider()
    },
-    getRunningTasksStatusType (row) {
-      if (!row.running_tasks || row.running_tasks.length === 0) {
-        return 'info'
-      } else if (this.activeNodeList && row.running_tasks.length === this.activeNodeList.length) {
-        return 'warning'
-      } else {
-        return 'warning'
+    getTasksByStatus (row, status) {
+      if (!row.latest_tasks) return []
+      return row.latest_tasks.filter(d => d.status === status)
+    },
+    getTaskCountByStatus (row, status) {
+      return this.getTasksByStatus(row, status).length
+    },
+    updateActiveSpider () {
+      if (this.activeSpider) {
+        for (let i = 0; i < this.spiderList.length; i++) {
+          const spider = this.spiderList[i]
+          if (this.activeSpider._id === spider._id) {
+            this.activeSpider = spider
+          }
+        }
+      }
+    },
+    onViewRunningTasks (row, ev) {
+      ev.stopPropagation()
+      this.activeSpider = row
+      this.isRunningTasksDialogVisible = true
+    },
+    getTasksByNode (row) {
+      if (!this.activeSpider.latest_tasks) {
+        return []
+      }
+      return this.activeSpider.latest_tasks
+        .filter(d => d.node_id === row._id && d.status === this.activeSpiderTaskStatus)
+        .map(d => {
+          d = JSON.parse(JSON.stringify(d))
+          d.create_ts = d.create_ts.match('^0001') ? 'NA' : dayjs(d.create_ts).format('YYYY-MM-DD HH:mm:ss')
+          d.start_ts = d.start_ts.match('^0001') ? 'NA' : dayjs(d.start_ts).format('YYYY-MM-DD HH:mm:ss')
+          d.finish_ts = d.finish_ts.match('^0001') ? 'NA' : dayjs(d.finish_ts).format('YYYY-MM-DD HH:mm:ss')
+          return d
+        })
+    },
+    onViewTask (row) {
+      this.$router.push(`/tasks/${row._id}`)
+      this.$st.sendEv('爬虫列表', '任务列表', '查看任务')
+    },
+    async onStop (row, ev) {
+      ev.stopPropagation()
+      const res = await this.$store.dispatch('task/cancelTask', row._id)
+      if (!res.data.error) {
+        this.$message.success(`Task "${row._id}" has been sent signal to stop`)
+        this.getList()
      }
    }
  },
@@ -856,6 +1081,9 @@ export default {
      this.filter.project_id = this.$route.params.project_id
    }

+    // fetch node list
+    await this.$store.dispatch('node/getNodeList')
+
    // fetch spider list
    await this.getList()

@@ -969,4 +1197,12 @@ export default {
  .actions {
    text-align: right;
  }
+
+  .el-table >>> .latest-tasks .el-tag {
+    margin: 3px 3px 0 0;
+  }
+
+  .legend .el-tag {
+    margin-right: 5px;
+  }
 </style>