diff --git a/backend/model/spider.go b/backend/model/spider.go index 61fb53ec..a741fc89 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -55,10 +55,14 @@ type Spider struct { GitSyncFrequency string `json:"git_sync_frequency" bson:"git_sync_frequency"` // Git 同步频率 GitSyncError string `json:"git_sync_error" bson:"git_sync_error"` // Git 同步错误 + // 长任务 + IsLongTask bool `json:"is_long_task" bson:"is_long_task"` // 是否为长任务 + // 前端展示 - LastRunTs time.Time `json:"last_run_ts"` // 最后一次执行时间 - LastStatus string `json:"last_status"` // 最后执行状态 - Config entity.ConfigSpiderData `json:"config"` // 可配置爬虫配置 + LastRunTs time.Time `json:"last_run_ts"` // 最后一次执行时间 + LastStatus string `json:"last_status"` // 最后执行状态 + Config entity.ConfigSpiderData `json:"config"` // 可配置爬虫配置 + LatestTasks []Task `json:"latest_tasks"` // 最近任务列表 // 时间 CreateTs time.Time `json:"create_ts" bson:"create_ts"` @@ -124,6 +128,18 @@ func (spider *Spider) GetLastTask() (Task, error) { return tasks[0], nil } +// 爬虫正在运行的任务 +func (spider *Spider) GetLatestTasks(latestN int) (tasks []Task, err error) { + tasks, err = GetTaskList(bson.M{"spider_id": spider.Id}, 0, latestN, "-create_ts") + if err != nil { + return tasks, err + } + if tasks == nil { + return tasks, err + } + return tasks, nil +} + // 删除爬虫 func (spider *Spider) Delete() error { s, c := database.GetCol("spiders") @@ -157,9 +173,18 @@ func GetSpiderList(filter interface{}, skip int, limit int, sortStr string) ([]S continue } + // 获取正在运行的爬虫 + latestTasks, err := spider.GetLatestTasks(50) // TODO: latestN 暂时写死，后面加入数据库 + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + continue + } + // 赋值 spiders[i].LastRunTs = task.CreateTs spiders[i].LastStatus = task.Status + spiders[i].LatestTasks = latestTasks } count, _ := c.Find(filter).Count() diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 39314970..dc84f462 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -35,13 +35,23 @@ func GetSpiderList(c *gin.Context) { sortKey, _ := c.GetQuery("sort_key") sortDirection, _ := c.GetQuery("sort_direction") - // 筛选 + // 筛选-名称 filter := bson.M{ "name": bson.M{"$regex": bson.RegEx{Pattern: keyword, Options: "im"}}, } + + // 筛选-类型 if t != "" && t != "all" { filter["type"] = t } + + // 筛选-是否为长任务 + if t == "long-task" { + delete(filter, "type") + filter["is_long_task"] = true + } + + // 筛选-项目 if pid == "" { // do nothing } else if pid == constants.ObjectIdNull { diff --git a/backend/services/task.go b/backend/services/task.go index e940b325..76aeed83 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -350,10 +350,9 @@ func SaveTaskResultCount(id string) func() { func ExecuteTask(id int) { if flag, ok := LockList.Load(id); ok { if flag.(bool) { - log.Debugf(GetWorkerPrefix(id) + "正在执行任务...") + log.Debugf(GetWorkerPrefix(id) + "running tasks...") return } - } // 上锁 @@ -378,6 +377,7 @@ func ExecuteTask(id int) { // 节点队列 queueCur := "tasks:node:" + node.Id.Hex() + // 节点队列任务 var msg string if msg, err = database.RedisClient.LPop(queueCur); err != nil { @@ -387,6 +387,7 @@ func ExecuteTask(id int) { } } + // 如果没有获取到任务，返回 if msg == "" { return } @@ -504,6 +505,8 @@ func ExecuteTask(id int) { log.Errorf(GetWorkerPrefix(id) + err.Error()) return } + + // 统计数据 t.Status = constants.StatusFinished // 任务状态: 已完成 t.FinishTs = time.Now() // 结束时间 t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长 @@ -849,6 +852,14 @@ func SendNotifications(u model.User, t model.Task, s model.Spider) { } } +func UnlockLongTask(s model.Spider, n model.Node) { + if s.IsLongTask { + colName := "long-tasks" + key := fmt.Sprintf("%s:%s", s.Id.Hex(), n.Id.Hex()) + _ = database.RedisClient.HDel(colName, key) + } +} + func InitTaskExecutor() error { c := cron.New(cron.WithSeconds()) Exec = &Executor{ diff --git a/frontend/src/App.vue b/frontend/src/App.vue index d66691c4..93d41827 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -47,6 +47,10 @@ export default { // get latest version await this.$store.dispatch('version/getLatestRelease') + + // remove loading-placeholder + const elLoading = document.querySelector('#loading-placeholder') + elLoading.remove() } } diff --git a/frontend/src/components/Common/CrawlConfirmDialog.vue b/frontend/src/components/Common/CrawlConfirmDialog.vue index e38af787..1d5e2d7a 100644 --- a/frontend/src/components/Common/CrawlConfirmDialog.vue +++ b/frontend/src/components/Common/CrawlConfirmDialog.vue @@ -68,7 +68,7 @@ 我已阅读并同意 《免责声明》 所有内容 -
+
跳转到任务详情页
@@ -172,6 +172,8 @@ export default { this.$router.push('/tasks/' + id) this.$st.sendEv('爬虫确认', '跳转到任务详情') } + + this.$emit('confirm') }) }, onClickDisclaimer () { diff --git a/frontend/src/components/Status/StatusTag.vue b/frontend/src/components/Status/StatusTag.vue index 0278f02e..273ba8e4 100644 --- a/frontend/src/components/Status/StatusTag.vue +++ b/frontend/src/components/Status/StatusTag.vue @@ -21,7 +21,8 @@ export default { running: { label: 'Running', type: 'warning' }, finished: { label: 'Finished', type: 'success' }, error: { label: 'Error', type: 'danger' }, - cancelled: { label: 'Cancelled', type: 'info' } + cancelled: { label: 'Cancelled', type: 'info' }, + abnormal: { label: 'Abnormal', type: 'danger' } } } }, @@ -43,6 +44,8 @@ export default { icon () { if (this.status === 'finished') { return 'el-icon-check' + } else if (this.status === 'pending') { + return 'el-icon-loading' } else if (this.status === 'running') { return 'el-icon-loading' } else if (this.status === 'error') { @@ -50,7 +53,7 @@ export default { } else if (this.status === 'cancelled') { return 'el-icon-video-pause' } else if (this.status === 'abnormal') { - return 'el-icon-question' + return 'el-icon-warning' } else { return 'el-icon-question' } diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index f311a093..5c7e03e2 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -39,6 +39,7 @@ export default { Error: '错误', NA: '未知', Cancelled: '已取消', + Abnormal: '异常', // 操作 Add: '添加', @@ -214,7 +215,8 @@ export default { 'SSH Public Key': 'SSH 公钥', 'Is Long Task': '是否为长任务', 'Long Task': '长任务', - 'Running Tasks': '运行任务数', + 'Running Task Count': '运行中的任务数', + 'Running Tasks': '运行中的任务', // 爬虫列表 'Name': '名称', diff --git a/frontend/src/store/modules/task.js b/frontend/src/store/modules/task.js index 595ec7b0..85270729 100644 --- a/frontend/src/store/modules/task.js +++ b/frontend/src/store/modules/task.js @@ -174,10 +174,13 @@ const actions = { link.remove() }, cancelTask ({ state, dispatch }, id) { - return request.post(`/tasks/${id}/cancel`) - .then(() => { - dispatch('getTaskData', id) - }) + return new Promise(resolve => { + request.post(`/tasks/${id}/cancel`) + .then(res => { + dispatch('getTaskData', id) + resolve(res) + }) + }) } } diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index f9f99b6c..2ddd2369 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -143,11 +143,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + @@ -226,6 +338,35 @@ + +
+ + + {{$t('Pending')}} + + + + {{$t('Running')}} + + + + {{$t('Finished')}} + + + + {{$t('Error')}} + + + + {{$t('Cancelled')}} + + + + {{$t('Abnormal')}} + +
+ + @@ -338,7 +526,7 @@ > - + @@ -401,7 +597,9 @@ export default { dialogVisible: false, addDialogVisible: false, crawlConfirmDialogVisible: false, + isRunningTasksDialogVisible: false, activeSpiderId: undefined, + activeSpider: undefined, filter: { project_id: '', keyword: '', @@ -539,7 +737,8 @@ export default { this.$utils.tour.nextStep('spider-list-add', currentStep) } }, - handle: undefined + handle: undefined, + activeSpiderTaskStatus: 'running' } }, computed: { @@ -576,11 +775,11 @@ export default { columns.push({ name: 'type', label: 'Spider Type', width: '120', sortable: true }) columns.push({ name: 'is_long_task', label: 'Is Long Task', width: '80' }) columns.push({ name: 'is_scrapy', label: 'Is Scrapy', width: '80' }) + columns.push({ name: 'latest_tasks', label: 'Latest Tasks', width: '180' }) columns.push({ name: 'last_status', label: 'Last Status', width: '120' }) columns.push({ name: 'last_run_ts', label: 'Last Run', width: '140' }) columns.push({ name: 'update_ts', label: 'Update Time', width: '140' }) columns.push({ name: 'create_ts', label: 'Create Time', width: '140' }) - columns.push({ name: 'running_tasks', label: 'Running Tasks', width: '120' }) columns.push({ name: 'remark', label: 'Remark', width: '140' }) return columns }, @@ -686,12 +885,6 @@ export default { onAddDialogClose () { this.addDialogVisible = false }, - onAddCustomizedDialogClose () { - this.addCustomizedDialogVisible = false - }, - onAddConfigurableDialogClose () { - this.addConfigurableDialogVisible = false - }, onEdit (row) { this.isEditMode = true this.$store.commit('spider/SET_SPIDER_FORM', row) @@ -719,6 +912,11 @@ export default { this.activeSpiderId = row._id this.$st.sendEv('爬虫列表', '点击运行') }, + onCrawlConfirm () { + setTimeout(() => { + this.getList() + }, 1000) + }, onView (row, ev) { ev.stopPropagation() this.$router.push('/spiders/' + row._id) @@ -768,21 +966,6 @@ export default { callback(data) }) }, - onAddConfigurableSiteSelect (item) { - this.spiderForm.site = item._id - }, - onAddConfigurableSpider () { - this.$refs['addConfigurableForm'].validate(res => { - if (res) { - this.addConfigurableLoading = true - this.$store.dispatch('spider/addSpider') - .finally(() => { - this.addConfigurableLoading = false - this.addConfigurableDialogVisible = false - }) - } - }) - }, onUploadSuccess (res) { // clear fileList this.fileList = [] @@ -836,14 +1019,56 @@ export default { project_id: this.filter.project_id } await this.$store.dispatch('spider/getSpiderList', params) + + // 更新当前爬虫（任务列表） + this.updateActiveSpider() }, - getRunningTasksStatusType (row) { - if (!row.running_tasks || row.running_tasks.length === 0) { - return 'info' - } else if (this.activeNodeList && row.running_tasks.length === this.activeNodeList.length) { - return 'warning' - } else { - return 'warning' + getTasksByStatus (row, status) { + if (!row.latest_tasks) return [] + return row.latest_tasks.filter(d => d.status === status) + }, + getTaskCountByStatus (row, status) { + return this.getTasksByStatus(row, status).length + }, + updateActiveSpider () { + if (this.activeSpider) { + for (let i = 0; i < this.spiderList.length; i++) { + const spider = this.spiderList[i] + if (this.activeSpider._id === spider._id) { + this.activeSpider = spider + } + } + } + }, + onViewRunningTasks (row, ev) { + ev.stopPropagation() + this.activeSpider = row + this.isRunningTasksDialogVisible = true + }, + getTasksByNode (row) { + if (!this.activeSpider.latest_tasks) { + return [] + } + return this.activeSpider.latest_tasks + .filter(d => d.node_id === row._id && d.status === this.activeSpiderTaskStatus) + .map(d => { + d = JSON.parse(JSON.stringify(d)) + d.create_ts = d.create_ts.match('^0001') ? 'NA' : dayjs(d.create_ts).format('YYYY-MM-DD HH:mm:ss') + d.start_ts = d.start_ts.match('^0001') ? 'NA' : dayjs(d.start_ts).format('YYYY-MM-DD HH:mm:ss') + d.finish_ts = d.finish_ts.match('^0001') ? 'NA' : dayjs(d.finish_ts).format('YYYY-MM-DD HH:mm:ss') + return d + }) + }, + onViewTask (row) { + this.$router.push(`/tasks/${row._id}`) + this.$st.sendEv('爬虫列表', '任务列表', '查看任务') + }, + async onStop (row, ev) { + ev.stopPropagation() + const res = await this.$store.dispatch('task/cancelTask', row._id) + if (!res.data.error) { + this.$message.success(`Task "${row._id}" has been sent signal to stop`) + this.getList() } } }, @@ -856,6 +1081,9 @@ export default { this.filter.project_id = this.$route.params.project_id } + // fetch node list + await this.$store.dispatch('node/getNodeList') + // fetch spider list await this.getList() @@ -969,4 +1197,12 @@ export default { .actions { text-align: right; } + + .el-table >>> .latest-tasks .el-tag { + margin: 3px 3px 0 0; + } + + .legend .el-tag { + margin-right: 5px; + }