From ce8fb8670089aea2e6939dd3aee55fd28e4021f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 24 Aug 2019 15:11:52 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/node.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/services/node.go b/backend/services/node.go index 1fa2370c..09b49dbf 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -124,6 +124,7 @@ func IsMaster() bool { return viper.GetString("server.master") == Yes } +// 所有调用IsMasterNode的方法，都永远会在master节点执行，所以GetCurrentNode方法返回永远是master节点 // 该ID的节点是否为主节点 func IsMasterNode(id string) bool { curNode, _ := GetCurrentNode() From c7e137a6aa04f627db0a5c98e51a09c793c3b23e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 27 Aug 2019 09:39:16 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E5=88=A0=E9=99=A4=E7=88=AC=E8=99=AB?= =?UTF-8?q?=E9=A1=BA=E5=B8=A6=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/spider.go | 11 +++++++++-- backend/routes/utils.go | 2 ++ backend/services/spider.go | 2 +- backend/services/task.go | 1 + 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/backend/model/spider.go b/backend/model/spider.go index c4c94edf..65782fe8 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -92,8 +92,6 @@ func (spider *Spider) GetLastTask() (Task, error) { return tasks[0], nil } - - func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, error) { s, c := database.GetCol("spiders") defer s.Close() @@ -165,6 +163,15 @@ func RemoveSpider(id bson.ObjectId) error { return err } + // gf上的文件 + s, gf := database.GetGridFs("files") + defer s.Close() + + if err := gf.RemoveId(result.FileId); err != nil { + log.Error("remove file error, id:" + result.FileId.Hex()) + return err + } + return nil } diff --git a/backend/routes/utils.go b/backend/routes/utils.go index 14c5853e..8044ffc9 100644 --- a/backend/routes/utils.go +++ b/backend/routes/utils.go @@ -1,11 +1,13 @@ package routes import ( + "github.com/apex/log" "github.com/gin-gonic/gin" "runtime/debug" ) func HandleError(statusCode int, c *gin.Context, err error) { + log.Errorf("handle error:" + err.Error()) debug.PrintStack() c.JSON(statusCode, Response{ Status: "ok", diff --git a/backend/services/spider.go b/backend/services/spider.go index f4f856e6..5c5c6e6e 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -320,7 +320,7 @@ func OnFileUpload(channel string, msgStr string) { // 从GridFS获取该文件 f, err := gf.OpenId(bson.ObjectIdHex(msg.FileId)) if err != nil { - log.Errorf(err.Error()) + log.Errorf("open file id" + msg.FileId + ", error: " + err.Error()) debug.PrintStack() return } diff --git a/backend/services/task.go b/backend/services/task.go index 8c0ff8a1..8c3c3407 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -472,6 +472,7 @@ func CancelTask(id string) (err error) { } func HandleTaskError(t model.Task, err error) { + log.Error("handle task error:" + err.Error()) t.Status = constants.StatusError t.Error = err.Error() t.FinishTs = time.Now() From cae11e37960e26d68c7876662e9357df12e0d509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Wed, 28 Aug 2019 16:00:09 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=89=93=E5=8D=B0?= =?UTF-8?q?=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/spider.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/services/spider.go b/backend/services/spider.go index 5c5c6e6e..ce416a79 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -295,6 +295,7 @@ func PublishSpider(spider model.Spider) (err error) { return } channel := "files:upload" + log.Info("publish files.upload event, file id:" + msg.FileId) if err = database.Publish(channel, string(msgStr)); err != nil { log.Errorf(err.Error()) debug.PrintStack() @@ -306,6 +307,7 @@ func PublishSpider(spider model.Spider) (err error) { // 上传爬虫回调 func OnFileUpload(channel string, msgStr string) { + log.Info("received files.upload event, msgStr:" + msgStr) s, gf := database.GetGridFs("files") defer s.Close() @@ -320,7 +322,7 @@ func OnFileUpload(channel string, msgStr string) { // 从GridFS获取该文件 f, err := gf.OpenId(bson.ObjectIdHex(msg.FileId)) if err != nil { - log.Errorf("open file id" + msg.FileId + ", error: " + err.Error()) + log.Errorf("open file id: " + msg.FileId + ", error: " + err.Error()) debug.PrintStack() return } From 369ef02fc71d2a0d63f54efdd0fcbf4b1a976fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 31 Aug 2019 10:08:37 +0800 Subject: [PATCH 4/4] fix #184 --- frontend/src/components/InfoView/TaskInfoView.vue | 6 +++--- frontend/src/views/task/TaskDetail.vue | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/InfoView/TaskInfoView.vue b/frontend/src/components/InfoView/TaskInfoView.vue index bfe6419a..e902e959 100644 --- a/frontend/src/components/InfoView/TaskInfoView.vue +++ b/frontend/src/components/InfoView/TaskInfoView.vue @@ -86,15 +86,15 @@ export default { return dayjs(str).format('YYYY-MM-DD HH:mm:ss') }, getWaitDuration (row) { - if (row.start_ts.match('^0001')) return 'NA' + if (!row.start_ts || row.start_ts.match('^0001')) return 'NA' return dayjs(row.start_ts).diff(row.create_ts, 'second') }, getRuntimeDuration (row) { - if (row.finish_ts.match('^0001')) return 'NA' + if (!row.finish_ts || row.finish_ts.match('^0001')) return 'NA' return dayjs(row.finish_ts).diff(row.start_ts, 'second') }, getTotalDuration (row) { - if (row.finish_ts.match('^0001')) return 'NA' + if (!row.finish_ts || row.finish_ts.match('^0001')) return 'NA' return dayjs(row.finish_ts).diff(row.create_ts, 'second') } } diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue index a1edb497..89309ea8 100644 --- a/frontend/src/views/task/TaskDetail.vue +++ b/frontend/src/views/task/TaskDetail.vue @@ -99,12 +99,12 @@ export default { this.$st.sendEv('任务详情-结果', '下载CSV') } }, - created () { - this.$store.dispatch('task/getTaskData', this.$route.params.id) + async created () { + await this.$store.dispatch('task/getTaskData', this.$route.params.id) this.$store.dispatch('task/getTaskLog', this.$route.params.id) this.$store.dispatch('task/getTaskResults', this.$route.params.id) - if (['running'].includes(this.taskForm.status)) { + if (this.taskForm && ['running'].includes(this.taskForm.status)) { this.handle = setInterval(() => { this.$store.dispatch('task/getTaskLog', this.$route.params.id) }, 5000)