From cce5bd71e8cc59114ebda4322194e17e35777a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 2 Sep 2019 09:40:24 +0800 Subject: [PATCH 1/8] =?UTF-8?q?fix=20=E7=88=AC=E8=99=AB=E5=88=97=E8=A1=A8?= =?UTF-8?q?=E5=9C=A8=E5=B0=8F=E5=B1=8F=E5=B9=95=E4=B8=8B=E9=94=99=E4=BD=8D?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/views/spider/SpiderList.vue | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 0380a6b0..74d1da2b 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -275,15 +275,15 @@ export default { }, // tableData, columns: [ - { name: 'name', label: 'Name', width: '180', align: 'left' }, + { name: 'name', label: 'Name', width: '160', align: 'left' }, // { name: 'site_name', label: 'Site', width: '140', align: 'left' }, { name: 'type', label: 'Spider Type', width: '120' }, // { name: 'cmd', label: 'Command Line', width: '200' }, { name: 'last_status', label: 'Last Status', width: '120' }, - { name: 'last_run_ts', label: 'Last Run', width: '160' }, - { name: 'create_ts', label: 'Create Time', width: '160' }, - { name: 'update_ts', label: 'Update Time', width: '160' }, - { name: 'remark', label: 'Remark', width: '160' } + { name: 'last_run_ts', label: 'Last Run', width: '140' }, + { name: 'create_ts', label: 'Create Time', width: '140' }, + { name: 'update_ts', label: 'Update Time', width: '140' }, + { name: 'remark', label: 'Remark', width: '140' } // { name: 'last_7d_tasks', label: 'Last 7-Day Tasks', width: '80' }, // { name: 'last_5_errors', label: 'Last 5-Run Errors', width: '80' } ], From 1e37d714b04f384e087e1ba5efc2fb9b958546e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 2 Sep 2019 10:40:16 +0800 Subject: [PATCH 2/8] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=8E=92=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/spider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/model/spider.go b/backend/model/spider.go index c1ef7b6e..2ba4c8c5 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -99,7 +99,7 @@ func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, error) { // 获取爬虫列表 spiders := []Spider{} - if err := c.Find(filter).Skip(skip).Limit(limit).Sort("name asc").All(&spiders); err != nil { + if err := c.Find(filter).Skip(skip).Limit(limit).Sort("+name").All(&spiders); err != nil { debug.PrintStack() return spiders, err } From 64a8fab961efac7765801ca0f4781cd2b3815412 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 2 Sep 2019 16:45:56 +0800 Subject: [PATCH 3/8] =?UTF-8?q?fix=20=E6=89=93=E5=8D=B0=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=8F=AA=E6=9C=892KB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/log.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/services/log.go b/backend/services/log.go index a83926f2..a7935015 100644 --- a/backend/services/log.go +++ b/backend/services/log.go @@ -35,7 +35,7 @@ func GetLocalLog(logPath string) (fileBytes []byte, err error) { } defer f.Close() - const bufLen = 2048 + const bufLen = 2 * 1024 * 1024 logBuf := make([]byte, bufLen) off := int64(0) From 69fc10fcc3505c89b7020d1ea66f9ce7870ef004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 2 Sep 2019 17:37:48 +0800 Subject: [PATCH 4/8] =?UTF-8?q?fix=20=E5=A6=82=E6=9E=9C=E4=BB=8Edir?= =?UTF-8?q?=E8=AF=BB=E5=8F=96=E7=88=AC=E8=99=AB=E4=B8=BA=E7=A9=BA=EF=BC=8C?= =?UTF-8?q?=E5=88=99=E7=A7=BB=E9=99=A4=E6=89=80=E6=9C=89=E7=9A=84=E7=88=AC?= =?UTF-8?q?=E8=99=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/spider.go | 18 ++++++++++++++++++ backend/services/spider.go | 20 +++++++++++++------- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/backend/model/spider.go b/backend/model/spider.go index 2ba4c8c5..e0e5f836 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -177,6 +177,24 @@ func RemoveSpider(id bson.ObjectId) error { return nil } +func RemoveAllSpider() error { + s, c := database.GetCol("spiders") + defer s.Close() + + spiders := []Spider{} + err := c.Find(nil).All(&spiders) + if err != nil { + log.Error("get all spiders error:" + err.Error()) + return err + } + for _, spider := range spiders { + if err := RemoveSpider(spider.Id); err != nil { + log.Error("remove spider error:" + err.Error()) + } + } + return nil +} + func GetSpiderCount() (int, error) { s, c := database.GetCol("spiders") defer s.Close() diff --git a/backend/services/spider.go b/backend/services/spider.go index 47c1fa33..e75b0418 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -89,16 +89,25 @@ func GetSpidersFromDir() ([]model.Spider, error) { // 将爬虫保存到数据库 func SaveSpiders(spiders []model.Spider) error { - // 遍历爬虫列表 + s, c := database.GetCol("spiders") + defer s.Close() + + if len(spiders) == 0 { + err := model.RemoveAllSpider() + if err != nil { + log.Error("remove all spider error:" + err.Error()) + return err + } + log.Info("get spider from dir is empty,removed all spider") + return nil + } + // 如果该爬虫不存在于数据库，则保存爬虫到数据库 for _, spider := range spiders { // 忽略非自定义爬虫 if spider.Type != constants.Customized { continue } - // 如果该爬虫不存在于数据库，则保存爬虫到数据库 - s, c := database.GetCol("spiders") - defer s.Close() var spider_ *model.Spider if err := c.Find(bson.M{"src": spider.Src}).One(&spider_); err != nil { // 不存在 @@ -106,11 +115,8 @@ func SaveSpiders(spiders []model.Spider) error { debug.PrintStack() return err } - } else { - // 存在 } } - return nil } From 45aaaa2c46c6a3918d1a9abe9f21ba239a8f1540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 2 Sep 2019 18:04:47 +0800 Subject: [PATCH 5/8] =?UTF-8?q?fix=20=E6=97=A0=E6=B3=95=E6=AD=A3=E5=B8=B8?= =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=9C=89=E9=97=AE=E9=A2=98=E7=9A=84=E7=88=AC?= =?UTF-8?q?=E8=99=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/spider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/services/spider.go b/backend/services/spider.go index e75b0418..a4f16bc8 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -259,8 +259,8 @@ func PublishAllSpiders() error { for _, spider := range spiders { // 发布爬虫 if err := PublishSpider(spider); err != nil { - log.Errorf(err.Error()) - return err + log.Errorf("publish spider error:" + err.Error()) + // return err } } From 41c3b98333270056231f23339f5eaab05b155877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 2 Sep 2019 18:14:34 +0800 Subject: [PATCH 6/8] =?UTF-8?q?=E8=BF=98=E5=8E=9F=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/spider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/services/spider.go b/backend/services/spider.go index a4f16bc8..1796e32a 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -362,7 +362,7 @@ func OnFileUpload(channel string, msgStr string) { // 解压缩临时文件到目标文件夹 dstPath := filepath.Join( viper.GetString("spider.path"), - //strings.Replace(msg.FileName, ".zip", "", -1), + strings.Replace(msg.FileName, ".zip", "", -1), ) if err := utils.DeCompress(tmpFile, dstPath); err != nil { log.Errorf(err.Error()) From 6675d075f2ecc4816f79235d05e3452bf601a909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 2 Sep 2019 20:49:00 +0800 Subject: [PATCH 7/8] =?UTF-8?q?fix=20=E5=88=A0=E9=99=A4spider=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=E5=88=A0=E9=99=A4task=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/task.go | 17 ++++++++++++++++- backend/routes/spider.go | 6 ++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/backend/model/task.go b/backend/model/task.go index 8ae782b5..968055a6 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -190,6 +190,21 @@ func RemoveTask(id string) error { return nil } +func RemoveTaskBySpiderId(id string) error { + tasks, err := GetTaskList(bson.M{"spider_id": id}, 0, constants.Infinite, "-create_ts") + if err != nil { + log.Error("get tasks error:" + err.Error()) + } + + for _, task := range tasks { + if err := RemoveTask(task.Id); err != nil { + log.Error("remove task error:" + err.Error()) + continue + } + } + return nil +} + func GetTaskCount(query interface{}) (int, error) { s, c := database.GetCol("tasks") defer s.Close() @@ -207,7 +222,7 @@ func GetDailyTaskStats(query bson.M) ([]TaskDailyItem, error) { defer s.Close() // 起始日期 - startDate := time.Now().Add(- 30 * 24 * time.Hour) + startDate := time.Now().Add(-30 * 24 * time.Hour) endDate := time.Now() // query diff --git a/backend/routes/spider.go b/backend/routes/spider.go index dceb2651..f1a3c9e5 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -229,6 +229,12 @@ func DeleteSpider(c *gin.Context) { return } + // 删除爬虫对应的task任务 + if err := model.RemoveTaskBySpiderId(spider.Id.Hex()); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + c.JSON(http.StatusOK, Response{ Status: "ok", Message: "success", From 4255b2f6b6bdb21bf9872730de7226810b019a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 3 Sep 2019 09:06:04 +0800 Subject: [PATCH 8/8] =?UTF-8?q?=E8=BF=98=E5=8E=9F=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/spider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/services/spider.go b/backend/services/spider.go index fca2b200..87b4a1d5 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -361,7 +361,7 @@ func OnFileUpload(message redis.Message) (err error) { // 解压缩临时文件到目标文件夹 dstPath := filepath.Join( viper.GetString("spider.path"), - strings.Replace(msg.FileName, ".zip", "", -1), + // strings.Replace(msg.FileName, ".zip", "", -1), ) if err := utils.DeCompress(tmpFile, dstPath); err != nil { log.Errorf(err.Error())