diff --git a/backend/model/spider.go b/backend/model/spider.go index c1ef7b6e..e0e5f836 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -99,7 +99,7 @@ func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, error) { // 获取爬虫列表 spiders := []Spider{} - if err := c.Find(filter).Skip(skip).Limit(limit).Sort("name asc").All(&spiders); err != nil { + if err := c.Find(filter).Skip(skip).Limit(limit).Sort("+name").All(&spiders); err != nil { debug.PrintStack() return spiders, err } @@ -177,6 +177,24 @@ func RemoveSpider(id bson.ObjectId) error { return nil } +func RemoveAllSpider() error { + s, c := database.GetCol("spiders") + defer s.Close() + + spiders := []Spider{} + err := c.Find(nil).All(&spiders) + if err != nil { + log.Error("get all spiders error:" + err.Error()) + return err + } + for _, spider := range spiders { + if err := RemoveSpider(spider.Id); err != nil { + log.Error("remove spider error:" + err.Error()) + } + } + return nil +} + func GetSpiderCount() (int, error) { s, c := database.GetCol("spiders") defer s.Close() diff --git a/backend/model/task.go b/backend/model/task.go index 8ae782b5..968055a6 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -190,6 +190,21 @@ func RemoveTask(id string) error { return nil } +func RemoveTaskBySpiderId(id string) error { + tasks, err := GetTaskList(bson.M{"spider_id": id}, 0, constants.Infinite, "-create_ts") + if err != nil { + log.Error("get tasks error:" + err.Error()) + } + + for _, task := range tasks { + if err := RemoveTask(task.Id); err != nil { + log.Error("remove task error:" + err.Error()) + continue + } + } + return nil +} + func GetTaskCount(query interface{}) (int, error) { s, c := database.GetCol("tasks") defer s.Close() @@ -207,7 +222,7 @@ func GetDailyTaskStats(query bson.M) ([]TaskDailyItem, error) { defer s.Close() // 起始日期 - startDate := time.Now().Add(- 30 * 24 * time.Hour) + startDate := time.Now().Add(-30 * 24 * time.Hour) endDate := time.Now() // query diff --git a/backend/routes/spider.go b/backend/routes/spider.go index dceb2651..f1a3c9e5 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -229,6 +229,12 @@ func DeleteSpider(c *gin.Context) { return } + // 删除爬虫对应的task任务 + if err := model.RemoveTaskBySpiderId(spider.Id.Hex()); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + c.JSON(http.StatusOK, Response{ Status: "ok", Message: "success", diff --git a/backend/services/log.go b/backend/services/log.go index da1d72e1..088a825a 100644 --- a/backend/services/log.go +++ b/backend/services/log.go @@ -35,7 +35,7 @@ func GetLocalLog(logPath string) (fileBytes []byte, err error) { } defer f.Close() - const bufLen = 2048 + const bufLen = 2 * 1024 * 1024 logBuf := make([]byte, bufLen) off := int64(0) diff --git a/backend/services/spider.go b/backend/services/spider.go index 61a561a9..87b4a1d5 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -91,16 +91,25 @@ func GetSpidersFromDir() ([]model.Spider, error) { // 将爬虫保存到数据库 func SaveSpiders(spiders []model.Spider) error { - // 遍历爬虫列表 + s, c := database.GetCol("spiders") + defer s.Close() + + if len(spiders) == 0 { + err := model.RemoveAllSpider() + if err != nil { + log.Error("remove all spider error:" + err.Error()) + return err + } + log.Info("get spider from dir is empty,removed all spider") + return nil + } + // 如果该爬虫不存在于数据库，则保存爬虫到数据库 for _, spider := range spiders { // 忽略非自定义爬虫 if spider.Type != constants.Customized { continue } - // 如果该爬虫不存在于数据库，则保存爬虫到数据库 - s, c := database.GetCol("spiders") - defer s.Close() var spider_ *model.Spider if err := c.Find(bson.M{"src": spider.Src}).One(&spider_); err != nil { // 不存在 @@ -108,11 +117,8 @@ func SaveSpiders(spiders []model.Spider) error { debug.PrintStack() return err } - } else { - // 存在 } } - return nil } @@ -252,8 +258,8 @@ func PublishAllSpiders() error { for _, spider := range spiders { // 发布爬虫 if err := PublishSpider(spider); err != nil { - log.Errorf(err.Error()) - return err + log.Errorf("publish spider error:" + err.Error()) + // return err } } @@ -355,7 +361,7 @@ func OnFileUpload(message redis.Message) (err error) { // 解压缩临时文件到目标文件夹 dstPath := filepath.Join( viper.GetString("spider.path"), - //strings.Replace(msg.FileName, ".zip", "", -1), + // strings.Replace(msg.FileName, ".zip", "", -1), ) if err := utils.DeCompress(tmpFile, dstPath); err != nil { log.Errorf(err.Error()) diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 0380a6b0..74d1da2b 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -275,15 +275,15 @@ export default { }, // tableData, columns: [ - { name: 'name', label: 'Name', width: '180', align: 'left' }, + { name: 'name', label: 'Name', width: '160', align: 'left' }, // { name: 'site_name', label: 'Site', width: '140', align: 'left' }, { name: 'type', label: 'Spider Type', width: '120' }, // { name: 'cmd', label: 'Command Line', width: '200' }, { name: 'last_status', label: 'Last Status', width: '120' }, - { name: 'last_run_ts', label: 'Last Run', width: '160' }, - { name: 'create_ts', label: 'Create Time', width: '160' }, - { name: 'update_ts', label: 'Update Time', width: '160' }, - { name: 'remark', label: 'Remark', width: '160' } + { name: 'last_run_ts', label: 'Last Run', width: '140' }, + { name: 'create_ts', label: 'Create Time', width: '140' }, + { name: 'update_ts', label: 'Update Time', width: '140' }, + { name: 'remark', label: 'Remark', width: '140' } // { name: 'last_7d_tasks', label: 'Last 7-Day Tasks', width: '80' }, // { name: 'last_5_errors', label: 'Last 5-Run Errors', width: '80' } ],