diff --git a/backend/model/spider.go b/backend/model/spider.go index 2ba4c8c5..e0e5f836 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -177,6 +177,24 @@ func RemoveSpider(id bson.ObjectId) error { return nil } +func RemoveAllSpider() error { + s, c := database.GetCol("spiders") + defer s.Close() + + spiders := []Spider{} + err := c.Find(nil).All(&spiders) + if err != nil { + log.Error("get all spiders error:" + err.Error()) + return err + } + for _, spider := range spiders { + if err := RemoveSpider(spider.Id); err != nil { + log.Error("remove spider error:" + err.Error()) + } + } + return nil +} + func GetSpiderCount() (int, error) { s, c := database.GetCol("spiders") defer s.Close() diff --git a/backend/services/spider.go b/backend/services/spider.go index 47c1fa33..e75b0418 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -89,16 +89,25 @@ func GetSpidersFromDir() ([]model.Spider, error) { // 将爬虫保存到数据库 func SaveSpiders(spiders []model.Spider) error { - // 遍历爬虫列表 + s, c := database.GetCol("spiders") + defer s.Close() + + if len(spiders) == 0 { + err := model.RemoveAllSpider() + if err != nil { + log.Error("remove all spider error:" + err.Error()) + return err + } + log.Info("get spider from dir is empty,removed all spider") + return nil + } + // 如果该爬虫不存在于数据库,则保存爬虫到数据库 for _, spider := range spiders { // 忽略非自定义爬虫 if spider.Type != constants.Customized { continue } - // 如果该爬虫不存在于数据库,则保存爬虫到数据库 - s, c := database.GetCol("spiders") - defer s.Close() var spider_ *model.Spider if err := c.Find(bson.M{"src": spider.Src}).One(&spider_); err != nil { // 不存在 @@ -106,11 +115,8 @@ func SaveSpiders(spiders []model.Spider) error { debug.PrintStack() return err } - } else { - // 存在 } } - return nil }