From 93dd3b714a7383aefaefb55b1a3e5a4cc38b1d6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com>
Date: Mon, 2 Sep 2019 17:37:48 +0800
Subject: [PATCH] =?UTF-8?q?fix=20=E5=A6=82=E6=9E=9C=E4=BB=8Edir=E8=AF=BB?=
 =?UTF-8?q?=E5=8F=96=E7=88=AC=E8=99=AB=E4=B8=BA=E7=A9=BA=EF=BC=8C=E5=88=99?=
 =?UTF-8?q?=E7=A7=BB=E9=99=A4=E6=89=80=E6=9C=89=E7=9A=84=E7=88=AC=E8=99=AB?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/model/spider.go    | 18 ++++++++++++++++++
 backend/services/spider.go | 20 +++++++++++++-------
 2 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/backend/model/spider.go b/backend/model/spider.go
index 2ba4c8c5..e0e5f836 100644
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -177,6 +177,24 @@ func RemoveSpider(id bson.ObjectId) error {
 	return nil
 }
 
+func RemoveAllSpider() error {
+	s, c := database.GetCol("spiders")
+	defer s.Close()
+
+	spiders := []Spider{}
+	err := c.Find(nil).All(&spiders)
+	if err != nil {
+		log.Error("get all spiders error:" + err.Error())
+		return err
+	}
+	for _, spider := range spiders {
+		if err := RemoveSpider(spider.Id); err != nil {
+			log.Error("remove spider error:" + err.Error())
+		}
+	}
+	return nil
+}
+
 func GetSpiderCount() (int, error) {
 	s, c := database.GetCol("spiders")
 	defer s.Close()
diff --git a/backend/services/spider.go b/backend/services/spider.go
index 47c1fa33..e75b0418 100644
--- a/backend/services/spider.go
+++ b/backend/services/spider.go
@@ -89,16 +89,25 @@ func GetSpidersFromDir() ([]model.Spider, error) {
 
 // 将爬虫保存到数据库
 func SaveSpiders(spiders []model.Spider) error {
-	// 遍历爬虫列表
+	s, c := database.GetCol("spiders")
+	defer s.Close()
+
+	if len(spiders) == 0 {
+		err := model.RemoveAllSpider()
+		if err != nil {
+			log.Error("remove all spider error:" + err.Error())
+			return err
+		}
+		log.Info("get spider from dir is empty,removed all spider")
+		return nil
+	}
+	// 如果该爬虫不存在于数据库，则保存爬虫到数据库
 	for _, spider := range spiders {
 		// 忽略非自定义爬虫
 		if spider.Type != constants.Customized {
 			continue
 		}
 
-		// 如果该爬虫不存在于数据库，则保存爬虫到数据库
-		s, c := database.GetCol("spiders")
-		defer s.Close()
 		var spider_ *model.Spider
 		if err := c.Find(bson.M{"src": spider.Src}).One(&spider_); err != nil {
 			// 不存在
@@ -106,11 +115,8 @@ func SaveSpiders(spiders []model.Spider) error {
 				debug.PrintStack()
 				return err
 			}
-		} else {
-			// 存在
 		}
 	}
-
 	return nil
 }