From 41e9edf940195e89a256a9faab34c6f45c179a25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com>
Date: Thu, 26 Sep 2019 20:53:05 +0800
Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90=E7=88=AC=E8=99=AB=E5=88=97?=
 =?UTF-8?q?=E8=A1=A8?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/model/spider.go                   |  8 +++++---
 backend/routes/spider.go                  | 11 +++++++++--
 backend/services/spider.go                |  5 ++---
 backend/services/spider_handler/spider.go |  3 ---
 4 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/backend/model/spider.go b/backend/model/spider.go
index dd7d505c..88fdc3ca 100644
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -93,7 +93,7 @@ func (spider *Spider) GetLastTask() (Task, error) {
 	return tasks[0], nil
 }
 
-func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, error) {
+func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, int, error) {
 	s, c := database.GetCol("spiders")
 	defer s.Close()
 
@@ -101,7 +101,7 @@ func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, error) {
 	spiders := []Spider{}
 	if err := c.Find(filter).Skip(skip).Limit(limit).Sort("+name").All(&spiders); err != nil {
 		debug.PrintStack()
-		return spiders, err
+		return spiders, 0, err
 	}
 
 	// 遍历爬虫列表
@@ -119,7 +119,9 @@ func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, error) {
 		spiders[i].LastStatus = task.Status
 	}
 
-	return spiders, nil
+	count, _ := c.Find(filter).Count()
+
+	return spiders, count, nil
 }
 
 func GetSpiderByName(name string) *Spider {
diff --git a/backend/routes/spider.go b/backend/routes/spider.go
index 76f89bf7..0e68bc54 100644
--- a/backend/routes/spider.go
+++ b/backend/routes/spider.go
@@ -24,7 +24,14 @@ import (
 )
 
 func GetSpiderList(c *gin.Context) {
-	results, err := model.GetSpiderList(nil, 0, 0)
+	pageNumStr, _ := c.GetQuery("pageNum")
+	pageSizeStr, _ := c.GetQuery("pageSize")
+	keyword, _ := c.GetQuery("keyword")
+	pageNum, _ := strconv.Atoi(pageNumStr)
+	pageSize, _ := strconv.Atoi(pageSizeStr)
+	skip := pageSize * (pageNum - 1)
+	filter := bson.M{"name": bson.M{"$regex": bson.RegEx{Pattern: keyword, Options: "im"}}}
+	results, count, err := model.GetSpiderList(filter, skip, pageSize)
 	if err != nil {
 		HandleError(http.StatusInternalServerError, c, err)
 		return
@@ -32,7 +39,7 @@ func GetSpiderList(c *gin.Context) {
 	c.JSON(http.StatusOK, Response{
 		Status:  "ok",
 		Message: "success",
-		Data:    results,
+		Data:    bson.M{"list": results, "total": count},
 	})
 }
 
diff --git a/backend/services/spider.go b/backend/services/spider.go
index 90925346..3e87eef3 100644
--- a/backend/services/spider.go
+++ b/backend/services/spider.go
@@ -97,7 +97,7 @@ func ReadFileByStep(filePath string, handle func([]byte, *mgo.GridFile), fileCre
 // 发布所有爬虫
 func PublishAllSpiders() {
 	// 获取爬虫列表
-	spiders, _ := model.GetSpiderList(nil, 0, constants.Infinite)
+	spiders, _, _ := model.GetSpiderList(nil, 0, constants.Infinite)
 	if len(spiders) == 0 {
 		return
 	}
@@ -143,7 +143,6 @@ func PublishSpider(spider model.Spider) {
 	// md5值不一样，则下载
 	md5Str := utils.ReadFile(md5)
 	if gfFile.Md5 != md5Str {
-		log.Infof("md5 is different, fileName=%s,  file-md5=%s , gf-file-md5=%s ", spider.Name, md5Str, gfFile.Md5)
 		spiderSync.RemoveSpiderFile()
 		spiderSync.Download()
 		spiderSync.CreateMd5File(gfFile.Md5)
@@ -155,7 +154,7 @@ func PublishSpider(spider model.Spider) {
 func InitSpiderService() error {
 	// 构造定时任务执行器
 	c := cron.New(cron.WithSeconds())
-	if _, err := c.AddFunc("0/15 * * * * *", PublishAllSpiders); err != nil {
+	if _, err := c.AddFunc("0 * * * * *", PublishAllSpiders); err != nil {
 		return err
 	}
 	// 启动定时任务
diff --git a/backend/services/spider_handler/spider.go b/backend/services/spider_handler/spider.go
index 87dedeb0..1d8a8ae1 100644
--- a/backend/services/spider_handler/spider.go
+++ b/backend/services/spider_handler/spider.go
@@ -56,7 +56,6 @@ func (s *SpiderSync) RemoveSpiderFile() {
 func (s *SpiderSync) CheckDownLoading(spiderId string, fileId string) (bool, string) {
 	key := s.GetLockDownloadKey(spiderId)
 	if _, err := database.RedisClient.HGet("spider", key); err == nil {
-		log.Infof("downloading spider file, spider_id: %s, file_id:%s", spiderId, fileId)
 		return true, key
 	}
 	return false, key
@@ -68,7 +67,6 @@ func (s *SpiderSync) Download() {
 	fileId := s.Spider.FileId.Hex()
 	isDownloading, key := s.CheckDownLoading(spiderId, fileId)
 	if isDownloading {
-		log.Infof("spider is downloading, spider_id: %s", spiderId)
 		return
 	} else {
 		_ = database.RedisClient.HSet("spider", key, key)
@@ -131,6 +129,5 @@ func (s *SpiderSync) Download() {
 		return
 	}
 
-	log.Infof("del key : %s", key)
 	_ = database.RedisClient.HDel("spider", key)
 }