From 23f0b0c9e22b09422fe818f46d00f1f15470cf67 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Mon, 3 Feb 2020 16:08:43 +0800
Subject: [PATCH] fixed unable to sync spiders to nodes error

---
 backend/services/spider.go                |  2 ++
 backend/services/spider_handler/spider.go | 14 ++++++++++----
 backend/utils/file.go                     |  2 +-
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/backend/services/spider.go b/backend/services/spider.go
index 36872460..fb785d85 100644
--- a/backend/services/spider.go
+++ b/backend/services/spider.go
@@ -368,6 +368,8 @@ func InitSpiderService() error {
 			}
 		}
 
+		// 发布所有爬虫
+		PublishAllSpiders()
 	}
 
 	return nil
diff --git a/backend/services/spider_handler/spider.go b/backend/services/spider_handler/spider.go
index cd8a1dbe..ddc94b57 100644
--- a/backend/services/spider_handler/spider.go
+++ b/backend/services/spider_handler/spider.go
@@ -4,6 +4,7 @@ import (
 	"crawlab/database"
 	"crawlab/model"
 	"crawlab/utils"
+	"fmt"
 	"github.com/apex/log"
 	"github.com/globalsign/mgo/bson"
 	"github.com/satori/go.uuid"
@@ -25,7 +26,7 @@ type SpiderSync struct {
 
 func (s *SpiderSync) CreateMd5File(md5 string) {
 	path := filepath.Join(viper.GetString("spider.path"), s.Spider.Name)
-	utils.CreateFilePath(path)
+	utils.CreateDirPath(path)
 
 	fileName := filepath.Join(path, Md5File)
 	file := utils.OpenFile(fileName)
@@ -66,10 +67,14 @@ func (s *SpiderSync) RemoveSpiderFile() {
 // 检测是否已经下载中
 func (s *SpiderSync) CheckDownLoading(spiderId string, fileId string) (bool, string) {
 	key := s.GetLockDownloadKey(spiderId)
-	if _, err := database.RedisClient.HGet("spider", key); err == nil {
-		return true, key
+	key2, err := database.RedisClient.HGet("spider", key)
+	if err != nil {
+		return false, key2
 	}
-	return false, key
+	if key2 == "" {
+		return false, key2
+	}
+	return true, key2
 }
 
 // 下载爬虫
@@ -78,6 +83,7 @@ func (s *SpiderSync) Download() {
 	fileId := s.Spider.FileId.Hex()
 	isDownloading, key := s.CheckDownLoading(spiderId, fileId)
 	if isDownloading {
+		log.Infof(fmt.Sprintf("spider is already being downloaded, spider id: %s", s.Spider.Id.Hex()))
 		return
 	} else {
 		_ = database.RedisClient.HSet("spider", key, key)
diff --git a/backend/utils/file.go b/backend/utils/file.go
index bfe92bd3..072930cf 100644
--- a/backend/utils/file.go
+++ b/backend/utils/file.go
@@ -55,7 +55,7 @@ func OpenFile(fileName string) *os.File {
 }
 
 // 创建文件夹
-func CreateFilePath(filePath string) {
+func CreateDirPath(filePath string) {
 	if !Exists(filePath) {
 		if err := os.MkdirAll(filePath, os.ModePerm); err != nil {
 			log.Errorf("create file error: %s, file_path: %s", err.Error(), filePath)