diff --git a/backend/routes/spider.go b/backend/routes/spider.go index e7faf051..c6ec2fbb 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -470,7 +470,7 @@ func UploadSpider(c *gin.Context) { } // 上传到GridFs - fid, err := services.UploadToGridFs(uploadFile.Filename, tmpFilePath) + fid, err := services.RetryUploadToGridFs(uploadFile.Filename, tmpFilePath) if err != nil { log.Errorf("upload to grid fs error: %s", err.Error()) debug.PrintStack() @@ -634,7 +634,7 @@ func UploadSpiderFromId(c *gin.Context) { } // 上传到GridFs - fid, err := services.UploadToGridFs(spider.Name, tmpFilePath) + fid, err := services.RetryUploadToGridFs(spider.Name, tmpFilePath) if err != nil { log.Errorf("upload to grid fs error: %s", err.Error()) debug.PrintStack() diff --git a/backend/services/config_spider.go b/backend/services/config_spider.go index 68c170df..c2135bed 100644 --- a/backend/services/config_spider.go +++ b/backend/services/config_spider.go @@ -223,7 +223,7 @@ func ProcessSpiderFilesFromConfigData(spider model.Spider, configData entity.Con } // 上传到GridFs - fid, err := UploadToGridFs(spiderZipFileName, tmpFilePath) + fid, err := RetryUploadToGridFs(spiderZipFileName, tmpFilePath) if err != nil { log.Errorf("upload to grid fs error: %s", err.Error()) return err diff --git a/backend/services/spider.go b/backend/services/spider.go index e8fc37c2..ec1d8441 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -8,6 +8,7 @@ import ( "crawlab/model" "crawlab/services/spider_handler" "crawlab/utils" + "errors" "fmt" "github.com/apex/log" "github.com/globalsign/mgo" @@ -69,10 +70,9 @@ func UploadSpiderToGridFsFromMaster(spider model.Spider) error { } // 上传到GridFs - fid, err := UploadToGridFs(spiderZipFileName, tmpFilePath) + fid, err := RetryUploadToGridFs(spiderZipFileName, tmpFilePath) if err != nil { log.Errorf("upload to grid fs error: %s", err.Error()) - return err } // 保存爬虫 FileId @@ -142,6 +142,26 @@ func UploadToGridFs(fileName string, filePath string) (fid bson.ObjectId, err er return fid, nil } +// 带重试功能的上传至 GridFS +func RetryUploadToGridFs(fileName string, filePath string) (fid bson.ObjectId, err error) { + maxErrCount := 10 + errCount := 0 + for { + if errCount > maxErrCount { + break + } + fid, err = UploadToGridFs(fileName, filePath) + if err != nil { + errCount++ + log.Errorf("upload to grid fs error: %s", err.Error()) + time.Sleep(3 * time.Second) + continue + } + return fid, nil + } + return fid, errors.New("unable to upload to gridfs, please re-upload the spider") +} + // 写入grid fs func WriteToGridFS(content []byte, f *mgo.GridFile) { if _, err := f.Write(content); err != nil {