diff --git a/backend/main.go b/backend/main.go index e802ea78..955fb77c 100644 --- a/backend/main.go +++ b/backend/main.go @@ -141,6 +141,7 @@ func main() { authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改 authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建 + authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建 authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除 authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名 authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 diff --git a/backend/routes/spider.go b/backend/routes/spider.go index effa9cb3..d9b929a5 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -477,6 +477,12 @@ func PostSpiderFile(c *gin.Context) { return } + // 同步到GridFS + if err := services.UploadSpiderToGridFsFromMaster(spider); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + // 返回结果 c.JSON(http.StatusOK, Response{ Status: "ok", @@ -512,6 +518,52 @@ func PutSpiderFile(c *gin.Context) { return } + // 同步到GridFS + if err := services.UploadSpiderToGridFsFromMaster(spider); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func PutSpiderDir(c *gin.Context) { + spiderId := c.Param("id") + var reqBody SpiderFileReqBody + if err := c.ShouldBindJSON(&reqBody); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + spider, err := model.GetSpider(bson.ObjectIdHex(spiderId)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 文件路径 + filePath := path.Join(spider.Src, reqBody.Path) + + // 如果文件已存在,则报错 + if utils.Exists(filePath) { + HandleErrorF(http.StatusInternalServerError, c, fmt.Sprintf(`%s already exists`, filePath)) + return + } + + // 创建文件夹 + if err := os.MkdirAll(filePath, 0777); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 同步到GridFS + if err := services.UploadSpiderToGridFsFromMaster(spider); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + c.JSON(http.StatusOK, Response{ Status: "ok", Message: "success", @@ -535,6 +587,13 @@ func DeleteSpiderFile(c *gin.Context) { HandleError(http.StatusInternalServerError, c, err) return } + + // 同步到GridFS + if err := services.UploadSpiderToGridFsFromMaster(spider); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + c.JSON(http.StatusOK, Response{ Status: "ok", Message: "success", @@ -563,15 +622,8 @@ func RenameSpiderFile(c *gin.Context) { return } - // 读取原文件 - content, err := ioutil.ReadFile(filePath) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) - return - } - - // 写入新文件 - if err := ioutil.WriteFile(newFilePath, []byte(content), 0777); err != nil { + // 重命名 + if err := os.Rename(filePath, newFilePath); err != nil { HandleError(http.StatusInternalServerError, c, err) return } @@ -581,6 +633,12 @@ func RenameSpiderFile(c *gin.Context) { HandleError(http.StatusInternalServerError, c, err) } + // 同步到GridFS + if err := services.UploadSpiderToGridFsFromMaster(spider); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + c.JSON(http.StatusOK, Response{ Status: "ok", Message: "success", diff --git a/backend/services/spider.go b/backend/services/spider.go index 3922d822..3515afa9 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -12,6 +12,7 @@ import ( "github.com/apex/log" "github.com/globalsign/mgo" "github.com/globalsign/mgo/bson" + uuid "github.com/satori/go.uuid" "github.com/spf13/viper" "os" "path/filepath" @@ -30,6 +31,48 @@ type SpiderUploadMessage struct { SpiderId string } +// 从主节点上传爬虫到GridFS +func UploadSpiderToGridFsFromMaster(spider model.Spider) error { + // 爬虫所在目录 + spiderDir := spider.Src + + // 打包为 zip 文件 + files, err := utils.GetFilesFromDir(spiderDir) + if err != nil { + return err + } + randomId := uuid.NewV4() + tmpFilePath := filepath.Join(viper.GetString("other.tmppath"), spider.Name+"."+randomId.String()+".zip") + spiderZipFileName := spider.Name + ".zip" + if err := utils.Compress(files, tmpFilePath); err != nil { + return err + } + + // 获取 GridFS 实例 + s, gf := database.GetGridFs("files") + defer s.Close() + + // 判断文件是否已经存在 + var gfFile model.GridFs + if err := gf.Find(bson.M{"filename": spiderZipFileName}).One(&gfFile); err == nil { + // 已经存在文件,则删除 + _ = gf.RemoveId(gfFile.Id) + } + + // 上传到GridFs + fid, err := UploadToGridFs(spiderZipFileName, tmpFilePath) + if err != nil { + log.Errorf("upload to grid fs error: %s", err.Error()) + return err + } + + // 保存爬虫 FileId + spider.FileId = fid + _ = spider.Save() + + return nil +} + // 上传zip文件到GridFS func UploadToGridFs(fileName string, filePath string) (fid bson.ObjectId, err error) { fid = ""