diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 16480ffa..1436e744 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -259,14 +259,25 @@ func CopySpider(c *gin.Context) { return } + // 检查新爬虫名称是否存在 + // 如果存在，则返回错误 + s := model.GetSpiderByName(reqBody.Name) + if s.Name != "" { + HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("spider name '%s' already exists", reqBody.Name)) + return + } + + // 被复制爬虫 spider, err := model.GetSpider(bson.ObjectIdHex(id)) if err != nil { HandleError(http.StatusInternalServerError, c, err) return } + // 复制爬虫 if err := services.CopySpider(spider, reqBody.Name); err != nil { HandleError(http.StatusInternalServerError, c, err) + return } c.JSON(http.StatusOK, Response{ diff --git a/backend/services/spider.go b/backend/services/spider.go index 27805da4..392b0488 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -15,11 +15,13 @@ import ( "github.com/satori/go.uuid" "github.com/spf13/viper" "gopkg.in/yaml.v2" + "io" "io/ioutil" "os" "path" "path/filepath" "runtime/debug" + "time" ) type SpiderFileData struct { @@ -293,7 +295,121 @@ func CancelSpider(id string) error { return nil } +func cloneGridFsFile(spider model.Spider, newName string) (err error) { + // 构造新爬虫 + newSpider := spider + newSpider.Id = bson.NewObjectId() + newSpider.Name = newName + newSpider.DisplayName = newName + newSpider.Src = path.Join(path.Dir(spider.Src), newName) + newSpider.CreateTs = time.Now() + newSpider.UpdateTs = time.Now() + + // GridFS连接实例 + s, gf := database.GetGridFs("files") + defer s.Close() + + // 被克隆爬虫的GridFS文件 + f, err := gf.OpenId(spider.FileId) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // 新爬虫的GridFS文件 + fNew, err := gf.Create(newSpider.Name + ".zip") + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // 生成唯一ID + randomId := uuid.NewV4() + tmpPath := viper.GetString("other.tmppath") + if !utils.Exists(tmpPath) { + if err := os.MkdirAll(tmpPath, 0777); err != nil { + log.Errorf("mkdir other.tmppath error: %v", err.Error()) + return err + } + } + + // 创建临时文件 + tmpFilePath := filepath.Join(tmpPath, randomId.String()+".zip") + tmpFile := utils.OpenFile(tmpFilePath) + + // 拷贝到临时文件 + if _, err := io.Copy(tmpFile, f); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // 关闭临时文件 + if err := tmpFile.Close(); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // 读取内容 + fContent, err := ioutil.ReadFile(tmpFilePath) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // 写入GridFS文件 + if _, err := fNew.Write(fContent); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // 关闭被克隆爬虫GridFS文件 + if err = f.Close(); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // 将新爬虫文件复制 + newSpider.FileId = fNew.Id().(bson.ObjectId) + + // 保存新爬虫 + if err := newSpider.Add(); err != nil { + return err + } + + // 关闭新爬虫GridFS文件 + if err := fNew.Close(); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // 删除临时文件 + if err := os.RemoveAll(tmpFilePath); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // 同步爬虫 + PublishSpider(newSpider) + + return nil +} + func CopySpider(spider model.Spider, newName string) error { + // 克隆GridFS文件 + if err := cloneGridFsFile(spider, newName); err != nil { + return err + } + + return nil } // 启动爬虫服务 diff --git a/backend/services/task.go b/backend/services/task.go index 76aeed83..3bcdc8d6 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -536,7 +536,7 @@ func SpiderFileCheck(t model.Task, spider model.Spider) error { // 判断爬虫文件是否存在 gfFile := model.GetGridFs(spider.FileId) if gfFile == nil { - t.Error = "找不到爬虫文件，请重新上传" + t.Error = "cannot find spider files, please re-upload" t.Status = constants.StatusError t.FinishTs = time.Now() // 结束时间 t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长 diff --git a/frontend/src/components/Common/CrawlConfirmDialog.vue b/frontend/src/components/Common/CrawlConfirmDialog.vue index aa89ea9a..93d62d9d 100644 --- a/frontend/src/components/Common/CrawlConfirmDialog.vue +++ b/frontend/src/components/Common/CrawlConfirmDialog.vue @@ -78,8 +78,6 @@ 跳转到任务详情页 - - - +