diff --git a/backend/routes/spider.go b/backend/routes/spider.go index dc84f462..4d9b824a 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -187,19 +187,33 @@ func PutSpider(c *gin.Context) { // 将FileId置空 spider.FileId = bson.ObjectIdHex(constants.ObjectIdNull) - // 创建爬虫目录 + // 爬虫目录 spiderDir := filepath.Join(viper.GetString("spider.path"), spider.Name) + + // 赋值到爬虫实例 + spider.Src = spiderDir + + // 移除已有爬虫目录 if utils.Exists(spiderDir) { if err := os.RemoveAll(spiderDir); err != nil { HandleError(http.StatusInternalServerError, c, err) return } } + + // 生成爬虫目录 if err := os.MkdirAll(spiderDir, 0777); err != nil { HandleError(http.StatusInternalServerError, c, err) return } - spider.Src = spiderDir + + // 如果为 Scrapy 项目,生成 Scrapy 项目 + if spider.IsScrapy { + if err := services.CreateScrapyProject(spider); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + } // 添加爬虫到数据库 if err := spider.Add(); err != nil { diff --git a/backend/services/scrapy.go b/backend/services/scrapy.go index 374df4ee..52c316c3 100644 --- a/backend/services/scrapy.go +++ b/backend/services/scrapy.go @@ -145,6 +145,27 @@ func CreateScrapySpider(s model.Spider, name string, domain string) (err error) cmd.Stderr = &stderr if err := cmd.Run(); err != nil { log.Errorf(err.Error()) + log.Errorf("stdout: " + stdout.String()) + log.Errorf("stderr: " + stderr.String()) + debug.PrintStack() + return err + } + + return +} + +func CreateScrapyProject(s model.Spider) (err error) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + cmd := exec.Command("scrapy", "startproject", s.Name, s.Src) + cmd.Dir = s.Src + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + log.Errorf(err.Error()) + log.Errorf("stdout: " + stdout.String()) + log.Errorf("stderr: " + stderr.String()) debug.PrintStack() return err }