diff --git a/backend/main.go b/backend/main.go index 63c19229..db96c0b7 100644 --- a/backend/main.go +++ b/backend/main.go @@ -180,6 +180,7 @@ func main() { authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings) // Scrapy 爬虫修改设置 authGroup.GET("/spiders/:id/scrapy/items", routes.GetSpiderScrapyItems) // Scrapy 爬虫 items authGroup.POST("/spiders/:id/scrapy/items", routes.PostSpiderScrapyItems) // Scrapy 爬虫修改 items + authGroup.GET("/spiders/:id/scrapy/pipelines", routes.GetSpiderScrapyPipelines) // Scrapy 爬虫 pipelines authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步 authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置 } diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 7b86469d..a16dbddc 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -1125,6 +1125,33 @@ func PostSpiderScrapyItems(c *gin.Context) { }) } +func GetSpiderScrapyPipelines(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid") + return + } + + spider, err := model.GetSpider(bson.ObjectIdHex(id)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + data, err := services.GetScrapyPipelines(spider) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: data, + }) +} + func PostSpiderSyncGit(c *gin.Context) { id := c.Param("id") diff --git a/backend/services/scrapy.go b/backend/services/scrapy.go index 5a7c4d4e..f58adb5f 100644 --- a/backend/services/scrapy.go +++ b/backend/services/scrapy.go @@ -201,6 +201,30 @@ func SaveScrapyItems(s model.Spider, itemsData []entity.ScrapyItem) (err error) return } +func GetScrapyPipelines(s model.Spider) (res []string, err error) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + cmd := exec.Command("crawlab", "pipelines") + cmd.Dir = s.Src + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + log.Errorf(err.Error()) + log.Errorf(stderr.String()) + debug.PrintStack() + return res, err + } + + if err := json.Unmarshal([]byte(stdout.String()), &res); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return res, err + } + + return res, nil +} + func CreateScrapySpider(s model.Spider, name string, domain string, template string) (err error) { var stdout bytes.Buffer var stderr bytes.Buffer diff --git a/frontend/src/components/Scrapy/SpiderScrapy.vue b/frontend/src/components/Scrapy/SpiderScrapy.vue index 0dcff280..7fd28bda 100644 --- a/frontend/src/components/Scrapy/SpiderScrapy.vue +++ b/frontend/src/components/Scrapy/SpiderScrapy.vue @@ -243,13 +243,13 @@ {{$t('Add Spider')}} -