From d7437c4eb1fe4d86d29f1d40f8b65f9b10f4a8a3 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 21 Feb 2020 17:12:57 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5scrapy=20pipelines=E6=94=AF?= =?UTF-8?q?=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/main.go | 1 + backend/routes/spider.go | 27 +++++++++++++++ backend/services/scrapy.go | 24 +++++++++++++ .../src/components/Scrapy/SpiderScrapy.vue | 34 ++++++++++++++----- frontend/src/store/modules/spider.js | 13 +++++++ frontend/src/views/spider/SpiderDetail.vue | 3 +- 6 files changed, 93 insertions(+), 9 deletions(-) diff --git a/backend/main.go b/backend/main.go index 63c19229..db96c0b7 100644 --- a/backend/main.go +++ b/backend/main.go @@ -180,6 +180,7 @@ func main() { authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings) // Scrapy 爬虫修改设置 authGroup.GET("/spiders/:id/scrapy/items", routes.GetSpiderScrapyItems) // Scrapy 爬虫 items authGroup.POST("/spiders/:id/scrapy/items", routes.PostSpiderScrapyItems) // Scrapy 爬虫修改 items + authGroup.GET("/spiders/:id/scrapy/pipelines", routes.GetSpiderScrapyPipelines) // Scrapy 爬虫 pipelines authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步 authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置 } diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 7b86469d..a16dbddc 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -1125,6 +1125,33 @@ func PostSpiderScrapyItems(c *gin.Context) { }) } +func GetSpiderScrapyPipelines(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid") + return + } + + spider, err := model.GetSpider(bson.ObjectIdHex(id)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + data, err := services.GetScrapyPipelines(spider) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: data, + }) +} + func PostSpiderSyncGit(c *gin.Context) { id := c.Param("id") diff --git a/backend/services/scrapy.go b/backend/services/scrapy.go index 5a7c4d4e..f58adb5f 100644 --- a/backend/services/scrapy.go +++ b/backend/services/scrapy.go @@ -201,6 +201,30 @@ func SaveScrapyItems(s model.Spider, itemsData []entity.ScrapyItem) (err error) return } +func GetScrapyPipelines(s model.Spider) (res []string, err error) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + cmd := exec.Command("crawlab", "pipelines") + cmd.Dir = s.Src + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + log.Errorf(err.Error()) + log.Errorf(stderr.String()) + debug.PrintStack() + return res, err + } + + if err := json.Unmarshal([]byte(stdout.String()), &res); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return res, err + } + + return res, nil +} + func CreateScrapySpider(s model.Spider, name string, domain string, template string) (err error) { var stdout bytes.Buffer var stderr bytes.Buffer diff --git a/frontend/src/components/Scrapy/SpiderScrapy.vue b/frontend/src/components/Scrapy/SpiderScrapy.vue index 0dcff280..7fd28bda 100644 --- a/frontend/src/components/Scrapy/SpiderScrapy.vue +++ b/frontend/src/components/Scrapy/SpiderScrapy.vue @@ -243,13 +243,13 @@ {{$t('Add Spider')}} -