From dc38642dca648444f2532ecc25bcec3ae3735888 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 17 Feb 2020 11:15:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5settings.py=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/constants/variable.go | 9 ++ backend/entity/spider.go | 6 + backend/go.mod | 3 +- backend/go.sum | 2 + backend/main.go | 43 +++---- backend/routes/spider.go | 32 +++++ backend/services/scrapy.go | 71 +++++++++++ .../vendor/github.com/royeo/dingrobot/LICENSE | 21 ---- .../github.com/royeo/dingrobot/README.md | 110 ---------------- .../github.com/royeo/dingrobot/dingrobot.go | 118 ------------------ .../github.com/royeo/dingrobot/message.go | 60 --------- backend/vendor/modules.txt | 4 +- .../src/components/Scrapy/SpiderScrapy.vue | 6 +- frontend/src/store/modules/spider.js | 3 + 14 files changed, 154 insertions(+), 334 deletions(-) create mode 100644 backend/constants/variable.go delete mode 100644 backend/vendor/github.com/royeo/dingrobot/LICENSE delete mode 100644 backend/vendor/github.com/royeo/dingrobot/README.md delete mode 100644 backend/vendor/github.com/royeo/dingrobot/dingrobot.go delete mode 100644 backend/vendor/github.com/royeo/dingrobot/message.go diff --git a/backend/constants/variable.go b/backend/constants/variable.go new file mode 100644 index 00000000..713fbe2d --- /dev/null +++ b/backend/constants/variable.go @@ -0,0 +1,9 @@ +package constants + +const ( + String = "string" + Number = "number" + Boolean = "boolean" + Array = "array" + Object = "object" +) diff --git a/backend/entity/spider.go b/backend/entity/spider.go index 7f5e02b4..6f8fbee1 100644 --- a/backend/entity/spider.go +++ b/backend/entity/spider.go @@ -4,3 +4,9 @@ type SpiderType struct { Type string `json:"type" bson:"_id"` Count int `json:"count" bson:"count"` } + +type ScrapySettingParam struct { + Key string + Value interface{} + Type string +} diff --git a/backend/go.mod b/backend/go.mod index cbc7d75b..24f64668 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -3,6 +3,7 @@ module crawlab go 1.12 require ( + github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd github.com/apex/log v1.1.1 github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/fsnotify/fsnotify v1.4.7 @@ -16,7 +17,7 @@ require ( github.com/matcornic/hermes v1.2.0 github.com/matcornic/hermes/v2 v2.0.2 // indirect github.com/pkg/errors v0.8.1 - github.com/royeo/dingrobot v1.0.0 + github.com/royeo/dingrobot v1.0.0 // indirect github.com/satori/go.uuid v1.2.0 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 github.com/spf13/viper v1.4.0 diff --git a/backend/go.sum b/backend/go.sum index a55ef74e..15458004 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -6,6 +6,8 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY= github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd h1:+CYOsXi89xOqBkj7CuEJjA2It+j+R3ngUZEydr6mtkw= +github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= diff --git a/backend/main.go b/backend/main.go index 758525aa..e295440a 100644 --- a/backend/main.go +++ b/backend/main.go @@ -154,27 +154,28 @@ func main() { } // 爬虫 { - authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 - authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 - authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫 - authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫 - authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫 - authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 - authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID） - authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 - authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 - authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取 - authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 - authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改 - authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建 - authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建 - authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除 - authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名 - authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 - authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 - authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务 - authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表 - authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings) // Scrapy 爬虫设置 + authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 + authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 + authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫 + authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫 + authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫 + authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 + authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID） + authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 + authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 + authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取 + authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 + authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改 + authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建 + authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建 + authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除 + authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名 + authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 + authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 + authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务 + authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表 + authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings) // Scrapy 爬虫设置 + authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings) // Scrapy 爬虫修改设置 } // 可配置爬虫 { diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 473fb480..2748a02e 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -956,3 +956,35 @@ func GetSpiderScrapySettings(c *gin.Context) { Data: data, }) } + +func PostSpiderScrapySettings(c *gin.Context) { + + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid") + return + } + + var reqData []entity.ScrapySettingParam + if err := c.ShouldBindJSON(&reqData); err != nil { + HandleErrorF(http.StatusBadRequest, c, "invalid request") + return + } + + spider, err := model.GetSpider(bson.ObjectIdHex(id)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + if err := services.SaveScrapySettings(spider, reqData); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} diff --git a/backend/services/scrapy.go b/backend/services/scrapy.go index f6e46820..ebe615cd 100644 --- a/backend/services/scrapy.go +++ b/backend/services/scrapy.go @@ -2,10 +2,17 @@ package services import ( "bytes" + "crawlab/constants" + "crawlab/entity" "crawlab/model" "encoding/json" + "fmt" + "github.com/Unknwon/goconfig" "github.com/apex/log" + "io/ioutil" + "os" "os/exec" + "path" "runtime/debug" "strings" ) @@ -60,3 +67,67 @@ func GetScrapySettings(s model.Spider) (res []map[string]interface{}, err error) return res, nil } + +func SaveScrapySettings(s model.Spider, settingsData []entity.ScrapySettingParam) (err error) { + // 读取 scrapy.cfg + cfg, err := goconfig.LoadConfigFile(path.Join(s.Src, "scrapy.cfg")) + if err != nil { + return + } + modName, err := cfg.GetValue("settings", "default") + if err != nil { + return + } + + // 定位到 settings.py 文件 + arr := strings.Split(modName, ".") + dirName := arr[0] + fileName := arr[1] + filePath := fmt.Sprintf("%s/%s/%s.py", s.Src, dirName, fileName) + + // 生成文件内容 + content := "" + for _, param := range settingsData { + var line string + switch param.Type { + case constants.String: + line = fmt.Sprintf("%s = '%s'", param.Key, param.Value) + case constants.Number: + line = fmt.Sprintf("%s = %s", param.Key, param.Value) + case constants.Boolean: + if param.Value.(bool) { + line = fmt.Sprintf("%s = %s", param.Key, "True") + } else { + line = fmt.Sprintf("%s = %s", param.Key, "False") + } + case constants.Array: + arr := param.Value.([]interface{}) + var arrStr []string + for _, s := range arr { + arrStr = append(arrStr, s.(string)) + } + line = fmt.Sprintf("%s = ['%s']", param.Key, strings.Join(arrStr, "','")) + case constants.Object: + value := param.Value.(map[string]interface{}) + var arr []string + for k, v := range value { + str := v.(float64) + arr = append(arr, fmt.Sprintf("'%s': %.0f", k, str)) + } + line = fmt.Sprintf("%s = {%s}", param.Key, strings.Join(arr, ",")) + } + content += line + "\n" + } + + // 写到 settings.py + if err := ioutil.WriteFile(filePath, []byte(content), os.ModePerm); err != nil { + return err + } + + // 同步到GridFS + if err := UploadSpiderToGridFsFromMaster(s); err != nil { + return err + } + + return +} diff --git a/backend/vendor/github.com/royeo/dingrobot/LICENSE b/backend/vendor/github.com/royeo/dingrobot/LICENSE deleted file mode 100644 index e2c20381..00000000 --- a/backend/vendor/github.com/royeo/dingrobot/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2018 Royeo - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/backend/vendor/github.com/royeo/dingrobot/README.md b/backend/vendor/github.com/royeo/dingrobot/README.md deleted file mode 100644 index c76c6313..00000000 --- a/backend/vendor/github.com/royeo/dingrobot/README.md +++ /dev/null @@ -1,110 +0,0 @@ -# Dingrobot - -钉钉自定义机器人 Golang API. - -支持的消息类型： -- 文本类型 -- link 类型 -- markdown 类型 -- 整体跳转 ActionCard 类型 - -## Installation - -Install: - -```sh -go get -u github.com/royeo/dingrobot -``` - -Import: - -```go -import "github.com/royeo/dingrobot" -``` - -## Quick start - -发送文本类型的消息： - -```go -func main() { - // You should replace the webhook here with your own. - webhook := "https://oapi.dingtalk.com/robot/send?access_token=xxx" - robot := dingrobot.NewRobot(webhook) - - content := "我就是我, @1825718XXXX 是不一样的烟火" - atMobiles := []string{"1825718XXXX"} - isAtAll := false - - err := robot.SendText(content, atMobiles, isAtAll) - if err != nil { - log.Fatal(err) - } -} -``` - -发送 link 类型的消息： - -```go -func main() { - // You should replace the webhook here with your own. - webhook := "https://oapi.dingtalk.com/robot/send?access_token=xxx" - robot := dingrobot.NewRobot(webhook) - - title := "自定义机器人协议" - text := "群机器人是钉钉群的高级扩展功能。群机器人可以将第三方服务的信息聚合到群聊中，实现自动化的信息同步。例如：通过聚合GitHub，GitLab等源码管理服务，实现源码更新同步；通过聚合Trello，JIRA等项目协调服务，实现项目信息同步。不仅如此，群机器人支持Webhook协议的自定义接入，支持更多可能性，例如：你可将运维报警提醒通过自定义机器人聚合到钉钉群。" - messageUrl := "https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7629140.0.0.Rqyvqo&treeId=257&articleId=105735&docType=1" - picUrl := "" - - err := robot.SendLink(title, text, messageUrl, picUrl) - if err != nil { - log.Fatal(err) - } -} -``` - -发送 markdown 类型的消息： - -```go -func main() { - // You should replace the webhook here with your own. - webhook := "https://oapi.dingtalk.com/robot/send?access_token=xxx" - robot := dingrobot.NewRobot(webhook) - - title := "杭州天气" - text := "#### 杭州天气  \n > 9度，@1825718XXXX 西北风1级，空气良89，相对温度73%\n\n > ![screenshot](http://i01.lw.aliimg.com/media/lALPBbCc1ZhJGIvNAkzNBLA_1200_588.png)\n  > ###### 10点20分发布 [天气](http://www.thinkpage.cn/) " - atMobiles := []string{"1825718XXXX"} - isAtAll := false - - err := robot.SendMarkdown(title, text, atMobiles, isAtAll) - if err != nil { - log.Fatal(err) - } -} -``` - -发送整体跳转 ActionCard 类型的消息： - -```go -func main() { - // You should replace the webhook here with your own. - webhook := "https://oapi.dingtalk.com/robot/send?access_token=xxx" - robot := dingrobot.NewRobot(webhook) - - title := "乔布斯 20 年前想打造一间苹果咖啡厅，而它正是 Apple Store 的前身" - text := "![screenshot](@lADOpwk3K80C0M0FoA) \n #### 乔布斯 20 年前想打造的苹果咖啡厅 \n\n Apple Store 的设计正从原来满满的科技感走向生活化，而其生活化的走向其实可以追溯到 20 年前苹果一个建立咖啡馆的计划" - singleTitle := "阅读全文" - singleURL := "https://www.dingtalk.com/" - btnOrientation := "0" - hideAvatar := "0" - - err := robot.SendActionCard(title, text, singleTitle, singleURL, btnOrientation, hideAvatar) - if err != nil { - log.Fatal(err) - } -} -``` - -## License - -MIT Copyright (c) 2018 Royeo diff --git a/backend/vendor/github.com/royeo/dingrobot/dingrobot.go b/backend/vendor/github.com/royeo/dingrobot/dingrobot.go deleted file mode 100644 index cd5b65b0..00000000 --- a/backend/vendor/github.com/royeo/dingrobot/dingrobot.go +++ /dev/null @@ -1,118 +0,0 @@ -package dingrobot - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" -) - -// Roboter is the interface implemented by Robot that can send multiple types of messages. -type Roboter interface { - SendText(content string, atMobiles []string, isAtAll bool) error - SendLink(title, text, messageURL, picURL string) error - SendMarkdown(title, text string, atMobiles []string, isAtAll bool) error - SendActionCard(title, text, singleTitle, singleURL, btnOrientation, hideAvatar string) error -} - -// Robot represents a dingtalk custom robot that can send messages to groups. -type Robot struct { - Webhook string -} - -// NewRobot returns a roboter that can send messages. -func NewRobot(webhook string) Roboter { - return Robot{Webhook: webhook} -} - -// SendText send a text type message. -func (r Robot) SendText(content string, atMobiles []string, isAtAll bool) error { - return r.send(&textMessage{ - MsgType: msgTypeText, - Text: textParams{ - Content: content, - }, - At: atParams{ - AtMobiles: atMobiles, - IsAtAll: isAtAll, - }, - }) -} - -// SendLink send a link type message. -func (r Robot) SendLink(title, text, messageURL, picURL string) error { - return r.send(&linkMessage{ - MsgType: msgTypeLink, - Link: linkParams{ - Title: title, - Text: text, - MessageURL: messageURL, - PicURL: picURL, - }, - }) -} - -// SendMarkdown send a markdown type message. -func (r Robot) SendMarkdown(title, text string, atMobiles []string, isAtAll bool) error { - return r.send(&markdownMessage{ - MsgType: msgTypeMarkdown, - Markdown: markdownParams{ - Title: title, - Text: text, - }, - At: atParams{ - AtMobiles: atMobiles, - IsAtAll: isAtAll, - }, - }) -} - -// SendActionCard send a action card type message. -func (r Robot) SendActionCard(title, text, singleTitle, singleURL, btnOrientation, hideAvatar string) error { - return r.send(&actionCardMessage{ - MsgType: msgTypeActionCard, - ActionCard: actionCardParams{ - Title: title, - Text: text, - SingleTitle: singleTitle, - SingleURL: singleURL, - BtnOrientation: btnOrientation, - HideAvatar: hideAvatar, - }, - }) -} - -type dingResponse struct { - Errcode int - Errmsg string -} - -func (r Robot) send(msg interface{}) error { - m, err := json.Marshal(msg) - if err != nil { - return err - } - - resp, err := http.Post(r.Webhook, "application/json", bytes.NewReader(m)) - if err != nil { - return err - } - defer resp.Body.Close() - - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - var dr dingResponse - err = json.Unmarshal(data, &dr) - if err != nil { - return err - } - if dr.Errcode != 0 { - return fmt.Errorf("dingrobot send failed: %v", dr.Errmsg) - } - - return nil -} diff --git a/backend/vendor/github.com/royeo/dingrobot/message.go b/backend/vendor/github.com/royeo/dingrobot/message.go deleted file mode 100644 index 82f286f8..00000000 --- a/backend/vendor/github.com/royeo/dingrobot/message.go +++ /dev/null @@ -1,60 +0,0 @@ -package dingrobot - -const ( - msgTypeText = "text" - msgTypeLink = "link" - msgTypeMarkdown = "markdown" - msgTypeActionCard = "actionCard" -) - -type textMessage struct { - MsgType string `json:"msgtype"` - Text textParams `json:"text"` - At atParams `json:"at"` -} - -type textParams struct { - Content string `json:"content"` -} - -type atParams struct { - AtMobiles []string `json:"atMobiles,omitempty"` - IsAtAll bool `json:"isAtAll,omitempty"` -} - -type linkMessage struct { - MsgType string `json:"msgtype"` - Link linkParams `json:"link"` -} - -type linkParams struct { - Title string `json:"title"` - Text string `json:"text"` - MessageURL string `json:"messageUrl"` - PicURL string `json:"picUrl,omitempty"` -} - -type markdownMessage struct { - MsgType string `json:"msgtype"` - Markdown markdownParams `json:"markdown"` - At atParams `json:"at"` -} - -type markdownParams struct { - Title string `json:"title"` - Text string `json:"text"` -} - -type actionCardMessage struct { - MsgType string `json:"msgtype"` - ActionCard actionCardParams `json:"actionCard"` -} - -type actionCardParams struct { - Title string `json:"title"` - Text string `json:"text"` - SingleTitle string `json:"singleTitle"` - SingleURL string `json:"singleURL"` - BtnOrientation string `json:"btnOrientation,omitempty"` - HideAvatar string `json:"hideAvatar,omitempty"` -} diff --git a/backend/vendor/modules.txt b/backend/vendor/modules.txt index dd2a41b3..4d0a5c01 100644 --- a/backend/vendor/modules.txt +++ b/backend/vendor/modules.txt @@ -2,6 +2,8 @@ github.com/Masterminds/semver # github.com/Masterminds/sprig v2.16.0+incompatible github.com/Masterminds/sprig +# github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd +github.com/Unknwon/goconfig # github.com/aokoli/goutils v1.0.1 github.com/aokoli/goutils # github.com/apex/log v1.1.1 @@ -82,8 +84,6 @@ github.com/olekukonko/tablewriter github.com/pelletier/go-toml # github.com/pkg/errors v0.8.1 github.com/pkg/errors -# github.com/royeo/dingrobot v1.0.0 -github.com/royeo/dingrobot # github.com/satori/go.uuid v1.2.0 github.com/satori/go.uuid # github.com/shurcooL/sanitized_anchor_name v1.0.0 diff --git a/frontend/src/components/Scrapy/SpiderScrapy.vue b/frontend/src/components/Scrapy/SpiderScrapy.vue index 852faf97..0843b8ca 100644 --- a/frontend/src/components/Scrapy/SpiderScrapy.vue +++ b/frontend/src/components/Scrapy/SpiderScrapy.vue @@ -244,7 +244,11 @@ export default { this.activeParamIndex = index this.onOpenDialog() }, - onSave () { + async onSave () { + const res = await this.$store.dispatch('spider/saveSpiderScrapySettings', this.$route.params.id) + if (!res.data.error) { + this.$message.success(this.$t('Saved successfully')) + } }, onAdd () { const data = JSON.parse(JSON.stringify(this.spiderScrapySettings)) diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js index 796ade0e..d93679d6 100644 --- a/frontend/src/store/modules/spider.js +++ b/frontend/src/store/modules/spider.js @@ -147,6 +147,9 @@ const actions = { } })) }, + async saveSpiderScrapySettings ({ state }, id) { + return request.post(`/spiders/${id}/scrapy/settings`, state.spiderScrapySettings) + }, crawlSpider ({ state, dispatch }, payload) { const { spiderId, runType, nodeIds, param } = payload return request.put(`/tasks`, {