git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

加入settings.py同步

Browse Source
This commit is contained in:
marvzhang
2020-02-17 11:15:16 +08:00
parent cdaf3363c8
commit dc38642dca
14 changed files with 154 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
backend/constants/variable.go Normal file
View File

@@ -0,0 +1,9 @@
package constants
const (
String = "string"
Number = "number"
Boolean = "boolean"
Array = "array"
Object = "object"
)

6
backend/entity/spider.go
View File

@@ -4,3 +4,9 @@ type SpiderType struct {
Type string `json:"type" bson:"_id"`
Count int `json:"count" bson:"count"`
}
type ScrapySettingParam struct {
Key string
Value interface{}
Type string
}

3
backend/go.mod
View File

@@ -3,6 +3,7 @@ module crawlab
go 1.12
require (
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd
github.com/apex/log v1.1.1
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/fsnotify/fsnotify v1.4.7
@@ -16,7 +17,7 @@ require (
github.com/matcornic/hermes v1.2.0
github.com/matcornic/hermes/v2 v2.0.2 // indirect
github.com/pkg/errors v0.8.1
github.com/royeo/dingrobot v1.0.0
github.com/royeo/dingrobot v1.0.0 // indirect
github.com/satori/go.uuid v1.2.0
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337
github.com/spf13/viper v1.4.0

2
backend/go.sum
View File

@@ -6,6 +6,8 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0
github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY=
github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd h1:+CYOsXi89xOqBkj7CuEJjA2It+j+R3ngUZEydr6mtkw=
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg=

43
backend/main.go
View File

@@ -154,27 +154,28 @@ func main() {
}
// 爬虫
{
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID）
authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表
authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings) // Scrapy 爬虫设置
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID）
authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表
authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings) // Scrapy 爬虫设置
authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings) // Scrapy 爬虫修改设置
}
// 可配置爬虫
{

32
backend/routes/spider.go
View File

@@ -956,3 +956,35 @@ func GetSpiderScrapySettings(c *gin.Context) {
Data: data,
})
}
func PostSpiderScrapySettings(c *gin.Context) {
id := c.Param("id")
if !bson.IsObjectIdHex(id) {
HandleErrorF(http.StatusBadRequest, c, "spider_id is invalid")
return
}
var reqData []entity.ScrapySettingParam
if err := c.ShouldBindJSON(&reqData); err != nil {
HandleErrorF(http.StatusBadRequest, c, "invalid request")
return
}
spider, err := model.GetSpider(bson.ObjectIdHex(id))
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
if err := services.SaveScrapySettings(spider, reqData); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}

71
backend/services/scrapy.go
View File

@@ -2,10 +2,17 @@ package services
import (
"bytes"
"crawlab/constants"
"crawlab/entity"
"crawlab/model"
"encoding/json"
"fmt"
"github.com/Unknwon/goconfig"
"github.com/apex/log"
"io/ioutil"
"os"
"os/exec"
"path"
"runtime/debug"
"strings"
)
@@ -60,3 +67,67 @@ func GetScrapySettings(s model.Spider) (res []map[string]interface{}, err error)
return res, nil
}
func SaveScrapySettings(s model.Spider, settingsData []entity.ScrapySettingParam) (err error) {
// 读取 scrapy.cfg
cfg, err := goconfig.LoadConfigFile(path.Join(s.Src, "scrapy.cfg"))
if err != nil {
return
}
modName, err := cfg.GetValue("settings", "default")
if err != nil {
return
}
// 定位到 settings.py 文件
arr := strings.Split(modName, ".")
dirName := arr[0]
fileName := arr[1]
filePath := fmt.Sprintf("%s/%s/%s.py", s.Src, dirName, fileName)
// 生成文件内容
content := ""
for _, param := range settingsData {
var line string
switch param.Type {
case constants.String:
line = fmt.Sprintf("%s = '%s'", param.Key, param.Value)
case constants.Number:
line = fmt.Sprintf("%s = %s", param.Key, param.Value)
case constants.Boolean:
if param.Value.(bool) {
line = fmt.Sprintf("%s = %s", param.Key, "True")
} else {
line = fmt.Sprintf("%s = %s", param.Key, "False")
}
case constants.Array:
arr := param.Value.([]interface{})
var arrStr []string
for _, s := range arr {
arrStr = append(arrStr, s.(string))
}
line = fmt.Sprintf("%s = ['%s']", param.Key, strings.Join(arrStr, "','"))
case constants.Object:
value := param.Value.(map[string]interface{})
var arr []string
for k, v := range value {
str := v.(float64)
arr = append(arr, fmt.Sprintf("'%s': %.0f", k, str))
}
line = fmt.Sprintf("%s = {%s}", param.Key, strings.Join(arr, ","))
}
content += line + "\n"
}
// 写到 settings.py
if err := ioutil.WriteFile(filePath, []byte(content), os.ModePerm); err != nil {
return err
}
// 同步到GridFS
if err := UploadSpiderToGridFsFromMaster(s); err != nil {
return err
}
return
}

21
backend/vendor/github.com/royeo/dingrobot/LICENSE generated vendored
View File

@@ -1,21 +0,0 @@
MIT License
Copyright (c) 2018 Royeo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

110
backend/vendor/github.com/royeo/dingrobot/README.md generated vendored
View File

@@ -1,110 +0,0 @@
# Dingrobot
钉钉自定义机器人 Golang API.
支持的消息类型：
- 文本类型
- link 类型
- markdown 类型
- 整体跳转 ActionCard 类型
## Installation
Install:
```sh
go get -u github.com/royeo/dingrobot
```
Import:
```go
import "github.com/royeo/dingrobot"
```
## Quick start
发送文本类型的消息：
```go
func main() {
// You should replace the webhook here with your own.
webhook := "https://oapi.dingtalk.com/robot/send?access_token=xxx"
robot := dingrobot.NewRobot(webhook)
content := "我就是我, @1825718XXXX 是不一样的烟火"
atMobiles := []string{"1825718XXXX"}
isAtAll := false
err := robot.SendText(content, atMobiles, isAtAll)
if err != nil {
log.Fatal(err)
}
}
```
发送 link 类型的消息：
```go
func main() {
// You should replace the webhook here with your own.
webhook := "https://oapi.dingtalk.com/robot/send?access_token=xxx"
robot := dingrobot.NewRobot(webhook)
title := "自定义机器人协议"
text := "群机器人是钉钉群的高级扩展功能。群机器人可以将第三方服务的信息聚合到群聊中，实现自动化的信息同步。例如：通过聚合GitHub，GitLab等源码管理服务，实现源码更新同步；通过聚合Trello，JIRA等项目协调服务，实现项目信息同步。不仅如此，群机器人支持Webhook协议的自定义接入，支持更多可能性，例如：你可将运维报警提醒通过自定义机器人聚合到钉钉群。"
messageUrl := "https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7629140.0.0.Rqyvqo&treeId=257&articleId=105735&docType=1"
picUrl := ""
err := robot.SendLink(title, text, messageUrl, picUrl)
if err != nil {
log.Fatal(err)
}
}
```
发送 markdown 类型的消息：
```go
func main() {
// You should replace the webhook here with your own.
webhook := "https://oapi.dingtalk.com/robot/send?access_token=xxx"
robot := dingrobot.NewRobot(webhook)
title := "杭州天气"
text := "#### 杭州天气  \n > 9度，@1825718XXXX 西北风1级，空气良89，相对温度73%\n\n > ![screenshot](http://i01.lw.aliimg.com/media/lALPBbCc1ZhJGIvNAkzNBLA_1200_588.png)\n  > ###### 10点20分发布 [天气](http://www.thinkpage.cn/) "
atMobiles := []string{"1825718XXXX"}
isAtAll := false
err := robot.SendMarkdown(title, text, atMobiles, isAtAll)
if err != nil {
log.Fatal(err)
}
}
```
发送整体跳转 ActionCard 类型的消息：
```go
func main() {
// You should replace the webhook here with your own.
webhook := "https://oapi.dingtalk.com/robot/send?access_token=xxx"
robot := dingrobot.NewRobot(webhook)
title := "乔布斯 20 年前想打造一间苹果咖啡厅，而它正是 Apple Store 的前身"
text := "![screenshot](@lADOpwk3K80C0M0FoA) \n #### 乔布斯 20 年前想打造的苹果咖啡厅 \n\n Apple Store 的设计正从原来满满的科技感走向生活化，而其生活化的走向其实可以追溯到 20 年前苹果一个建立咖啡馆的计划"
singleTitle := "阅读全文"
singleURL := "https://www.dingtalk.com/"
btnOrientation := "0"
hideAvatar := "0"
err := robot.SendActionCard(title, text, singleTitle, singleURL, btnOrientation, hideAvatar)
if err != nil {
log.Fatal(err)
}
}
```
## License
MIT Copyright (c) 2018 Royeo

118
backend/vendor/github.com/royeo/dingrobot/dingrobot.go generated vendored
View File

@@ -1,118 +0,0 @@
package dingrobot
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
)
// Roboter is the interface implemented by Robot that can send multiple types of messages.
type Roboter interface {
SendText(content string, atMobiles []string, isAtAll bool) error
SendLink(title, text, messageURL, picURL string) error
SendMarkdown(title, text string, atMobiles []string, isAtAll bool) error
SendActionCard(title, text, singleTitle, singleURL, btnOrientation, hideAvatar string) error
}
// Robot represents a dingtalk custom robot that can send messages to groups.
type Robot struct {
Webhook string
}
// NewRobot returns a roboter that can send messages.
func NewRobot(webhook string) Roboter {
return Robot{Webhook: webhook}
}
// SendText send a text type message.
func (r Robot) SendText(content string, atMobiles []string, isAtAll bool) error {
return r.send(&textMessage{
MsgType: msgTypeText,
Text: textParams{
Content: content,
},
At: atParams{
AtMobiles: atMobiles,
IsAtAll: isAtAll,
},
})
}
// SendLink send a link type message.
func (r Robot) SendLink(title, text, messageURL, picURL string) error {
return r.send(&linkMessage{
MsgType: msgTypeLink,
Link: linkParams{
Title: title,
Text: text,
MessageURL: messageURL,
PicURL: picURL,
},
})
}
// SendMarkdown send a markdown type message.
func (r Robot) SendMarkdown(title, text string, atMobiles []string, isAtAll bool) error {
return r.send(&markdownMessage{
MsgType: msgTypeMarkdown,
Markdown: markdownParams{
Title: title,
Text: text,
},
At: atParams{
AtMobiles: atMobiles,
IsAtAll: isAtAll,
},
})
}
// SendActionCard send a action card type message.
func (r Robot) SendActionCard(title, text, singleTitle, singleURL, btnOrientation, hideAvatar string) error {
return r.send(&actionCardMessage{
MsgType: msgTypeActionCard,
ActionCard: actionCardParams{
Title: title,
Text: text,
SingleTitle: singleTitle,
SingleURL: singleURL,
BtnOrientation: btnOrientation,
HideAvatar: hideAvatar,
},
})
}
type dingResponse struct {
Errcode int
Errmsg string
}
func (r Robot) send(msg interface{}) error {
m, err := json.Marshal(msg)
if err != nil {
return err
}
resp, err := http.Post(r.Webhook, "application/json", bytes.NewReader(m))
if err != nil {
return err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
var dr dingResponse
err = json.Unmarshal(data, &dr)
if err != nil {
return err
}
if dr.Errcode != 0 {
return fmt.Errorf("dingrobot send failed: %v", dr.Errmsg)
}
return nil
}

60
backend/vendor/github.com/royeo/dingrobot/message.go generated vendored
View File

@@ -1,60 +0,0 @@
package dingrobot
const (
msgTypeText = "text"
msgTypeLink = "link"
msgTypeMarkdown = "markdown"
msgTypeActionCard = "actionCard"
)
type textMessage struct {
MsgType string `json:"msgtype"`
Text textParams `json:"text"`
At atParams `json:"at"`
}
type textParams struct {
Content string `json:"content"`
}
type atParams struct {
AtMobiles []string `json:"atMobiles,omitempty"`
IsAtAll bool `json:"isAtAll,omitempty"`
}
type linkMessage struct {
MsgType string `json:"msgtype"`
Link linkParams `json:"link"`
}
type linkParams struct {
Title string `json:"title"`
Text string `json:"text"`
MessageURL string `json:"messageUrl"`
PicURL string `json:"picUrl,omitempty"`
}
type markdownMessage struct {
MsgType string `json:"msgtype"`
Markdown markdownParams `json:"markdown"`
At atParams `json:"at"`
}
type markdownParams struct {
Title string `json:"title"`
Text string `json:"text"`
}
type actionCardMessage struct {
MsgType string `json:"msgtype"`
ActionCard actionCardParams `json:"actionCard"`
}
type actionCardParams struct {
Title string `json:"title"`
Text string `json:"text"`
SingleTitle string `json:"singleTitle"`
SingleURL string `json:"singleURL"`
BtnOrientation string `json:"btnOrientation,omitempty"`
HideAvatar string `json:"hideAvatar,omitempty"`
}

4
backend/vendor/modules.txt vendored
View File

@@ -2,6 +2,8 @@
github.com/Masterminds/semver
# github.com/Masterminds/sprig v2.16.0+incompatible
github.com/Masterminds/sprig
# github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd
github.com/Unknwon/goconfig
# github.com/aokoli/goutils v1.0.1
github.com/aokoli/goutils
# github.com/apex/log v1.1.1
@@ -82,8 +84,6 @@ github.com/olekukonko/tablewriter
github.com/pelletier/go-toml
# github.com/pkg/errors v0.8.1
github.com/pkg/errors
# github.com/royeo/dingrobot v1.0.0
github.com/royeo/dingrobot
# github.com/satori/go.uuid v1.2.0
github.com/satori/go.uuid
# github.com/shurcooL/sanitized_anchor_name v1.0.0

6
frontend/src/components/Scrapy/SpiderScrapy.vue
View File

@@ -244,7 +244,11 @@ export default {
this.activeParamIndex = index
this.onOpenDialog()
},
onSave () {
async onSave () {
const res = await this.$store.dispatch('spider/saveSpiderScrapySettings', this.$route.params.id)
if (!res.data.error) {
this.$message.success(this.$t('Saved successfully'))
}
},
onAdd () {
const data = JSON.parse(JSON.stringify(this.spiderScrapySettings))

3
frontend/src/store/modules/spider.js
View File

@@ -147,6 +147,9 @@ const actions = {
}
}))
},
async saveSpiderScrapySettings ({ state }, id) {
return request.post(`/spiders/${id}/scrapy/settings`, state.spiderScrapySettings)
},
crawlSpider ({ state, dispatch }, payload) {
const { spiderId, runType, nodeIds, param } = payload
return request.put(`/tasks`, {