git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-29 18:00:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

加入用户设置与邮件通知

Browse Source
This commit is contained in:
marvzhang
2020-01-13 19:56:13 +08:00
parent 117ae581de
commit c67c1be52a
22 changed files with 405 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
backend/conf/config.yml
View File

@@ -37,4 +37,13 @@ other:
tmppath: "/tmp"
version: 0.4.3
setting:
allowRegister: "N"
allowRegister: "N"
notification:
mail:
server: ''
port: ''
senderEmail: ''
senderIdentity: ''
smtp:
user: ''
password: ''

7
backend/constants/notification.go Normal file
View File

@@ -0,0 +1,7 @@
package constants
const (
NotificationTriggerOnTaskEnd = "notification_trigger_on_task_end"
NotificationTriggerOnTaskError = "notification_trigger_on_task_error"
NotificationTriggerNever = "notification_trigger_never"
)

2
backend/go.mod
View File

@@ -19,7 +19,9 @@ require (
github.com/satori/go.uuid v1.2.0
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337
github.com/spf13/viper v1.4.0
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/go-playground/validator.v9 v9.29.1
gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737
gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect
gopkg.in/yaml.v2 v2.2.2
)

4
backend/go.sum
View File

@@ -235,6 +235,8 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -245,6 +247,8 @@ gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2G
gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvRQyEAKbw1xc=
gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737 h1:NvePS/smRcFQ4bMtTddFtknbGCtoBkJxGmpSpVRafCc=
gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA=
gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=

1
backend/main.go
View File

@@ -204,6 +204,7 @@ func main() {
authGroup.POST("/users/:id", routes.PostUser) // 更改用户
authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户
authGroup.GET("/me", routes.GetMe) // 获取自己账户
authGroup.POST("/me", routes.PostMe) // 修改自己账户
// release版本
authGroup.GET("/version", routes.GetVersion) // 获取发布的版本
// 系统

33
backend/model/schedule.go
View File

@@ -23,6 +23,7 @@ type Schedule struct {
NodeIds []bson.ObjectId `json:"node_ids" bson:"node_ids"`
Status string `json:"status" bson:"status"`
Enabled bool `json:"enabled" bson:"enabled"`
UserId bson.ObjectId `json:"user_id" bson:"user_id"`
// 前端展示
SpiderName string `json:"spider_name" bson:"spider_name"`
@@ -49,27 +50,6 @@ func (sch *Schedule) Delete() error {
return c.RemoveId(sch.Id)
}
//func (sch *Schedule) SyncNodeIdAndSpiderId(node Node, spider Spider) {
// sch.syncNodeId(node)
// sch.syncSpiderId(spider)
//}
//func (sch *Schedule) syncNodeId(node Node) {
// if node.Id.Hex() == sch.NodeId.Hex() {
// return
// }
// sch.NodeId = node.Id
// _ = sch.Save()
//}
//func (sch *Schedule) syncSpiderId(spider Spider) {
// if spider.Id.Hex() == sch.SpiderId.Hex() {
// return
// }
// sch.SpiderId = spider.Id
// _ = sch.Save()
//}
func GetScheduleList(filter interface{}) ([]Schedule, error) {
s, c := database.GetCol("schedules")
defer s.Close()
@@ -125,13 +105,8 @@ func UpdateSchedule(id bson.ObjectId, item Schedule) error {
if err := c.FindId(id).One(&result); err != nil {
return err
}
//node, err := GetNode(item.NodeId)
//if err != nil {
// return err
//}
item.UpdateTs = time.Now()
//item.NodeKey = node.Key
if err := item.Save(); err != nil {
return err
}
@@ -142,15 +117,9 @@ func AddSchedule(item Schedule) error {
s, c := database.GetCol("schedules")
defer s.Close()
//node, err := GetNode(item.NodeId)
//if err != nil {
// return err
//}
item.Id = bson.NewObjectId()
item.CreateTs = time.Now()
item.UpdateTs = time.Now()
//item.NodeKey = node.Key
if err := c.Insert(&item); err != nil {
debug.PrintStack()

1
backend/model/task.go
View File

@@ -25,6 +25,7 @@ type Task struct {
RuntimeDuration float64 `json:"runtime_duration" bson:"runtime_duration"`
TotalDuration float64 `json:"total_duration" bson:"total_duration"`
Pid int `json:"pid" bson:"pid"`
UserId bson.ObjectId `json:"user_id" bson:"user_id"`
// 前端数据
SpiderName string `json:"spider_name"`

6
backend/model/user.go
View File

@@ -16,11 +16,17 @@ type User struct {
Username string `json:"username" bson:"username"`
Password string `json:"password" bson:"password"`
Role string `json:"role" bson:"role"`
Email string `json:"email" bson:"email"`
Setting UserSetting `json:"setting" bson:"setting"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
}
type UserSetting struct {
NotificationTrigger string `json:"notification_trigger" bson:"notification_trigger"`
}
func (user *User) Save() error {
s, c := database.GetCol("users")
defer s.Close()

3
backend/routes/schedule.go
View File

@@ -76,6 +76,9 @@ func PutSchedule(c *gin.Context) {
return
}
// 加入用户ID
item.UserId = services.GetCurrentUser(c).Id
// 更新数据库
if err := model.AddSchedule(item); err != nil {
HandleError(http.StatusInternalServerError, c, err)

3
backend/routes/task.go
View File

@@ -112,6 +112,7 @@ func PutTask(c *gin.Context) {
SpiderId: reqBody.SpiderId,
NodeId: node.Id,
Param: reqBody.Param,
UserId: services.GetCurrentUser(c).Id,
}
if err := services.AddTask(t); err != nil {
@@ -124,6 +125,7 @@ func PutTask(c *gin.Context) {
t := model.Task{
SpiderId: reqBody.SpiderId,
Param: reqBody.Param,
UserId: services.GetCurrentUser(c).Id,
}
if err := services.AddTask(t); err != nil {
HandleError(http.StatusInternalServerError, c, err)
@@ -136,6 +138,7 @@ func PutTask(c *gin.Context) {
SpiderId: reqBody.SpiderId,
NodeId: nodeId,
Param: reqBody.Param,
UserId: services.GetCurrentUser(c).Id,
}
if err := services.AddTask(t); err != nil {

38
backend/routes/user.go
View File

@@ -22,6 +22,7 @@ type UserRequestData struct {
Username string `json:"username"`
Password string `json:"password"`
Role string `json:"role"`
Email string `json:"email"`
}
func GetUser(c *gin.Context) {
@@ -99,6 +100,7 @@ func PutUser(c *gin.Context) {
Username: strings.ToLower(reqData.Username),
Password: utils.EncryptPassword(reqData.Password),
Role: reqData.Role,
Email: reqData.Email,
}
if err := user.Add(); err != nil {
HandleError(http.StatusInternalServerError, c, err)
@@ -205,3 +207,39 @@ func GetMe(c *gin.Context) {
User: user,
}, nil)
}
func PostMe(c *gin.Context) {
type ReqBody struct {
Email string `json:"email"`
Password string `json:"password"`
NotificationTrigger string `json:"notification_trigger"`
}
ctx := context.WithGinContext(c)
user := ctx.User()
if user == nil {
ctx.FailedWithError(constants.ErrorUserNotFound, http.StatusUnauthorized)
return
}
var reqBody ReqBody
if err := c.ShouldBindJSON(&reqBody); err != nil {
HandleErrorF(http.StatusBadRequest, c, "invalid request")
return
}
if reqBody.Email != "" {
user.Email = reqBody.Email
}
if reqBody.Password != "" {
user.Password = utils.EncryptPassword(reqBody.Password)
}
if reqBody.NotificationTrigger != "" {
user.Setting.NotificationTrigger = reqBody.NotificationTrigger
}
if err := user.Save(); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}

130
backend/services/notification/mail.go
View File

@@ -1,7 +1,131 @@
package notification
import "github.com/matcornic/hermes"
import (
"errors"
"github.com/apex/log"
"github.com/matcornic/hermes"
"gopkg.in/gomail.v2"
"net/mail"
"os"
"runtime/debug"
"strconv"
)
func SendMail() error {
hermes
func SendMail(toEmail string, subject string, content string) error {
// hermes instance
h := hermes.Hermes{
Product: hermes.Product{
Name: "Hermes",
Link: "https://example-hermes.com/",
Logo: "http://www.duchess-france.org/wp-content/uploads/2016/01/gopher.png",
},
}
// config
port, _ := strconv.Atoi(os.Getenv("CRAWLAB_NOTIFICATION_MAIL_PORT"))
password := os.Getenv("CRAWLAB_NOTIFICATION_MAIL_SMTP_PASSWORD")
SMTPUser := os.Getenv("CRAWLAB_NOTIFICATION_MAIL_SMTP_USER")
smtpConfig := smtpAuthentication{
Server: os.Getenv("CRAWLAB_NOTIFICATION_MAIL_SERVER"),
Port: port,
SenderEmail: os.Getenv("CRAWLAB_NOTIFICATION_MAIL_SENDEREMAIL"),
SenderIdentity: os.Getenv("CRAWLAB_NOTIFICATION_MAIL_SENDERIDENTITY"),
SMTPPassword: password,
SMTPUser: SMTPUser,
}
options := sendOptions{
To: toEmail,
Subject: subject,
}
// email instance
email := hermes.Email{
Body: hermes.Body{
FreeMarkdown: hermes.Markdown(content),
},
}
// generate html
html, err := h.GenerateHTML(email)
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return err
}
// generate text
text, err := h.GeneratePlainText(email)
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return err
}
// send the email
if err := send(smtpConfig, options, html, text); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return err
}
return nil
}
type smtpAuthentication struct {
Server string
Port int
SenderEmail string
SenderIdentity string
SMTPUser string
SMTPPassword string
}
// sendOptions are options for sending an email
type sendOptions struct {
To string
Subject string
}
// send sends the email
func send(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, txtBody string) error {
if smtpConfig.Server == "" {
return errors.New("SMTP server config is empty")
}
if smtpConfig.Port == 0 {
return errors.New("SMTP port config is empty")
}
if smtpConfig.SMTPUser == "" {
return errors.New("SMTP user is empty")
}
if smtpConfig.SenderIdentity == "" {
return errors.New("SMTP sender identity is empty")
}
if smtpConfig.SenderEmail == "" {
return errors.New("SMTP sender email is empty")
}
if options.To == "" {
return errors.New("no receiver emails configured")
}
from := mail.Address{
Name: smtpConfig.SenderIdentity,
Address: smtpConfig.SenderEmail,
}
m := gomail.NewMessage()
m.SetHeader("From", from.String())
m.SetHeader("To", options.To)
m.SetHeader("Subject", options.Subject)
m.SetBody("text/plain", txtBody)
m.AddAlternative("text/html", htmlBody)
d := gomail.NewPlainDialer(smtpConfig.Server, smtpConfig.Port, smtpConfig.SMTPUser, smtpConfig.SMTPPassword)
return d.DialAndSend(m)
}

3
backend/services/schedule.go
View File

@@ -34,6 +34,7 @@ func AddScheduleTask(s model.Schedule) func() {
SpiderId: s.SpiderId,
NodeId: node.Id,
Param: s.Param,
UserId: s.UserId,
}
if err := AddTask(t); err != nil {
@@ -46,6 +47,7 @@ func AddScheduleTask(s model.Schedule) func() {
Id: id.String(),
SpiderId: s.SpiderId,
Param: s.Param,
UserId: s.UserId,
}
if err := AddTask(t); err != nil {
log.Errorf(err.Error())
@@ -60,6 +62,7 @@ func AddScheduleTask(s model.Schedule) func() {
SpiderId: s.SpiderId,
NodeId: nodeId,
Param: s.Param,
UserId: s.UserId,
}
if err := AddTask(t); err != nil {

27
backend/services/task.go
View File

@@ -6,9 +6,11 @@ import (
"crawlab/entity"
"crawlab/lib/cron"
"crawlab/model"
"crawlab/services/notification"
"crawlab/utils"
"encoding/json"
"errors"
"fmt"
"github.com/apex/log"
"github.com/globalsign/mgo/bson"
uuid "github.com/satori/go.uuid"
@@ -499,6 +501,31 @@ func ExecuteTask(id int) {
t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长
t.TotalDuration = t.FinishTs.Sub(t.CreateTs).Seconds() // 总时长
// 获得触发任务用户
user, err := model.GetUser(t.UserId)
if err != nil {
log.Errorf(GetWorkerPrefix(id) + err.Error())
return
}
// 如果是任务结束时发送通知，则发送通知
if user.Email != "" &&
user.Setting.NotificationTrigger == constants.NotificationTriggerOnTaskEnd {
emailContent := fmt.Sprintf(`
# Your task has finished
The stats of the task are as below.
Metric | Value
--- | ---
Create Time | %s
Start Time | %s
Finish Time | %s
Total Duration | %.1f
`, t.CreateTs, t.StartTs, t.FinishTs, t.TotalDuration)
_ = notification.SendMail(user.Email, fmt.Sprintf("[%s] Crawlab Task Finished", spider.Name), emailContent)
}
// 保存任务
if err := t.Save(); err != nil {
log.Errorf(GetWorkerPrefix(id) + err.Error())

6
backend/services/user.go
View File

@@ -6,6 +6,7 @@ import (
"crawlab/utils"
"errors"
"github.com/dgrijalva/jwt-go"
"github.com/gin-gonic/gin"
"github.com/globalsign/mgo/bson"
"github.com/spf13/viper"
"time"
@@ -91,3 +92,8 @@ func CheckToken(tokenStr string) (user model.User, err error) {
return
}
func GetCurrentUser(c *gin.Context) *model.User {
data, _ := c.Get("currentUser")
return data.(*model.User)
}

4
backend/vendor/modules.txt vendored
View File

@@ -121,10 +121,14 @@ golang.org/x/sys/unix
# golang.org/x/text v0.3.0
golang.org/x/text/transform
golang.org/x/text/unicode/norm
# gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc
gopkg.in/alexcesaro/quotedprintable.v3
# gopkg.in/go-playground/validator.v8 v8.18.2
gopkg.in/go-playground/validator.v8
# gopkg.in/go-playground/validator.v9 v9.29.1
gopkg.in/go-playground/validator.v9
# gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737
gopkg.in/gomail.v2
# gopkg.in/russross/blackfriday.v2 v2.0.0
gopkg.in/russross/blackfriday.v2
# gopkg.in/yaml.v2 v2.2.2