diff --git a/backend/conf/config.yml b/backend/conf/config.yml index a51a34c9..6e3e0611 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -37,4 +37,13 @@ other: tmppath: "/tmp" version: 0.4.3 setting: - allowRegister: "N" \ No newline at end of file + allowRegister: "N" +notification: + mail: + server: '' + port: '' + senderEmail: '' + senderIdentity: '' + smtp: + user: '' + password: '' diff --git a/backend/constants/notification.go b/backend/constants/notification.go new file mode 100644 index 00000000..cad3b19b --- /dev/null +++ b/backend/constants/notification.go @@ -0,0 +1,7 @@ +package constants + +const ( + NotificationTriggerOnTaskEnd = "notification_trigger_on_task_end" + NotificationTriggerOnTaskError = "notification_trigger_on_task_error" + NotificationTriggerNever = "notification_trigger_never" +) diff --git a/backend/go.mod b/backend/go.mod index 497c5b4d..791c6771 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -19,7 +19,9 @@ require ( github.com/satori/go.uuid v1.2.0 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 github.com/spf13/viper v1.4.0 + gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/go-playground/validator.v9 v9.29.1 + gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737 gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.2.2 ) diff --git a/backend/go.sum b/backend/go.sum index 5a186edd..37fe6119 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -235,6 +235,8 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -245,6 +247,8 @@ gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2G gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvRQyEAKbw1xc= gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737 h1:NvePS/smRcFQ4bMtTddFtknbGCtoBkJxGmpSpVRafCc= +gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA= gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI= diff --git a/backend/main.go b/backend/main.go index 08cdf70f..2b052171 100644 --- a/backend/main.go +++ b/backend/main.go @@ -204,6 +204,7 @@ func main() { authGroup.POST("/users/:id", routes.PostUser) // 更改用户 authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户 authGroup.GET("/me", routes.GetMe) // 获取自己账户 + authGroup.POST("/me", routes.PostMe) // 修改自己账户 // release版本 authGroup.GET("/version", routes.GetVersion) // 获取发布的版本 // 系统 diff --git a/backend/model/schedule.go b/backend/model/schedule.go index 3b654b74..d98dabf6 100644 --- a/backend/model/schedule.go +++ b/backend/model/schedule.go @@ -23,6 +23,7 @@ type Schedule struct { NodeIds []bson.ObjectId `json:"node_ids" bson:"node_ids"` Status string `json:"status" bson:"status"` Enabled bool `json:"enabled" bson:"enabled"` + UserId bson.ObjectId `json:"user_id" bson:"user_id"` // 前端展示 SpiderName string `json:"spider_name" bson:"spider_name"` @@ -49,27 +50,6 @@ func (sch *Schedule) Delete() error { return c.RemoveId(sch.Id) } -//func (sch *Schedule) SyncNodeIdAndSpiderId(node Node, spider Spider) { -// sch.syncNodeId(node) -// sch.syncSpiderId(spider) -//} - -//func (sch *Schedule) syncNodeId(node Node) { -// if node.Id.Hex() == sch.NodeId.Hex() { -// return -// } -// sch.NodeId = node.Id -// _ = sch.Save() -//} - -//func (sch *Schedule) syncSpiderId(spider Spider) { -// if spider.Id.Hex() == sch.SpiderId.Hex() { -// return -// } -// sch.SpiderId = spider.Id -// _ = sch.Save() -//} - func GetScheduleList(filter interface{}) ([]Schedule, error) { s, c := database.GetCol("schedules") defer s.Close() @@ -125,13 +105,8 @@ func UpdateSchedule(id bson.ObjectId, item Schedule) error { if err := c.FindId(id).One(&result); err != nil { return err } - //node, err := GetNode(item.NodeId) - //if err != nil { - // return err - //} item.UpdateTs = time.Now() - //item.NodeKey = node.Key if err := item.Save(); err != nil { return err } @@ -142,15 +117,9 @@ func AddSchedule(item Schedule) error { s, c := database.GetCol("schedules") defer s.Close() - //node, err := GetNode(item.NodeId) - //if err != nil { - // return err - //} - item.Id = bson.NewObjectId() item.CreateTs = time.Now() item.UpdateTs = time.Now() - //item.NodeKey = node.Key if err := c.Insert(&item); err != nil { debug.PrintStack() diff --git a/backend/model/task.go b/backend/model/task.go index 6762bd54..a2792061 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -25,6 +25,7 @@ type Task struct { RuntimeDuration float64 `json:"runtime_duration" bson:"runtime_duration"` TotalDuration float64 `json:"total_duration" bson:"total_duration"` Pid int `json:"pid" bson:"pid"` + UserId bson.ObjectId `json:"user_id" bson:"user_id"` // 前端数据 SpiderName string `json:"spider_name"` diff --git a/backend/model/user.go b/backend/model/user.go index 19313e97..246fa448 100644 --- a/backend/model/user.go +++ b/backend/model/user.go @@ -16,11 +16,17 @@ type User struct { Username string `json:"username" bson:"username"` Password string `json:"password" bson:"password"` Role string `json:"role" bson:"role"` + Email string `json:"email" bson:"email"` + Setting UserSetting `json:"setting" bson:"setting"` CreateTs time.Time `json:"create_ts" bson:"create_ts"` UpdateTs time.Time `json:"update_ts" bson:"update_ts"` } +type UserSetting struct { + NotificationTrigger string `json:"notification_trigger" bson:"notification_trigger"` +} + func (user *User) Save() error { s, c := database.GetCol("users") defer s.Close() diff --git a/backend/routes/schedule.go b/backend/routes/schedule.go index c7ef474a..3776019c 100644 --- a/backend/routes/schedule.go +++ b/backend/routes/schedule.go @@ -76,6 +76,9 @@ func PutSchedule(c *gin.Context) { return } + // 加入用户ID + item.UserId = services.GetCurrentUser(c).Id + // 更新数据库 if err := model.AddSchedule(item); err != nil { HandleError(http.StatusInternalServerError, c, err) diff --git a/backend/routes/task.go b/backend/routes/task.go index d5e3cacc..07105f2d 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -112,6 +112,7 @@ func PutTask(c *gin.Context) { SpiderId: reqBody.SpiderId, NodeId: node.Id, Param: reqBody.Param, + UserId: services.GetCurrentUser(c).Id, } if err := services.AddTask(t); err != nil { @@ -124,6 +125,7 @@ func PutTask(c *gin.Context) { t := model.Task{ SpiderId: reqBody.SpiderId, Param: reqBody.Param, + UserId: services.GetCurrentUser(c).Id, } if err := services.AddTask(t); err != nil { HandleError(http.StatusInternalServerError, c, err) @@ -136,6 +138,7 @@ func PutTask(c *gin.Context) { SpiderId: reqBody.SpiderId, NodeId: nodeId, Param: reqBody.Param, + UserId: services.GetCurrentUser(c).Id, } if err := services.AddTask(t); err != nil { diff --git a/backend/routes/user.go b/backend/routes/user.go index 33b6a958..3d636f41 100644 --- a/backend/routes/user.go +++ b/backend/routes/user.go @@ -22,6 +22,7 @@ type UserRequestData struct { Username string `json:"username"` Password string `json:"password"` Role string `json:"role"` + Email string `json:"email"` } func GetUser(c *gin.Context) { @@ -99,6 +100,7 @@ func PutUser(c *gin.Context) { Username: strings.ToLower(reqData.Username), Password: utils.EncryptPassword(reqData.Password), Role: reqData.Role, + Email: reqData.Email, } if err := user.Add(); err != nil { HandleError(http.StatusInternalServerError, c, err) @@ -205,3 +207,39 @@ func GetMe(c *gin.Context) { User: user, }, nil) } + +func PostMe(c *gin.Context) { + type ReqBody struct { + Email string `json:"email"` + Password string `json:"password"` + NotificationTrigger string `json:"notification_trigger"` + } + ctx := context.WithGinContext(c) + user := ctx.User() + if user == nil { + ctx.FailedWithError(constants.ErrorUserNotFound, http.StatusUnauthorized) + return + } + var reqBody ReqBody + if err := c.ShouldBindJSON(&reqBody); err != nil { + HandleErrorF(http.StatusBadRequest, c, "invalid request") + return + } + if reqBody.Email != "" { + user.Email = reqBody.Email + } + if reqBody.Password != "" { + user.Password = utils.EncryptPassword(reqBody.Password) + } + if reqBody.NotificationTrigger != "" { + user.Setting.NotificationTrigger = reqBody.NotificationTrigger + } + if err := user.Save(); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} diff --git a/backend/services/notification/mail.go b/backend/services/notification/mail.go index dd1490dc..48c86351 100644 --- a/backend/services/notification/mail.go +++ b/backend/services/notification/mail.go @@ -1,7 +1,131 @@ package notification -import "github.com/matcornic/hermes" +import ( + "errors" + "github.com/apex/log" + "github.com/matcornic/hermes" + "gopkg.in/gomail.v2" + "net/mail" + "os" + "runtime/debug" + "strconv" +) -func SendMail() error { - hermes +func SendMail(toEmail string, subject string, content string) error { + // hermes instance + h := hermes.Hermes{ + Product: hermes.Product{ + Name: "Hermes", + Link: "https://example-hermes.com/", + Logo: "http://www.duchess-france.org/wp-content/uploads/2016/01/gopher.png", + }, + } + + // config + port, _ := strconv.Atoi(os.Getenv("CRAWLAB_NOTIFICATION_MAIL_PORT")) + password := os.Getenv("CRAWLAB_NOTIFICATION_MAIL_SMTP_PASSWORD") + SMTPUser := os.Getenv("CRAWLAB_NOTIFICATION_MAIL_SMTP_USER") + smtpConfig := smtpAuthentication{ + Server: os.Getenv("CRAWLAB_NOTIFICATION_MAIL_SERVER"), + Port: port, + SenderEmail: os.Getenv("CRAWLAB_NOTIFICATION_MAIL_SENDEREMAIL"), + SenderIdentity: os.Getenv("CRAWLAB_NOTIFICATION_MAIL_SENDERIDENTITY"), + SMTPPassword: password, + SMTPUser: SMTPUser, + } + options := sendOptions{ + To: toEmail, + Subject: subject, + } + + // email instance + email := hermes.Email{ + Body: hermes.Body{ + FreeMarkdown: hermes.Markdown(content), + }, + } + + // generate html + html, err := h.GenerateHTML(email) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // generate text + text, err := h.GeneratePlainText(email) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // send the email + if err := send(smtpConfig, options, html, text); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + return nil +} + +type smtpAuthentication struct { + Server string + Port int + SenderEmail string + SenderIdentity string + SMTPUser string + SMTPPassword string +} + +// sendOptions are options for sending an email +type sendOptions struct { + To string + Subject string +} + +// send sends the email +func send(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, txtBody string) error { + + if smtpConfig.Server == "" { + return errors.New("SMTP server config is empty") + } + if smtpConfig.Port == 0 { + return errors.New("SMTP port config is empty") + } + + if smtpConfig.SMTPUser == "" { + return errors.New("SMTP user is empty") + } + + if smtpConfig.SenderIdentity == "" { + return errors.New("SMTP sender identity is empty") + } + + if smtpConfig.SenderEmail == "" { + return errors.New("SMTP sender email is empty") + } + + if options.To == "" { + return errors.New("no receiver emails configured") + } + + from := mail.Address{ + Name: smtpConfig.SenderIdentity, + Address: smtpConfig.SenderEmail, + } + + m := gomail.NewMessage() + m.SetHeader("From", from.String()) + m.SetHeader("To", options.To) + m.SetHeader("Subject", options.Subject) + + m.SetBody("text/plain", txtBody) + m.AddAlternative("text/html", htmlBody) + + d := gomail.NewPlainDialer(smtpConfig.Server, smtpConfig.Port, smtpConfig.SMTPUser, smtpConfig.SMTPPassword) + + return d.DialAndSend(m) } diff --git a/backend/services/schedule.go b/backend/services/schedule.go index c321c393..b5268a88 100644 --- a/backend/services/schedule.go +++ b/backend/services/schedule.go @@ -34,6 +34,7 @@ func AddScheduleTask(s model.Schedule) func() { SpiderId: s.SpiderId, NodeId: node.Id, Param: s.Param, + UserId: s.UserId, } if err := AddTask(t); err != nil { @@ -46,6 +47,7 @@ func AddScheduleTask(s model.Schedule) func() { Id: id.String(), SpiderId: s.SpiderId, Param: s.Param, + UserId: s.UserId, } if err := AddTask(t); err != nil { log.Errorf(err.Error()) @@ -60,6 +62,7 @@ func AddScheduleTask(s model.Schedule) func() { SpiderId: s.SpiderId, NodeId: nodeId, Param: s.Param, + UserId: s.UserId, } if err := AddTask(t); err != nil { diff --git a/backend/services/task.go b/backend/services/task.go index 7a2eed2a..ea7c47c3 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -6,9 +6,11 @@ import ( "crawlab/entity" "crawlab/lib/cron" "crawlab/model" + "crawlab/services/notification" "crawlab/utils" "encoding/json" "errors" + "fmt" "github.com/apex/log" "github.com/globalsign/mgo/bson" uuid "github.com/satori/go.uuid" @@ -499,6 +501,31 @@ func ExecuteTask(id int) { t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长 t.TotalDuration = t.FinishTs.Sub(t.CreateTs).Seconds() // 总时长 + // 获得触发任务用户 + user, err := model.GetUser(t.UserId) + if err != nil { + log.Errorf(GetWorkerPrefix(id) + err.Error()) + return + } + + // 如果是任务结束时发送通知，则发送通知 + if user.Email != "" && + user.Setting.NotificationTrigger == constants.NotificationTriggerOnTaskEnd { + emailContent := fmt.Sprintf(` +# Your task has finished + +The stats of the task are as below. + +Metric | Value +--- | --- +Create Time | %s +Start Time | %s +Finish Time | %s +Total Duration | %.1f +`, t.CreateTs, t.StartTs, t.FinishTs, t.TotalDuration) + _ = notification.SendMail(user.Email, fmt.Sprintf("[%s] Crawlab Task Finished", spider.Name), emailContent) + } + // 保存任务 if err := t.Save(); err != nil { log.Errorf(GetWorkerPrefix(id) + err.Error()) diff --git a/backend/services/user.go b/backend/services/user.go index 61fd952e..f20b9e7a 100644 --- a/backend/services/user.go +++ b/backend/services/user.go @@ -6,6 +6,7 @@ import ( "crawlab/utils" "errors" "github.com/dgrijalva/jwt-go" + "github.com/gin-gonic/gin" "github.com/globalsign/mgo/bson" "github.com/spf13/viper" "time" @@ -91,3 +92,8 @@ func CheckToken(tokenStr string) (user model.User, err error) { return } + +func GetCurrentUser(c *gin.Context) *model.User { + data, _ := c.Get("currentUser") + return data.(*model.User) +} diff --git a/backend/vendor/modules.txt b/backend/vendor/modules.txt index b8784ecb..a73e7de6 100644 --- a/backend/vendor/modules.txt +++ b/backend/vendor/modules.txt @@ -121,10 +121,14 @@ golang.org/x/sys/unix # golang.org/x/text v0.3.0 golang.org/x/text/transform golang.org/x/text/unicode/norm +# gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc +gopkg.in/alexcesaro/quotedprintable.v3 # gopkg.in/go-playground/validator.v8 v8.18.2 gopkg.in/go-playground/validator.v8 # gopkg.in/go-playground/validator.v9 v9.29.1 gopkg.in/go-playground/validator.v9 +# gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737 +gopkg.in/gomail.v2 # gopkg.in/russross/blackfriday.v2 v2.0.0 gopkg.in/russross/blackfriday.v2 # gopkg.in/yaml.v2 v2.2.2 diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index e739b979..841316fb 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -11,6 +11,7 @@ export default { 'Schedules': '定时任务', 'Deploys': '部署', 'Sites': '网站', + 'Setting': '设置', // 标签 'Overview': '概览', @@ -323,6 +324,7 @@ export default { 'The schedule has been removed': '已删除定时任务', 'The schedule has been added': '已添加定时任务', 'The schedule has been saved': '已保存定时任务', + 'Email format invalid': '邮箱地址格式不正确', // 登录 'Sign in': '登录', @@ -343,6 +345,16 @@ export default { 'Role': '角色', 'Edit User': '更改用户', 'Users': '用户', + 'Email': '邮箱', + 'Optional': '可选', + + // 设置 + 'Notification Trigger': '通知触发', + 'On Task End': '当任务结束', + 'On Task Error': '当任务发生错误', + 'Never': '从不', + + // 其他 tagsView: { closeOthers: '关闭其他', close: '关闭', diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js index fdb7c39f..c019836a 100644 --- a/frontend/src/router/index.js +++ b/frontend/src/router/index.js @@ -210,6 +210,24 @@ export const constantRouterMap = [ } ] }, + { + path: '/setting', + component: Layout, + meta: { + title: 'Setting', + icon: 'fa fa-gear' + }, + children: [ + { + path: '', + component: () => import('../views/setting/Setting'), + meta: { + title: 'Setting', + icon: 'fa fa-gear' + } + } + ] + }, { path: '*', redirect: '/404', hidden: true } ] diff --git a/frontend/src/store/modules/user.js b/frontend/src/store/modules/user.js index 2b3a79d7..af778816 100644 --- a/frontend/src/store/modules/user.js +++ b/frontend/src/store/modules/user.js @@ -91,6 +91,11 @@ const user = { }) }, + // 修改用户信息 + postInfo ({ commit }, form) { + return request.post('/me', form) + }, + // 注册 register ({ dispatch, commit, state }, userInfo) { return new Promise((resolve, reject) => { diff --git a/frontend/src/views/login/index.vue b/frontend/src/views/login/index.vue index 664d05e5..5bad9935 100644 --- a/frontend/src/views/login/index.vue +++ b/frontend/src/views/login/index.vue @@ -34,6 +34,14 @@ @keyup.enter.native="onKeyEnter" /> + + + @@ -104,7 +112,8 @@ export default { loginForm: { username: '', password: '', - confirmPassword: '' + confirmPassword: '', + email: '' }, loginRules: { username: [{ required: true, trigger: 'blur', validator: validateUsername }], diff --git a/frontend/src/views/setting/Setting.vue b/frontend/src/views/setting/Setting.vue new file mode 100644 index 00000000..83695f59 --- /dev/null +++ b/frontend/src/views/setting/Setting.vue @@ -0,0 +1,102 @@ + + + + + diff --git a/frontend/src/views/user/UserList.vue b/frontend/src/views/user/UserList.vue index 26cbedea..9389f08c 100644 --- a/frontend/src/views/user/UserList.vue +++ b/frontend/src/views/user/UserList.vue @@ -15,6 +15,9 @@ + + +