diff --git a/backend/model/spider.go b/backend/model/spider.go
index 666ed7d1..8b44481e 100644
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -64,6 +64,10 @@ type Spider struct {
DedupField string `json:"dedup_field" bson:"dedup_field"` // 去重字段
DedupMethod string `json:"dedup_method" bson:"dedup_method"` // 去重方式
+ // Web Hook
+ IsWebHook bool `json:"is_web_hook" bson:"is_web_hook"` // 是否开启 Web Hook
+ WebHookUrl string `json:"web_hook_url" bson:"web_hook_url"` // Web Hook URL
+
// 前端展示
LastRunTs time.Time `json:"last_run_ts"` // 最后一次执行时间
LastStatus string `json:"last_status"` // 最后执行状态
diff --git a/backend/services/task.go b/backend/services/task.go
index df8b4071..db81ed4c 100644
--- a/backend/services/task.go
+++ b/backend/services/task.go
@@ -15,8 +15,10 @@ import (
"fmt"
"github.com/apex/log"
"github.com/globalsign/mgo/bson"
+ "github.com/imroc/req"
uuid "github.com/satori/go.uuid"
"github.com/spf13/viper"
+ "net/http"
"os"
"os/exec"
"path"
@@ -555,12 +557,22 @@ func ExecuteTask(id int) {
cmd += " " + t.Param
}
+ // 获得触发任务用户
+ user, err := model.GetUser(t.UserId)
+ if err != nil {
+ log.Errorf(GetWorkerPrefix(id) + err.Error())
+ return
+ }
+
// 任务赋值
t.NodeId = node.Id // 任务节点信息
t.StartTs = time.Now() // 任务开始时间
t.Status = constants.StatusRunning // 任务状态
t.WaitDuration = t.StartTs.Sub(t.CreateTs).Seconds() // 等待时长
+ // 发送 Web Hook 请求 (任务开始)
+ go SendWebHookRequest(user, t, spider)
+
// 文件检查
if err := SpiderFileCheck(t, spider); err != nil {
log.Errorf("spider file check error: %s", err.Error())
@@ -595,13 +607,6 @@ func ExecuteTask(id int) {
cronExecErrLog.Start()
defer cronExecErrLog.Stop()
- // 获得触发任务用户
- user, err := model.GetUser(t.UserId)
- if err != nil {
- log.Errorf(GetWorkerPrefix(id) + err.Error())
- return
- }
-
// 执行Shell命令
if err := ExecuteShellCmd(cmd, cwd, t, spider, user); err != nil {
log.Errorf(GetWorkerPrefix(id) + err.Error())
@@ -611,6 +616,10 @@ func ExecuteTask(id int) {
if user.Setting.NotificationTrigger == constants.NotificationTriggerOnTaskEnd || user.Setting.NotificationTrigger == constants.NotificationTriggerOnTaskError {
SendNotifications(user, t, spider)
}
+
+ // 发送 Web Hook 请求 (任务开始)
+ go SendWebHookRequest(user, t, spider)
+
return
}
@@ -630,6 +639,9 @@ func ExecuteTask(id int) {
t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长
t.TotalDuration = t.FinishTs.Sub(t.CreateTs).Seconds() // 总时长
+ // 发送 Web Hook 请求 (任务结束)
+ go SendWebHookRequest(user, t, spider)
+
// 如果是任务结束时发送通知，则发送通知
if user.Setting.NotificationTrigger == constants.NotificationTriggerOnTaskEnd {
SendNotifications(user, t, spider)
@@ -664,10 +676,6 @@ func FinishUpTask(s model.Spider, t model.Task) {
}()
}
-func MonitorTask(s model.Spider, t model.Task) {
-
-}
-
func SpiderFileCheck(t model.Task, spider model.Spider) error {
// 判断爬虫文件是否存在
gfFile := model.GetGridFs(spider.FileId)
@@ -997,6 +1005,44 @@ func SendNotifications(u model.User, t model.Task, s model.Spider) {
}
}
+func SendWebHookRequest(u model.User, t model.Task, s model.Spider) {
+ type RequestBody struct {
+ Status string `json:"status"`
+ Task model.Task `json:"task"`
+ Spider model.Spider `json:"spider"`
+ UserName string `json:"user_name"`
+ }
+
+ if s.IsWebHook && s.WebHookUrl != "" {
+ // request header
+ header := req.Header{
+ "Content-Type": "application/json; charset=utf-8",
+ }
+
+ // request body
+ reqBody := RequestBody{
+ Status: t.Status,
+ UserName: u.Username,
+ Task: t,
+ Spider: s,
+ }
+
+ // make POST http request
+ res, err := req.Post(s.WebHookUrl, header, req.BodyJSON(reqBody))
+ if err != nil {
+ log.Errorf("sent web hook request with error: " + err.Error())
+ debug.PrintStack()
+ return
+ }
+ if res.Response().StatusCode != http.StatusOK {
+ log.Errorf(fmt.Sprintf("sent web hook request with error http code: %d, task_id: %s, status: %s", res.Response().StatusCode, t.Id, t.Status))
+ debug.PrintStack()
+ return
+ }
+ log.Infof(fmt.Sprintf("sent web hook request, task_id: %s, status: %s)", t.Id, t.Status))
+ }
+}
+
func InitTaskExecutor() error {
// 构造任务执行器
c := cron.New(cron.WithSeconds())
diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue
index e9139adc..d66d7632 100644
--- a/frontend/src/components/InfoView/SpiderInfoView.vue
+++ b/frontend/src/components/InfoView/SpiderInfoView.vue
@@ -96,7 +96,7 @@
-
+
+
+
+
+
+
diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index 06f07239..81f12171 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -232,6 +232,7 @@ export default {
'Please enter de-duplicated field': '请输入去重字段',
'Overwrite': '覆盖',
'Ignore': '忽略',
+ 'De-Duplication': '去重',
// 爬虫列表
'Name': '名称',
@@ -654,6 +655,7 @@ docker run -d --restart always --name crawlab_worker \\
'Deleted API token successfully': '成功删除 API Token',
'Are you sure to add an API token?': '确认创建 API Token?',
'Are you sure to delete this API token?': '确认删除该 API Token?',
+ 'Please enter Web Hook URL': '请输入 Web Hook URL',
// 其他
'Star crawlab-team/crawlab on GitHub': '在 GitHub 上为 Crawlab 加星吧'
diff --git a/frontend/src/views/layout/components/Sidebar/index.vue b/frontend/src/views/layout/components/Sidebar/index.vue
index 1b16c86f..a7461196 100644
--- a/frontend/src/views/layout/components/Sidebar/index.vue
+++ b/frontend/src/views/layout/components/Sidebar/index.vue
@@ -93,6 +93,6 @@ export default {
.sidebar-container .sidebar-logo .version {
margin-left: 5px;
font-weight: normal;
- font-size: 12px;
+ font-size: 11px;
}