git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

加入Web Hook

Browse Source
This commit is contained in:
marvzhang
2020-04-20 13:44:13 +08:00
parent 28683975e1
commit c662577a85
5 changed files with 80 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/model/spider.go
View File

@@ -64,6 +64,10 @@ type Spider struct {
DedupField string `json:"dedup_field" bson:"dedup_field"` // 去重字段
DedupMethod string `json:"dedup_method" bson:"dedup_method"` // 去重方式
// Web Hook
IsWebHook bool `json:"is_web_hook" bson:"is_web_hook"` // 是否开启 Web Hook
WebHookUrl string `json:"web_hook_url" bson:"web_hook_url"` // Web Hook URL
// 前端展示
LastRunTs time.Time `json:"last_run_ts"` // 最后一次执行时间
LastStatus string `json:"last_status"` // 最后执行状态

68
backend/services/task.go
View File

@@ -15,8 +15,10 @@ import (
"fmt"
"github.com/apex/log"
"github.com/globalsign/mgo/bson"
"github.com/imroc/req"
uuid "github.com/satori/go.uuid"
"github.com/spf13/viper"
"net/http"
"os"
"os/exec"
"path"
@@ -555,12 +557,22 @@ func ExecuteTask(id int) {
cmd += " " + t.Param
}
// 获得触发任务用户
user, err := model.GetUser(t.UserId)
if err != nil {
log.Errorf(GetWorkerPrefix(id) + err.Error())
return
}
// 任务赋值
t.NodeId = node.Id // 任务节点信息
t.StartTs = time.Now() // 任务开始时间
t.Status = constants.StatusRunning // 任务状态
t.WaitDuration = t.StartTs.Sub(t.CreateTs).Seconds() // 等待时长
// 发送 Web Hook 请求 (任务开始)
go SendWebHookRequest(user, t, spider)
// 文件检查
if err := SpiderFileCheck(t, spider); err != nil {
log.Errorf("spider file check error: %s", err.Error())
@@ -595,13 +607,6 @@ func ExecuteTask(id int) {
cronExecErrLog.Start()
defer cronExecErrLog.Stop()
// 获得触发任务用户
user, err := model.GetUser(t.UserId)
if err != nil {
log.Errorf(GetWorkerPrefix(id) + err.Error())
return
}
// 执行Shell命令
if err := ExecuteShellCmd(cmd, cwd, t, spider, user); err != nil {
log.Errorf(GetWorkerPrefix(id) + err.Error())
@@ -611,6 +616,10 @@ func ExecuteTask(id int) {
if user.Setting.NotificationTrigger == constants.NotificationTriggerOnTaskEnd || user.Setting.NotificationTrigger == constants.NotificationTriggerOnTaskError {
SendNotifications(user, t, spider)
}
// 发送 Web Hook 请求 (任务开始)
go SendWebHookRequest(user, t, spider)
return
}
@@ -630,6 +639,9 @@ func ExecuteTask(id int) {
t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长
t.TotalDuration = t.FinishTs.Sub(t.CreateTs).Seconds() // 总时长
// 发送 Web Hook 请求 (任务结束)
go SendWebHookRequest(user, t, spider)
// 如果是任务结束时发送通知，则发送通知
if user.Setting.NotificationTrigger == constants.NotificationTriggerOnTaskEnd {
SendNotifications(user, t, spider)
@@ -664,10 +676,6 @@ func FinishUpTask(s model.Spider, t model.Task) {
}()
}
func MonitorTask(s model.Spider, t model.Task) {
}
func SpiderFileCheck(t model.Task, spider model.Spider) error {
// 判断爬虫文件是否存在
gfFile := model.GetGridFs(spider.FileId)
@@ -997,6 +1005,44 @@ func SendNotifications(u model.User, t model.Task, s model.Spider) {
}
}
func SendWebHookRequest(u model.User, t model.Task, s model.Spider) {
type RequestBody struct {
Status string `json:"status"`
Task model.Task `json:"task"`
Spider model.Spider `json:"spider"`
UserName string `json:"user_name"`
}
if s.IsWebHook && s.WebHookUrl != "" {
// request header
header := req.Header{
"Content-Type": "application/json; charset=utf-8",
}
// request body
reqBody := RequestBody{
Status: t.Status,
UserName: u.Username,
Task: t,
Spider: s,
}
// make POST http request
res, err := req.Post(s.WebHookUrl, header, req.BodyJSON(reqBody))
if err != nil {
log.Errorf("sent web hook request with error: " + err.Error())
debug.PrintStack()
return
}
if res.Response().StatusCode != http.StatusOK {
log.Errorf(fmt.Sprintf("sent web hook request with error http code: %d, task_id: %s, status: %s", res.Response().StatusCode, t.Id, t.Status))
debug.PrintStack()
return
}
log.Infof(fmt.Sprintf("sent web hook request, task_id: %s, status: %s)", t.Id, t.Status))
}
}
func InitTaskExecutor() error {
// 构造任务执行器
c := cron.New(cron.WithSeconds())

17
frontend/src/components/InfoView/SpiderInfoView.vue
View File

@@ -96,7 +96,7 @@
</el-form-item>
</el-col>
</el-row>
<el-form-item v-if="!isView && !isConfigurable" :label="$t('Is De-Duplicated')" prop="dedup_field"
<el-form-item v-if="!isView" :label="$t('De-Duplication')" prop="dedup_field"
:rules="dedupRules">
<div style="display: flex; align-items: center; height: 40px">
<el-switch
@@ -123,6 +123,21 @@
/>
</div>
</el-form-item>
<el-form-item v-if="!isView" label="Web Hook">
<div style="display: flex; align-items: center; height: 40px">
<el-switch
v-model="spiderForm.is_web_hook"
active-color="#13ce66"
:disabled="isView || isPublic"
/>
<el-input
v-if="spiderForm.is_web_hook"
v-model="spiderForm.web_hook_url"
:placeholder="$t('Please enter Web Hook URL')"
style="margin-left: 20px"
/>
</div>
</el-form-item>
<el-row>
<el-col :span="6">
<el-form-item v-if="!isView" :label="$t('Is Public')" prop="is_public">

2
frontend/src/i18n/zh.js
View File

@@ -232,6 +232,7 @@ export default {
'Please enter de-duplicated field': '请输入去重字段',
'Overwrite': '覆盖',
'Ignore': '忽略',
'De-Duplication': '去重',
// 爬虫列表
'Name': '名称',
@@ -654,6 +655,7 @@ docker run -d --restart always --name crawlab_worker \\
'Deleted API token successfully': '成功删除 API Token',
'Are you sure to add an API token?': '确认创建 API Token?',
'Are you sure to delete this API token?': '确认删除该 API Token?',
'Please enter Web Hook URL': '请输入 Web Hook URL',
// 其他
'Star crawlab-team/crawlab on GitHub': '在 GitHub 上为 Crawlab 加星吧'

2
frontend/src/views/layout/components/Sidebar/index.vue
View File

@@ -93,6 +93,6 @@ export default {
.sidebar-container .sidebar-logo .version {
margin-left: 5px;
font-weight: normal;
font-size: 12px;
font-size: 11px;
}
</style>