mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-22 17:31:03 +01:00
Merge remote-tracking branch 'origin/develop' into develop
This commit is contained in:
@@ -46,4 +46,4 @@ notification:
|
||||
senderIdentity: ''
|
||||
smtp:
|
||||
user: ''
|
||||
password: ''
|
||||
password: ''
|
||||
152
backend/main.go
152
backend/main.go
@@ -138,79 +138,99 @@ func main() {
|
||||
}
|
||||
authGroup := app.Group("/", middlewares.AuthorizationMiddleware())
|
||||
{
|
||||
// 路由
|
||||
// 节点
|
||||
authGroup.GET("/nodes", routes.GetNodeList) // 节点列表
|
||||
authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情
|
||||
authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点
|
||||
authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表
|
||||
authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表
|
||||
authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点
|
||||
authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表
|
||||
authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表
|
||||
authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表
|
||||
authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖
|
||||
authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖
|
||||
authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言
|
||||
{
|
||||
authGroup.GET("/nodes", routes.GetNodeList) // 节点列表
|
||||
authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情
|
||||
authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点
|
||||
authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表
|
||||
authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表
|
||||
authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点
|
||||
authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表
|
||||
authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表
|
||||
authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表
|
||||
authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖
|
||||
authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖
|
||||
authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言
|
||||
}
|
||||
// 爬虫
|
||||
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
|
||||
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
|
||||
authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
|
||||
authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
|
||||
authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
|
||||
authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
|
||||
authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫(ID)
|
||||
authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
|
||||
authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
|
||||
authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
|
||||
authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
|
||||
authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
|
||||
authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
|
||||
authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
|
||||
authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
|
||||
authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
|
||||
authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
|
||||
authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
|
||||
authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
|
||||
{
|
||||
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
|
||||
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
|
||||
authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
|
||||
authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
|
||||
authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
|
||||
authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
|
||||
authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫(ID)
|
||||
authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
|
||||
authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
|
||||
authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
|
||||
authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
|
||||
authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
|
||||
authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
|
||||
authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
|
||||
authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
|
||||
authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
|
||||
authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
|
||||
authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
|
||||
authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
|
||||
}
|
||||
// 可配置爬虫
|
||||
authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置
|
||||
authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置
|
||||
authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫
|
||||
authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫
|
||||
authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫
|
||||
authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫
|
||||
authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表
|
||||
{
|
||||
authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置
|
||||
authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置
|
||||
authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫
|
||||
authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫
|
||||
authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫
|
||||
authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫
|
||||
authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表
|
||||
}
|
||||
// 任务
|
||||
authGroup.GET("/tasks", routes.GetTaskList) // 任务列表
|
||||
authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情
|
||||
authGroup.PUT("/tasks", routes.PutTask) // 派发任务
|
||||
authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务
|
||||
authGroup.DELETE("/tasks_multiple", routes.DeleteMultipleTask) // 删除多个任务
|
||||
authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) //删除指定状态的任务
|
||||
authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务
|
||||
authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志
|
||||
authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果
|
||||
authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果
|
||||
{
|
||||
authGroup.GET("/tasks", routes.GetTaskList) // 任务列表
|
||||
authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情
|
||||
authGroup.PUT("/tasks", routes.PutTask) // 派发任务
|
||||
authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务
|
||||
authGroup.DELETE("/tasks_multiple", routes.DeleteMultipleTask) // 删除多个任务
|
||||
authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) //删除指定状态的任务
|
||||
authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务
|
||||
authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志
|
||||
authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果
|
||||
authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果
|
||||
}
|
||||
// 定时任务
|
||||
authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表
|
||||
authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情
|
||||
authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务
|
||||
authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
|
||||
authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
|
||||
authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务
|
||||
authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务
|
||||
{
|
||||
authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表
|
||||
authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情
|
||||
authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务
|
||||
authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
|
||||
authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
|
||||
authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务
|
||||
authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务
|
||||
}
|
||||
// 用户
|
||||
{
|
||||
authGroup.GET("/users", routes.GetUserList) // 用户列表
|
||||
authGroup.GET("/users/:id", routes.GetUser) // 用户详情
|
||||
authGroup.POST("/users/:id", routes.PostUser) // 更改用户
|
||||
authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户
|
||||
authGroup.GET("/me", routes.GetMe) // 获取自己账户
|
||||
authGroup.POST("/me", routes.PostMe) // 修改自己账户
|
||||
}
|
||||
// 系统
|
||||
{
|
||||
authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表
|
||||
authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
|
||||
}
|
||||
// 全局变量
|
||||
{
|
||||
authGroup.POST("/variable", routes.PostVariable) // 新增
|
||||
authGroup.PUT("/variable/:id", routes.PutVariable) //修改
|
||||
authGroup.DELETE("/variable/:id", routes.DeleteVariable) //删除
|
||||
authGroup.GET("/variables", routes.GetVariableList) // 列表
|
||||
}
|
||||
// 统计数据
|
||||
authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据
|
||||
// 用户
|
||||
authGroup.GET("/users", routes.GetUserList) // 用户列表
|
||||
authGroup.GET("/users/:id", routes.GetUser) // 用户详情
|
||||
authGroup.POST("/users/:id", routes.PostUser) // 更改用户
|
||||
authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户
|
||||
authGroup.GET("/me", routes.GetMe) // 获取自己账户
|
||||
authGroup.POST("/me", routes.PostMe) // 修改自己账户
|
||||
// 系统
|
||||
authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表
|
||||
authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
|
||||
// 文件
|
||||
authGroup.GET("/file", routes.GetFile) // 获取文件
|
||||
}
|
||||
|
||||
97
backend/model/variable.go
Normal file
97
backend/model/variable.go
Normal file
@@ -0,0 +1,97 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"crawlab/database"
|
||||
"errors"
|
||||
"github.com/apex/log"
|
||||
"github.com/globalsign/mgo/bson"
|
||||
"runtime/debug"
|
||||
)
|
||||
|
||||
/**
|
||||
全局变量
|
||||
*/
|
||||
|
||||
type Variable struct {
|
||||
Id bson.ObjectId `json:"_id" bson:"_id"`
|
||||
Key string `json:"key" bson:"key"`
|
||||
Value string `json:"value" bson:"value"`
|
||||
Remark string `json:"remark" bson:"remark"`
|
||||
}
|
||||
|
||||
func (model *Variable) Save() error {
|
||||
s, c := database.GetCol("variable")
|
||||
defer s.Close()
|
||||
|
||||
if err := c.UpdateId(model.Id, model); err != nil {
|
||||
log.Errorf("update variable error: %s", err.Error())
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (model *Variable) Add() error {
|
||||
s, c := database.GetCol("variable")
|
||||
defer s.Close()
|
||||
|
||||
// key 去重
|
||||
_, err := GetByKey(model.Key)
|
||||
if err == nil {
|
||||
return errors.New("key already exists")
|
||||
}
|
||||
|
||||
model.Id = bson.NewObjectId()
|
||||
if err := c.Insert(model); err != nil {
|
||||
log.Errorf("add variable error: %s", err.Error())
|
||||
debug.PrintStack()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (model *Variable) Delete() error {
|
||||
s, c := database.GetCol("variable")
|
||||
defer s.Close()
|
||||
|
||||
if err := c.RemoveId(model.Id); err != nil {
|
||||
log.Errorf("remove variable error: %s", err.Error())
|
||||
debug.PrintStack()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetByKey(key string) (Variable, error) {
|
||||
s, c := database.GetCol("variable")
|
||||
defer s.Close()
|
||||
|
||||
var model Variable
|
||||
if err := c.Find(bson.M{"key": key}).One(&model); err != nil {
|
||||
log.Errorf("variable found error: %s, key: %s", err.Error(), key)
|
||||
return model, err
|
||||
}
|
||||
return model, nil
|
||||
}
|
||||
|
||||
func GetVariable(id bson.ObjectId) (Variable, error) {
|
||||
s, c := database.GetCol("variable")
|
||||
defer s.Close()
|
||||
|
||||
var model Variable
|
||||
if err := c.FindId(id).One(&model); err != nil {
|
||||
log.Errorf("variable found error: %s", err.Error())
|
||||
return model, err
|
||||
}
|
||||
return model, nil
|
||||
}
|
||||
|
||||
func GetVariableList() []Variable {
|
||||
s, c := database.GetCol("variable")
|
||||
defer s.Close()
|
||||
|
||||
var list []Variable
|
||||
if err := c.Find(nil).All(&list); err != nil {
|
||||
|
||||
}
|
||||
return list
|
||||
}
|
||||
62
backend/routes/variable.go
Normal file
62
backend/routes/variable.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"crawlab/model"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/globalsign/mgo/bson"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// 新增
|
||||
func PostVariable(c *gin.Context) {
|
||||
var variable model.Variable
|
||||
if err := c.ShouldBindJSON(&variable); err != nil {
|
||||
HandleError(http.StatusBadRequest, c, err)
|
||||
return
|
||||
}
|
||||
if err := variable.Add(); err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
// 修改
|
||||
func PutVariable(c *gin.Context) {
|
||||
var id = c.Param("id")
|
||||
var variable model.Variable
|
||||
if err := c.ShouldBindJSON(&variable); err != nil {
|
||||
HandleError(http.StatusBadRequest, c, err)
|
||||
return
|
||||
}
|
||||
variable.Id = bson.ObjectIdHex(id)
|
||||
if err := variable.Save(); err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
// 删除
|
||||
func DeleteVariable(c *gin.Context) {
|
||||
var idStr = c.Param("id")
|
||||
var id = bson.ObjectIdHex(idStr)
|
||||
variable, err := model.GetVariable(id)
|
||||
if err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
variable.Id = id
|
||||
if err := variable.Delete(); err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
|
||||
}
|
||||
|
||||
// 列表
|
||||
func GetVariableList(c *gin.Context) {
|
||||
list := model.GetVariableList()
|
||||
HandleSuccessData(c, list)
|
||||
}
|
||||
@@ -200,6 +200,12 @@ func (s *Scheduler) Update() error {
|
||||
return err
|
||||
}
|
||||
|
||||
user, err := model.GetUserByUsername("admin")
|
||||
if err != nil {
|
||||
log.Errorf("get admin user error: %s", err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// 遍历任务列表
|
||||
for i := 0; i < len(sList); i++ {
|
||||
// 单个任务
|
||||
@@ -209,6 +215,11 @@ func (s *Scheduler) Update() error {
|
||||
continue
|
||||
}
|
||||
|
||||
// 兼容以前版本
|
||||
if job.UserId.Hex() == "" {
|
||||
job.UserId = user.Id
|
||||
}
|
||||
|
||||
// 添加到定时任务
|
||||
if err := s.AddJob(job); err != nil {
|
||||
log.Errorf("add job error: %s, job: %s, cron: %s", err.Error(), job.Name, job.Cron)
|
||||
|
||||
@@ -12,12 +12,11 @@ import (
|
||||
"github.com/apex/log"
|
||||
"github.com/globalsign/mgo"
|
||||
"github.com/globalsign/mgo/bson"
|
||||
uuid "github.com/satori/go.uuid"
|
||||
"github.com/satori/go.uuid"
|
||||
"github.com/spf13/viper"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type SpiderFileData struct {
|
||||
@@ -192,21 +191,14 @@ func PublishSpider(spider model.Spider) {
|
||||
md5 := filepath.Join(path, spider_handler.Md5File)
|
||||
if !utils.Exists(md5) {
|
||||
log.Infof("md5 file not found: %s", md5)
|
||||
spiderSync.RemoveSpiderFile()
|
||||
spiderSync.Download()
|
||||
spiderSync.CreateMd5File(gfFile.Md5)
|
||||
spiderSync.RemoveDownCreate(gfFile.Md5)
|
||||
return
|
||||
}
|
||||
// md5值不一样,则下载
|
||||
md5Str := utils.ReadFileOneLine(md5)
|
||||
// 去掉空格以及换行符
|
||||
md5Str = strings.Replace(md5Str, " ", "", -1)
|
||||
md5Str = strings.Replace(md5Str, "\n", "", -1)
|
||||
md5Str := utils.GetSpiderMd5Str(md5)
|
||||
if gfFile.Md5 != md5Str {
|
||||
log.Infof("md5 is different, gf-md5:%s, file-md5:%s", gfFile.Md5, md5Str)
|
||||
spiderSync.RemoveSpiderFile()
|
||||
spiderSync.Download()
|
||||
spiderSync.CreateMd5File(gfFile.Md5)
|
||||
spiderSync.RemoveDownCreate(gfFile.Md5)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,6 +38,12 @@ func (s *SpiderSync) CreateMd5File(md5 string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SpiderSync) RemoveDownCreate(md5 string) {
|
||||
s.RemoveSpiderFile()
|
||||
s.Download()
|
||||
s.CreateMd5File(md5)
|
||||
}
|
||||
|
||||
// 获得下载锁的key
|
||||
func (s *SpiderSync) GetLockDownloadKey(spiderId string) string {
|
||||
node, _ := model.GetCurrentNode()
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"crawlab/lib/cron"
|
||||
"crawlab/model"
|
||||
"crawlab/services/notification"
|
||||
"crawlab/services/spider_handler"
|
||||
"crawlab/utils"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -144,7 +145,11 @@ func SetEnv(cmd *exec.Cmd, envs []model.Env, taskId string, dataCol string) *exe
|
||||
cmd.Env = append(cmd.Env, env.Name+"="+env.Value)
|
||||
}
|
||||
|
||||
// TODO 全局环境变量
|
||||
// 全局环境变量
|
||||
variables := model.GetVariableList()
|
||||
for _, variable := range variables {
|
||||
cmd.Env = append(cmd.Env, variable.Key+"="+variable.Value)
|
||||
}
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -446,15 +451,9 @@ func ExecuteTask(id int) {
|
||||
t.Status = constants.StatusRunning // 任务状态
|
||||
t.WaitDuration = t.StartTs.Sub(t.CreateTs).Seconds() // 等待时长
|
||||
|
||||
// 判断爬虫文件是否存在
|
||||
gfFile := model.GetGridFs(spider.FileId)
|
||||
if gfFile == nil {
|
||||
t.Error = "找不到爬虫文件,请重新上传"
|
||||
t.Status = constants.StatusError
|
||||
t.FinishTs = time.Now() // 结束时间
|
||||
t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长
|
||||
t.TotalDuration = t.FinishTs.Sub(t.CreateTs).Seconds() // 总时长
|
||||
_ = t.Save()
|
||||
// 文件检查
|
||||
if err := SpiderFileCheck(t, spider); err != nil {
|
||||
log.Errorf("spider file check error: %s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
@@ -534,6 +533,30 @@ func ExecuteTask(id int) {
|
||||
log.Infof(GetWorkerPrefix(id) + "任务(ID:" + t.Id + ")" + "执行完毕. 消耗时间:" + durationStr + "秒")
|
||||
}
|
||||
|
||||
func SpiderFileCheck(t model.Task, spider model.Spider) error {
|
||||
// 判断爬虫文件是否存在
|
||||
gfFile := model.GetGridFs(spider.FileId)
|
||||
if gfFile == nil {
|
||||
t.Error = "找不到爬虫文件,请重新上传"
|
||||
t.Status = constants.StatusError
|
||||
t.FinishTs = time.Now() // 结束时间
|
||||
t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长
|
||||
t.TotalDuration = t.FinishTs.Sub(t.CreateTs).Seconds() // 总时长
|
||||
_ = t.Save()
|
||||
return errors.New(t.Error)
|
||||
}
|
||||
|
||||
// 判断md5值是否一致
|
||||
path := filepath.Join(viper.GetString("spider.path"), spider.Name)
|
||||
md5File := filepath.Join(path, spider_handler.Md5File)
|
||||
md5 := utils.GetSpiderMd5Str(md5File)
|
||||
if gfFile.Md5 != md5 {
|
||||
spiderSync := spider_handler.SpiderSync{Spider: spider}
|
||||
spiderSync.RemoveDownCreate(gfFile.Md5)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetTaskLog(id string) (logStr string, err error) {
|
||||
task, err := model.GetTask(id)
|
||||
|
||||
@@ -676,19 +699,6 @@ func AddTask(t model.Task) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func HandleTaskError(t model.Task, err error) {
|
||||
log.Error("handle task error:" + err.Error())
|
||||
t.Status = constants.StatusError
|
||||
t.Error = err.Error()
|
||||
t.FinishTs = time.Now()
|
||||
if err := t.Save(); err != nil {
|
||||
log.Errorf(err.Error())
|
||||
debug.PrintStack()
|
||||
return
|
||||
}
|
||||
debug.PrintStack()
|
||||
}
|
||||
|
||||
func GetTaskEmailMarkdownContent(t model.Task, s model.Spider) string {
|
||||
n, _ := model.GetNode(t.NodeId)
|
||||
errMsg := ""
|
||||
|
||||
@@ -33,7 +33,14 @@ func ReadFileOneLine(fileName string) string {
|
||||
return ""
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func GetSpiderMd5Str(file string) string {
|
||||
md5Str := ReadFileOneLine(file)
|
||||
// 去掉空格以及换行符
|
||||
md5Str = strings.Replace(md5Str, " ", "", -1)
|
||||
md5Str = strings.Replace(md5Str, "\n", "", -1)
|
||||
return md5Str
|
||||
}
|
||||
|
||||
// 创建文件
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
NODE_ENV='development'
|
||||
VUE_APP_BASE_URL=http://localhost:8000
|
||||
VUE_APP_CRAWLAB_BASE_URL=https://api.crawlab.cn
|
||||
VUE_APP_CRAWLAB_BASE_URL=https://api.crawlab.cn
|
||||
@@ -365,6 +365,11 @@ export default {
|
||||
'Never': '从不',
|
||||
'DingTalk Robot Webhook': '钉钉机器人 Webhook',
|
||||
'Wechat Robot Webhook': '微信机器人 Webhook',
|
||||
'Password Settings': '密码设置',
|
||||
'Notify Settings': '通知设置',
|
||||
'Global Variable': '全局变量',
|
||||
'Add Global Variable': '新增全局变量',
|
||||
'Are you sure to delete this global variable': '确定删除该全局变量?',
|
||||
|
||||
// 其他
|
||||
tagsView: {
|
||||
|
||||
@@ -9,6 +9,8 @@ const user = {
|
||||
avatar: '',
|
||||
roles: [],
|
||||
userList: [],
|
||||
globalVariableList: [],
|
||||
globalVariableForm: {},
|
||||
userForm: {},
|
||||
userInfo: undefined,
|
||||
adminPaths: [
|
||||
@@ -61,6 +63,9 @@ const user = {
|
||||
},
|
||||
SET_TOTAL_COUNT: (state, value) => {
|
||||
state.totalCount = value
|
||||
},
|
||||
SET_GLOBAL_VARIABLE_LIST: (state, value) => {
|
||||
state.globalVariableList = value
|
||||
}
|
||||
},
|
||||
|
||||
@@ -148,6 +153,23 @@ const user = {
|
||||
// 添加用户
|
||||
addUser ({ dispatch, commit, state }) {
|
||||
return request.put('/users', state.userForm)
|
||||
},
|
||||
// 新增全局变量
|
||||
addGlobalVariable ({ commit, state }) {
|
||||
return request.post(`/variable`, state.globalVariableForm)
|
||||
.then(() => {
|
||||
state.globalVariableForm = {}
|
||||
})
|
||||
},
|
||||
// 获取全局变量列表
|
||||
getGlobalVariable ({ commit, state }) {
|
||||
request.get('/variables').then((response) => {
|
||||
commit('SET_GLOBAL_VARIABLE_LIST', response.data.data)
|
||||
})
|
||||
},
|
||||
// 删除全局变量
|
||||
deleteGlobalVariable ({ commit, state }, id) {
|
||||
return request.delete(`/variable/${id}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,47 +1,98 @@
|
||||
<template>
|
||||
<div class="app-container">
|
||||
<!-- 新增全局变量 -->
|
||||
<el-dialog :title="$t('Add Global Variable')"
|
||||
:visible.sync="addDialogVisible">
|
||||
<el-form label-width="80px" ref="globalVariableForm">
|
||||
<el-form-item label="Key">
|
||||
<el-input size="small" v-model="globalVariableForm.key"/>
|
||||
</el-form-item>
|
||||
<el-form-item label="Value">
|
||||
<el-input size="small" v-model="globalVariableForm.value"/>
|
||||
</el-form-item>
|
||||
<el-form-item label="Remark">
|
||||
<el-input size="small" v-model="globalVariableForm.remark"/>
|
||||
</el-form-item>
|
||||
<el-form-item>
|
||||
<el-button @click="addDialogVisible = false" type="danger" size="small">{{$t('Cancel')}}</el-button>
|
||||
<el-button @click="addGlobalVariableHandle(false)" type="success" size="small">{{$t('Save')}}</el-button>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
</el-dialog>
|
||||
|
||||
<el-tabs v-model="activeName" @tab-click="tabActiveHandle">
|
||||
<el-tab-pane :label="$t('Password Settings')" name="passwordSetting">
|
||||
<el-form :model="userInfo" class="setting-form" ref="setting-form" label-width="200px" :rules="rules"
|
||||
inline-message>
|
||||
<el-form-item prop="username" :label="$t('Username')">
|
||||
<el-input v-model="userInfo.username" disabled></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item prop="password" :label="$t('Password')">
|
||||
<el-input v-model="userInfo.password" type="password" :placeholder="$t('Password')"></el-input>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
</el-tab-pane>
|
||||
<el-tab-pane :label="$t('Notify Settings')" name="notifySetting">
|
||||
<el-form :model="userInfo" class="setting-form" ref="setting-form" label-width="200px" :rules="rules"
|
||||
inline-message>
|
||||
<el-form-item :label="$t('Notification Trigger Timing')">
|
||||
<el-radio-group v-model="userInfo.setting.notification_trigger">
|
||||
<el-radio label="notification_trigger_on_task_end">
|
||||
{{$t('On Task End')}}
|
||||
</el-radio>
|
||||
<el-radio label="notification_trigger_on_task_error">
|
||||
{{$t('On Task Error')}}
|
||||
</el-radio>
|
||||
<el-radio label="notification_trigger_never">
|
||||
{{$t('Never')}}
|
||||
</el-radio>
|
||||
</el-radio-group>
|
||||
</el-form-item>
|
||||
<el-form-item prop="enabledNotifications" :label="$t('消息通知方式')">
|
||||
<el-checkbox-group v-model="userInfo.setting.enabled_notifications">
|
||||
<el-checkbox label="notification_type_mail">{{$t('邮件')}}</el-checkbox>
|
||||
<el-checkbox label="notification_type_ding_talk">{{$t('钉钉')}}</el-checkbox>
|
||||
<el-checkbox label="notification_type_wechat">{{$t('企业微信')}}</el-checkbox>
|
||||
</el-checkbox-group>
|
||||
</el-form-item>
|
||||
<el-form-item prop="email" :label="$t('Email')">
|
||||
<el-input v-model="userInfo.email" :placeholder="$t('Email')"></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item prop="setting.ding_talk_robot_webhook" :label="$t('DingTalk Robot Webhook')">
|
||||
<el-input v-model="userInfo.setting.ding_talk_robot_webhook"
|
||||
:placeholder="$t('DingTalk Robot Webhook')"></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item prop="setting.wechat_robot_webhook" :label="$t('Wechat Robot Webhook')">
|
||||
<el-input v-model="userInfo.setting.wechat_robot_webhook" :placeholder="$t('Wechat Robot Webhook')"></el-input>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
</el-tab-pane>
|
||||
<el-tab-pane :label="$t('Global Variable')" name="globalVariable">
|
||||
<el-form :inline="true">
|
||||
<el-form-item>
|
||||
<el-button size="mini" @click="addGlobalVariableHandle(true)"
|
||||
type="success">
|
||||
{{$t('Add')}}
|
||||
</el-button>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
<el-table :data="globalVariableList" border>
|
||||
<el-table-column prop="key" label="Key"/>
|
||||
<el-table-column prop="value" label="Value"/>
|
||||
<el-table-column prop="remark" label="Remark"/>
|
||||
<el-table-column prop="" :label="$t('Action')" width="80">
|
||||
<template slot-scope="scope">
|
||||
<el-button @click="deleteGlobalVariableHandle(scope.row._id)" icon="el-icon-delete" type="danger" size="mini"></el-button>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
</el-tab-pane>
|
||||
</el-tabs>
|
||||
<el-form :model="userInfo" class="setting-form" ref="setting-form" label-width="200px" :rules="rules"
|
||||
inline-message>
|
||||
<el-form-item prop="username" :label="$t('Username')">
|
||||
<el-input v-model="userInfo.username" disabled></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item prop="password" :label="$t('Password')">
|
||||
<el-input v-model="userInfo.password" type="password" :placeholder="$t('Password')"></el-input>
|
||||
</el-form-item>
|
||||
<div style="border-bottom: 1px solid #DCDFE6"></div>
|
||||
<el-form-item :label="$t('Notification Trigger Timing')">
|
||||
<el-radio-group v-model="userInfo.setting.notification_trigger">
|
||||
<el-radio label="notification_trigger_on_task_end">
|
||||
{{$t('On Task End')}}
|
||||
</el-radio>
|
||||
<el-radio label="notification_trigger_on_task_error">
|
||||
{{$t('On Task Error')}}
|
||||
</el-radio>
|
||||
<el-radio label="notification_trigger_never">
|
||||
{{$t('Never')}}
|
||||
</el-radio>
|
||||
</el-radio-group>
|
||||
</el-form-item>
|
||||
<el-form-item prop="enabledNotifications" :label="$t('消息通知方式')">
|
||||
<el-checkbox-group v-model="userInfo.setting.enabled_notifications">
|
||||
<el-checkbox label="notification_type_mail">{{$t('邮件')}}</el-checkbox>
|
||||
<el-checkbox label="notification_type_ding_talk">{{$t('钉钉')}}</el-checkbox>
|
||||
<el-checkbox label="notification_type_wechat">{{$t('企业微信')}}</el-checkbox>
|
||||
</el-checkbox-group>
|
||||
</el-form-item>
|
||||
<el-form-item prop="email" :label="$t('Email')">
|
||||
<el-input v-model="userInfo.email" :placeholder="$t('Email')"></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item prop="setting.ding_talk_robot_webhook" :label="$t('DingTalk Robot Webhook')">
|
||||
<el-input v-model="userInfo.setting.ding_talk_robot_webhook"
|
||||
:placeholder="$t('DingTalk Robot Webhook')"></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item prop="setting.wechat_robot_webhook" :label="$t('Wechat Robot Webhook')">
|
||||
<el-input v-model="userInfo.setting.wechat_robot_webhook" :placeholder="$t('Wechat Robot Webhook')"></el-input>
|
||||
</el-form-item>
|
||||
inline-message v-if="activeName !== 'globalVariable'">
|
||||
<el-form-item>
|
||||
<div class="buttons">
|
||||
<el-button type="success" @click="saveUserInfo">{{$t('Save')}}</el-button>
|
||||
<el-button size="small" type="success" @click="saveUserInfo">{{$t('Save')}}</el-button>
|
||||
</div>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
@@ -49,6 +100,7 @@
|
||||
</template>
|
||||
|
||||
<script>
|
||||
import { mapState } from 'vuex'
|
||||
export default {
|
||||
name: 'Setting',
|
||||
data () {
|
||||
@@ -92,16 +144,55 @@ export default {
|
||||
'setting.ding_talk_robot_webhook': [{ trigger: 'blur', validator: validateDingTalkRobotWebhook }],
|
||||
'setting.wechat_robot_webhook': [{ trigger: 'blur', validator: validateWechatRobotWebhook }]
|
||||
},
|
||||
isShowDingTalkAppSecret: false
|
||||
isShowDingTalkAppSecret: false,
|
||||
activeName: 'passwordSetting',
|
||||
addDialogVisible: false
|
||||
}
|
||||
},
|
||||
computed: {
|
||||
...mapState('user', [
|
||||
'globalVariableList',
|
||||
'globalVariableForm'
|
||||
])
|
||||
},
|
||||
methods: {
|
||||
deleteGlobalVariableHandle (id) {
|
||||
this.$confirm(this.$t('Are you sure to delete this global variable'), this.$t('Notification'), {
|
||||
confirmButtonText: this.$t('Confirm'),
|
||||
cancelButtonText: this.$t('Cancel'),
|
||||
type: 'warning'
|
||||
}).then(() => {
|
||||
this.$store.dispatch('user/deleteGlobalVariable', id).then(() => {
|
||||
this.$store.dispatch('user/getGlobalVariable')
|
||||
})
|
||||
}).catch(() => {})
|
||||
},
|
||||
addGlobalVariableHandle (isShow) {
|
||||
if (isShow) {
|
||||
this.addDialogVisible = true
|
||||
return
|
||||
}
|
||||
this.$store.dispatch('user/addGlobalVariable')
|
||||
.then(() => {
|
||||
this.addDialogVisible = false
|
||||
this.$st.sendEv('设置', '添加全局变量')
|
||||
})
|
||||
.then(() => {
|
||||
this.$store.dispatch('user/getGlobalVariable')
|
||||
})
|
||||
},
|
||||
getUserInfo () {
|
||||
const data = localStorage.getItem('user_info')
|
||||
if (!data) return {}
|
||||
if (!data) {
|
||||
return {}
|
||||
}
|
||||
this.userInfo = JSON.parse(data)
|
||||
if (!this.userInfo.setting) this.userInfo.setting = {}
|
||||
if (!this.userInfo.setting.enabled_notifications) this.userInfo.setting.enabled_notifications = []
|
||||
if (!this.userInfo.setting) {
|
||||
this.userInfo.setting = {}
|
||||
}
|
||||
if (!this.userInfo.setting.enabled_notifications) {
|
||||
this.userInfo.setting.enabled_notifications = []
|
||||
}
|
||||
},
|
||||
saveUserInfo () {
|
||||
this.$refs['setting-form'].validate(async valid => {
|
||||
@@ -114,10 +205,12 @@ export default {
|
||||
}
|
||||
})
|
||||
this.$st.sendEv('设置', '保存')
|
||||
}
|
||||
},
|
||||
tabActiveHandle () {}
|
||||
},
|
||||
async created () {
|
||||
await this.$store.dispatch('user/getInfo')
|
||||
this.$store.dispatch('user/getGlobalVariable')
|
||||
this.getUserInfo()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user