diff --git a/backend/main.go b/backend/main.go index 20d70b22..8841c3f4 100644 --- a/backend/main.go +++ b/backend/main.go @@ -136,81 +136,101 @@ func main() { } authGroup := app.Group("/", middlewares.AuthorizationMiddleware()) { - // 路由 // 节点 - authGroup.GET("/nodes", routes.GetNodeList) // 节点列表 - authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情 - authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点 - authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表 - authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表 - authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点 - authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表 - authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表 - authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表 - authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖 - authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖 - authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言 + { + authGroup.GET("/nodes", routes.GetNodeList) // 节点列表 + authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情 + authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点 + authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表 + authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表 + authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点 + authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表 + authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表 + authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表 + authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖 + authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖 + authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言 + } // 爬虫 - authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 - authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 - authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫 - authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫 - authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫 - authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 - authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID） - authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 - authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 - authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取 - authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 - authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改 - authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建 - authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建 - authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除 - authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名 - authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 - authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 - authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务 + { + authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 + authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 + authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫 + authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫 + authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫 + authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 + authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID） + authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 + authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 + authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取 + authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 + authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改 + authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建 + authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建 + authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除 + authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名 + authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 + authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 + authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务 + } // 可配置爬虫 - authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置 - authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置 - authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫 - authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫 - authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫 - authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫 - authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表 + { + authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置 + authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置 + authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫 + authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫 + authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫 + authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫 + authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表 + } // 任务 - authGroup.GET("/tasks", routes.GetTaskList) // 任务列表 - authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情 - authGroup.PUT("/tasks", routes.PutTask) // 派发任务 - authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务 - authGroup.DELETE("/tasks_multiple", routes.DeleteMultipleTask) // 删除多个任务 - authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) //删除指定状态的任务 - authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务 - authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志 - authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果 - authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果 + { + authGroup.GET("/tasks", routes.GetTaskList) // 任务列表 + authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情 + authGroup.PUT("/tasks", routes.PutTask) // 派发任务 + authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务 + authGroup.DELETE("/tasks_multiple", routes.DeleteMultipleTask) // 删除多个任务 + authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) //删除指定状态的任务 + authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务 + authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志 + authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果 + authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果 + } // 定时任务 - authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表 - authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情 - authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务 - authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务 - authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务 - authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务 - authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务 + { + authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表 + authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情 + authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务 + authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务 + authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务 + authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务 + authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务 + } + // 用户 + { + authGroup.GET("/users", routes.GetUserList) // 用户列表 + authGroup.GET("/users/:id", routes.GetUser) // 用户详情 + authGroup.POST("/users/:id", routes.PostUser) // 更改用户 + authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户 + authGroup.GET("/me", routes.GetMe) // 获取自己账户 + authGroup.POST("/me", routes.PostMe) // 修改自己账户 + } + // 系统 + { + authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表 + authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON + } + // 全局变量 + { + authGroup.POST("/variable", routes.PostVariable) // 新增 + authGroup.PUT("/variable/:id", routes.PutVariable) //修改 + authGroup.DELETE("/variable/:id", routes.DeleteVariable) //删除 + authGroup.GET("/variables", routes.GetVariableList) // 列表 + } // 统计数据 authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据 - // 用户 - authGroup.GET("/users", routes.GetUserList) // 用户列表 - authGroup.GET("/users/:id", routes.GetUser) // 用户详情 - authGroup.POST("/users/:id", routes.PostUser) // 更改用户 - authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户 - authGroup.GET("/me", routes.GetMe) // 获取自己账户 - authGroup.POST("/me", routes.PostMe) // 修改自己账户 // release版本 authGroup.GET("/version", routes.GetVersion) // 获取发布的版本 - // 系统 - authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表 - authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON // 文件 authGroup.GET("/file", routes.GetFile) // 获取文件 } diff --git a/backend/model/variable.go b/backend/model/variable.go new file mode 100644 index 00000000..3af2188e --- /dev/null +++ b/backend/model/variable.go @@ -0,0 +1,97 @@ +package model + +import ( + "crawlab/database" + "errors" + "github.com/apex/log" + "github.com/globalsign/mgo/bson" + "runtime/debug" +) + +/** +全局变量 +*/ + +type Variable struct { + Id bson.ObjectId `json:"_id" bson:"_id"` + Key string `json:"key" bson:"key"` + Value string `json:"value" bson:"value"` + Remark string `json:"remark" bson:"remark"` +} + +func (model *Variable) Save() error { + s, c := database.GetCol("variable") + defer s.Close() + + if err := c.UpdateId(model.Id, model); err != nil { + log.Errorf("update variable error: %s", err.Error()) + return err + } + return nil +} + +func (model *Variable) Add() error { + s, c := database.GetCol("variable") + defer s.Close() + + // key 去重 + _, err := GetByKey(model.Key) + if err == nil { + return errors.New("key already exists") + } + + model.Id = bson.NewObjectId() + if err := c.Insert(model); err != nil { + log.Errorf("add variable error: %s", err.Error()) + debug.PrintStack() + return err + } + return nil +} + +func (model *Variable) Delete() error { + s, c := database.GetCol("variable") + defer s.Close() + + if err := c.RemoveId(model.Id); err != nil { + log.Errorf("remove variable error: %s", err.Error()) + debug.PrintStack() + return err + } + return nil +} + +func GetByKey(key string) (Variable, error) { + s, c := database.GetCol("variable") + defer s.Close() + + var model Variable + if err := c.Find(bson.M{"key": key}).One(&model); err != nil { + log.Errorf("variable found error: %s, key: %s", err.Error(), key) + return model, err + } + return model, nil +} + +func GetVariable(id bson.ObjectId) (Variable, error) { + s, c := database.GetCol("variable") + defer s.Close() + + var model Variable + if err := c.FindId(id).One(&model); err != nil { + log.Errorf("variable found error: %s", err.Error()) + return model, err + } + return model, nil +} + +func GetVariableList() []Variable { + s, c := database.GetCol("variable") + defer s.Close() + + var list []Variable + if err := c.Find(nil).All(&list); err != nil { + + } + return list +} diff --git a/backend/routes/variable.go b/backend/routes/variable.go new file mode 100644 index 00000000..56f51ed7 --- /dev/null +++ b/backend/routes/variable.go @@ -0,0 +1,62 @@ +package routes + +import ( + "crawlab/model" + "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" + "net/http" +) + +// 新增 +func PostVariable(c *gin.Context) { + var variable model.Variable + if err := c.ShouldBindJSON(&variable); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + if err := variable.Add(); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + HandleSuccess(c) +} + +// 修改 +func PutVariable(c *gin.Context) { + var id = c.Param("id") + var variable model.Variable + if err := c.ShouldBindJSON(&variable); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + variable.Id = bson.ObjectIdHex(id) + if err := variable.Save(); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + HandleSuccess(c) +} + +// 删除 +func DeleteVariable(c *gin.Context) { + var idStr = c.Param("id") + var id = bson.ObjectIdHex(idStr) + variable, err := model.GetVariable(id) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + variable.Id = id + if err := variable.Delete(); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + HandleSuccess(c) + +} + +// 列表 +func GetVariableList(c *gin.Context) { + list := model.GetVariableList() + HandleSuccessData(c, list) +} diff --git a/backend/services/task.go b/backend/services/task.go index da4eabc8..b41f0d81 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -144,7 +144,11 @@ func SetEnv(cmd *exec.Cmd, envs []model.Env, taskId string, dataCol string) *exe cmd.Env = append(cmd.Env, env.Name+"="+env.Value) } - // TODO 全局环境变量 + // 全局环境变量 + variables := model.GetVariableList() + for _, variable := range variables { + cmd.Env = append(cmd.Env, variable.Key+"="+variable.Value) + } return cmd }