mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-22 17:31:03 +01:00
feature 新增全局环境变量
This commit is contained in:
陈景阳
152
backend/main.go
152
backend/main.go
@@ -136,81 +136,101 @@ func main() {
|
||||
}
|
||||
authGroup := app.Group("/", middlewares.AuthorizationMiddleware())
|
||||
{
|
||||
// 路由
|
||||
// 节点
|
||||
authGroup.GET("/nodes", routes.GetNodeList) // 节点列表
|
||||
authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情
|
||||
authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点
|
||||
authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表
|
||||
authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表
|
||||
authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点
|
||||
authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表
|
||||
authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表
|
||||
authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表
|
||||
authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖
|
||||
authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖
|
||||
authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言
|
||||
{
|
||||
authGroup.GET("/nodes", routes.GetNodeList) // 节点列表
|
||||
authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情
|
||||
authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点
|
||||
authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表
|
||||
authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表
|
||||
authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点
|
||||
authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表
|
||||
authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表
|
||||
authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表
|
||||
authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖
|
||||
authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖
|
||||
authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言
|
||||
}
|
||||
// 爬虫
|
||||
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
|
||||
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
|
||||
authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
|
||||
authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
|
||||
authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
|
||||
authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
|
||||
authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID）
|
||||
authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
|
||||
authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
|
||||
authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
|
||||
authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
|
||||
authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
|
||||
authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
|
||||
authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
|
||||
authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
|
||||
authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
|
||||
authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
|
||||
authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
|
||||
authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
|
||||
{
|
||||
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
|
||||
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
|
||||
authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
|
||||
authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
|
||||
authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
|
||||
authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
|
||||
authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID）
|
||||
authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
|
||||
authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
|
||||
authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
|
||||
authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
|
||||
authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
|
||||
authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
|
||||
authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
|
||||
authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
|
||||
authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
|
||||
authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
|
||||
authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
|
||||
authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
|
||||
}
|
||||
// 可配置爬虫
|
||||
authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置
|
||||
authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置
|
||||
authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫
|
||||
authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫
|
||||
authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫
|
||||
authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫
|
||||
authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表
|
||||
{
|
||||
authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置
|
||||
authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置
|
||||
authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫
|
||||
authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫
|
||||
authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫
|
||||
authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫
|
||||
authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表
|
||||
}
|
||||
// 任务
|
||||
authGroup.GET("/tasks", routes.GetTaskList) // 任务列表
|
||||
authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情
|
||||
authGroup.PUT("/tasks", routes.PutTask) // 派发任务
|
||||
authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务
|
||||
authGroup.DELETE("/tasks_multiple", routes.DeleteMultipleTask) // 删除多个任务
|
||||
authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) //删除指定状态的任务
|
||||
authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务
|
||||
authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志
|
||||
authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果
|
||||
authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果
|
||||
{
|
||||
authGroup.GET("/tasks", routes.GetTaskList) // 任务列表
|
||||
authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情
|
||||
authGroup.PUT("/tasks", routes.PutTask) // 派发任务
|
||||
authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务
|
||||
authGroup.DELETE("/tasks_multiple", routes.DeleteMultipleTask) // 删除多个任务
|
||||
authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) //删除指定状态的任务
|
||||
authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务
|
||||
authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志
|
||||
authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果
|
||||
authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果
|
||||
}
|
||||
// 定时任务
|
||||
authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表
|
||||
authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情
|
||||
authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务
|
||||
authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
|
||||
authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
|
||||
authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务
|
||||
authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务
|
||||
{
|
||||
authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表
|
||||
authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情
|
||||
authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务
|
||||
authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
|
||||
authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
|
||||
authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务
|
||||
authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务
|
||||
}
|
||||
// 用户
|
||||
{
|
||||
authGroup.GET("/users", routes.GetUserList) // 用户列表
|
||||
authGroup.GET("/users/:id", routes.GetUser) // 用户详情
|
||||
authGroup.POST("/users/:id", routes.PostUser) // 更改用户
|
||||
authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户
|
||||
authGroup.GET("/me", routes.GetMe) // 获取自己账户
|
||||
authGroup.POST("/me", routes.PostMe) // 修改自己账户
|
||||
}
|
||||
// 系统
|
||||
{
|
||||
authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表
|
||||
authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
|
||||
}
|
||||
// 全局变量
|
||||
{
|
||||
authGroup.POST("/variable", routes.PostVariable) // 新增
|
||||
authGroup.PUT("/variable/:id", routes.PutVariable) //修改
|
||||
authGroup.DELETE("/variable/:id", routes.DeleteVariable) //删除
|
||||
authGroup.GET("/variables", routes.GetVariableList) // 列表
|
||||
}
|
||||
// 统计数据
|
||||
authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据
|
||||
// 用户
|
||||
authGroup.GET("/users", routes.GetUserList) // 用户列表
|
||||
authGroup.GET("/users/:id", routes.GetUser) // 用户详情
|
||||
authGroup.POST("/users/:id", routes.PostUser) // 更改用户
|
||||
authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户
|
||||
authGroup.GET("/me", routes.GetMe) // 获取自己账户
|
||||
authGroup.POST("/me", routes.PostMe) // 修改自己账户
|
||||
// release版本
|
||||
authGroup.GET("/version", routes.GetVersion) // 获取发布的版本
|
||||
// 系统
|
||||
authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表
|
||||
authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
|
||||
// 文件
|
||||
authGroup.GET("/file", routes.GetFile) // 获取文件
|
||||
}
|
||||
|
||||
97
backend/model/variable.go
Normal file
97
backend/model/variable.go
Normal file
@@ -0,0 +1,97 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"crawlab/database"
|
||||
"errors"
|
||||
"github.com/apex/log"
|
||||
"github.com/globalsign/mgo/bson"
|
||||
"runtime/debug"
|
||||
)
|
||||
|
||||
/**
|
||||
全局变量
|
||||
*/
|
||||
|
||||
type Variable struct {
|
||||
Id bson.ObjectId `json:"_id" bson:"_id"`
|
||||
Key string `json:"key" bson:"key"`
|
||||
Value string `json:"value" bson:"value"`
|
||||
Remark string `json:"remark" bson:"remark"`
|
||||
}
|
||||
|
||||
func (model *Variable) Save() error {
|
||||
s, c := database.GetCol("variable")
|
||||
defer s.Close()
|
||||
|
||||
if err := c.UpdateId(model.Id, model); err != nil {
|
||||
log.Errorf("update variable error: %s", err.Error())
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (model *Variable) Add() error {
|
||||
s, c := database.GetCol("variable")
|
||||
defer s.Close()
|
||||
|
||||
// key 去重
|
||||
_, err := GetByKey(model.Key)
|
||||
if err == nil {
|
||||
return errors.New("key already exists")
|
||||
}
|
||||
|
||||
model.Id = bson.NewObjectId()
|
||||
if err := c.Insert(model); err != nil {
|
||||
log.Errorf("add variable error: %s", err.Error())
|
||||
debug.PrintStack()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (model *Variable) Delete() error {
|
||||
s, c := database.GetCol("variable")
|
||||
defer s.Close()
|
||||
|
||||
if err := c.RemoveId(model.Id); err != nil {
|
||||
log.Errorf("remove variable error: %s", err.Error())
|
||||
debug.PrintStack()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetByKey(key string) (Variable, error) {
|
||||
s, c := database.GetCol("variable")
|
||||
defer s.Close()
|
||||
|
||||
var model Variable
|
||||
if err := c.Find(bson.M{"key": key}).One(&model); err != nil {
|
||||
log.Errorf("variable found error: %s, key: %s", err.Error(), key)
|
||||
return model, err
|
||||
}
|
||||
return model, nil
|
||||
}
|
||||
|
||||
func GetVariable(id bson.ObjectId) (Variable, error) {
|
||||
s, c := database.GetCol("variable")
|
||||
defer s.Close()
|
||||
|
||||
var model Variable
|
||||
if err := c.FindId(id).One(&model); err != nil {
|
||||
log.Errorf("variable found error: %s", err.Error())
|
||||
return model, err
|
||||
}
|
||||
return model, nil
|
||||
}
|
||||
|
||||
func GetVariableList() []Variable {
|
||||
s, c := database.GetCol("variable")
|
||||
defer s.Close()
|
||||
|
||||
var list []Variable
|
||||
if err := c.Find(nil).All(&list); err != nil {
|
||||
|
||||
}
|
||||
return list
|
||||
}
|
||||
62
backend/routes/variable.go
Normal file
62
backend/routes/variable.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"crawlab/model"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/globalsign/mgo/bson"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// 新增
|
||||
func PostVariable(c *gin.Context) {
|
||||
var variable model.Variable
|
||||
if err := c.ShouldBindJSON(&variable); err != nil {
|
||||
HandleError(http.StatusBadRequest, c, err)
|
||||
return
|
||||
}
|
||||
if err := variable.Add(); err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
// 修改
|
||||
func PutVariable(c *gin.Context) {
|
||||
var id = c.Param("id")
|
||||
var variable model.Variable
|
||||
if err := c.ShouldBindJSON(&variable); err != nil {
|
||||
HandleError(http.StatusBadRequest, c, err)
|
||||
return
|
||||
}
|
||||
variable.Id = bson.ObjectIdHex(id)
|
||||
if err := variable.Save(); err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
// 删除
|
||||
func DeleteVariable(c *gin.Context) {
|
||||
var idStr = c.Param("id")
|
||||
var id = bson.ObjectIdHex(idStr)
|
||||
variable, err := model.GetVariable(id)
|
||||
if err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
variable.Id = id
|
||||
if err := variable.Delete(); err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
|
||||
}
|
||||
|
||||
// 列表
|
||||
func GetVariableList(c *gin.Context) {
|
||||
list := model.GetVariableList()
|
||||
HandleSuccessData(c, list)
|
||||
}
|
||||
@@ -144,7 +144,11 @@ func SetEnv(cmd *exec.Cmd, envs []model.Env, taskId string, dataCol string) *exe
|
||||
cmd.Env = append(cmd.Env, env.Name+"="+env.Value)
|
||||
}
|
||||
|
||||
// TODO 全局环境变量
|
||||
// 全局环境变量
|
||||
variables := model.GetVariableList()
|
||||
for _, variable := range variables {
|
||||
cmd.Env = append(cmd.Env, variable.Key+"="+variable.Value)
|
||||
}
|
||||
return cmd
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user