git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-28 17:50:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #495 from wo10378931/develop

Browse Source 
Develop
This commit is contained in:
Marvin Zhang
2020-02-01 13:52:01 +08:00
committed by GitHub
parent 8bea3f62cf 8f54633eff
commit fa01b0fdfa
13 changed files with 472 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/conf/config.yml
View File

@@ -46,4 +46,4 @@ notification:
senderIdentity: ''
smtp:
user: ''
password: ''
password: ''

152
backend/main.go
View File

@@ -138,79 +138,99 @@ func main() {
}
authGroup := app.Group("/", middlewares.AuthorizationMiddleware())
{
// 路由
// 节点
authGroup.GET("/nodes", routes.GetNodeList) // 节点列表
authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情
authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点
authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表
authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表
authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点
authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表
authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表
authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表
authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖
authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖
authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言
{
authGroup.GET("/nodes", routes.GetNodeList) // 节点列表
authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情
authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点
authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表
authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表
authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点
authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表
authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表
authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表
authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖
authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖
authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言
}
// 爬虫
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID）
authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
{
authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID）
authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
}
// 可配置爬虫
authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置
authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置
authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫
authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫
authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫
authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫
authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表
{
authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置
authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置
authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫
authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫
authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫
authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫
authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表
}
// 任务
authGroup.GET("/tasks", routes.GetTaskList) // 任务列表
authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情
authGroup.PUT("/tasks", routes.PutTask) // 派发任务
authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务
authGroup.DELETE("/tasks_multiple", routes.DeleteMultipleTask) // 删除多个任务
authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) //删除指定状态的任务
authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务
authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志
authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果
authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果
{
authGroup.GET("/tasks", routes.GetTaskList) // 任务列表
authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情
authGroup.PUT("/tasks", routes.PutTask) // 派发任务
authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务
authGroup.DELETE("/tasks_multiple", routes.DeleteMultipleTask) // 删除多个任务
authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) //删除指定状态的任务
authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务
authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志
authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果
authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果
}
// 定时任务
authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表
authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情
authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务
authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务
authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务
{
authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表
authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情
authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务
authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务
authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务
}
// 用户
{
authGroup.GET("/users", routes.GetUserList) // 用户列表
authGroup.GET("/users/:id", routes.GetUser) // 用户详情
authGroup.POST("/users/:id", routes.PostUser) // 更改用户
authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户
authGroup.GET("/me", routes.GetMe) // 获取自己账户
authGroup.POST("/me", routes.PostMe) // 修改自己账户
}
// 系统
{
authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表
authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
}
// 全局变量
{
authGroup.POST("/variable", routes.PostVariable) // 新增
authGroup.PUT("/variable/:id", routes.PutVariable) //修改
authGroup.DELETE("/variable/:id", routes.DeleteVariable) //删除
authGroup.GET("/variables", routes.GetVariableList) // 列表
}
// 统计数据
authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据
// 用户
authGroup.GET("/users", routes.GetUserList) // 用户列表
authGroup.GET("/users/:id", routes.GetUser) // 用户详情
authGroup.POST("/users/:id", routes.PostUser) // 更改用户
authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户
authGroup.GET("/me", routes.GetMe) // 获取自己账户
authGroup.POST("/me", routes.PostMe) // 修改自己账户
// 系统
authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表
authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
// 文件
authGroup.GET("/file", routes.GetFile) // 获取文件
}

97
backend/model/variable.go Normal file
View File

@@ -0,0 +1,97 @@
package model
import (
"crawlab/database"
"errors"
"github.com/apex/log"
"github.com/globalsign/mgo/bson"
"runtime/debug"
)
/**
全局变量
*/
type Variable struct {
Id bson.ObjectId `json:"_id" bson:"_id"`
Key string `json:"key" bson:"key"`
Value string `json:"value" bson:"value"`
Remark string `json:"remark" bson:"remark"`
}
func (model *Variable) Save() error {
s, c := database.GetCol("variable")
defer s.Close()
if err := c.UpdateId(model.Id, model); err != nil {
log.Errorf("update variable error: %s", err.Error())
return err
}
return nil
}
func (model *Variable) Add() error {
s, c := database.GetCol("variable")
defer s.Close()
// key 去重
_, err := GetByKey(model.Key)
if err == nil {
return errors.New("key already exists")
}
model.Id = bson.NewObjectId()
if err := c.Insert(model); err != nil {
log.Errorf("add variable error: %s", err.Error())
debug.PrintStack()
return err
}
return nil
}
func (model *Variable) Delete() error {
s, c := database.GetCol("variable")
defer s.Close()
if err := c.RemoveId(model.Id); err != nil {
log.Errorf("remove variable error: %s", err.Error())
debug.PrintStack()
return err
}
return nil
}
func GetByKey(key string) (Variable, error) {
s, c := database.GetCol("variable")
defer s.Close()
var model Variable
if err := c.Find(bson.M{"key": key}).One(&model); err != nil {
log.Errorf("variable found error: %s, key: %s", err.Error(), key)
return model, err
}
return model, nil
}
func GetVariable(id bson.ObjectId) (Variable, error) {
s, c := database.GetCol("variable")
defer s.Close()
var model Variable
if err := c.FindId(id).One(&model); err != nil {
log.Errorf("variable found error: %s", err.Error())
return model, err
}
return model, nil
}
func GetVariableList() []Variable {
s, c := database.GetCol("variable")
defer s.Close()
var list []Variable
if err := c.Find(nil).All(&list); err != nil {
}
return list
}

62
backend/routes/variable.go Normal file
View File

@@ -0,0 +1,62 @@
package routes
import (
"crawlab/model"
"github.com/gin-gonic/gin"
"github.com/globalsign/mgo/bson"
"net/http"
)
// 新增
func PostVariable(c *gin.Context) {
var variable model.Variable
if err := c.ShouldBindJSON(&variable); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
if err := variable.Add(); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
HandleSuccess(c)
}
// 修改
func PutVariable(c *gin.Context) {
var id = c.Param("id")
var variable model.Variable
if err := c.ShouldBindJSON(&variable); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
variable.Id = bson.ObjectIdHex(id)
if err := variable.Save(); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
HandleSuccess(c)
}
// 删除
func DeleteVariable(c *gin.Context) {
var idStr = c.Param("id")
var id = bson.ObjectIdHex(idStr)
variable, err := model.GetVariable(id)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
variable.Id = id
if err := variable.Delete(); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
HandleSuccess(c)
}
// 列表
func GetVariableList(c *gin.Context) {
list := model.GetVariableList()
HandleSuccessData(c, list)
}

11
backend/services/schedule.go
View File

@@ -200,6 +200,12 @@ func (s *Scheduler) Update() error {
return err
}
user, err := model.GetUserByUsername("admin")
if err != nil {
log.Errorf("get admin user error: %s", err.Error())
return err
}
// 遍历任务列表
for i := 0; i < len(sList); i++ {
// 单个任务
@@ -209,6 +215,11 @@ func (s *Scheduler) Update() error {
continue
}
// 兼容以前版本
if job.UserId.Hex() == "" {
job.UserId = user.Id
}
// 添加到定时任务
if err := s.AddJob(job); err != nil {
log.Errorf("add job error: %s, job: %s, cron: %s", err.Error(), job.Name, job.Cron)

16
backend/services/spider.go
View File

@@ -12,12 +12,11 @@ import (
"github.com/apex/log"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
uuid "github.com/satori/go.uuid"
"github.com/satori/go.uuid"
"github.com/spf13/viper"
"os"
"path/filepath"
"runtime/debug"
"strings"
)
type SpiderFileData struct {
@@ -192,21 +191,14 @@ func PublishSpider(spider model.Spider) {
md5 := filepath.Join(path, spider_handler.Md5File)
if !utils.Exists(md5) {
log.Infof("md5 file not found: %s", md5)
spiderSync.RemoveSpiderFile()
spiderSync.Download()
spiderSync.CreateMd5File(gfFile.Md5)
spiderSync.RemoveDownCreate(gfFile.Md5)
return
}
// md5值不一样，则下载
md5Str := utils.ReadFileOneLine(md5)
// 去掉空格以及换行符
md5Str = strings.Replace(md5Str, " ", "", -1)
md5Str = strings.Replace(md5Str, "\n", "", -1)
md5Str := utils.GetSpiderMd5Str(md5)
if gfFile.Md5 != md5Str {
log.Infof("md5 is different, gf-md5:%s, file-md5:%s", gfFile.Md5, md5Str)
spiderSync.RemoveSpiderFile()
spiderSync.Download()
spiderSync.CreateMd5File(gfFile.Md5)
spiderSync.RemoveDownCreate(gfFile.Md5)
return
}
}

6
backend/services/spider_handler/spider.go
View File

@@ -38,6 +38,12 @@ func (s *SpiderSync) CreateMd5File(md5 string) {
}
}
func (s *SpiderSync) RemoveDownCreate(md5 string) {
s.RemoveSpiderFile()
s.Download()
s.CreateMd5File(md5)
}
// 获得下载锁的key
func (s *SpiderSync) GetLockDownloadKey(spiderId string) string {
node, _ := model.GetCurrentNode()

56
backend/services/task.go
View File

@@ -7,6 +7,7 @@ import (
"crawlab/lib/cron"
"crawlab/model"
"crawlab/services/notification"
"crawlab/services/spider_handler"
"crawlab/utils"
"encoding/json"
"errors"
@@ -144,7 +145,11 @@ func SetEnv(cmd *exec.Cmd, envs []model.Env, taskId string, dataCol string) *exe
cmd.Env = append(cmd.Env, env.Name+"="+env.Value)
}
// TODO 全局环境变量
// 全局环境变量
variables := model.GetVariableList()
for _, variable := range variables {
cmd.Env = append(cmd.Env, variable.Key+"="+variable.Value)
}
return cmd
}
@@ -446,15 +451,9 @@ func ExecuteTask(id int) {
t.Status = constants.StatusRunning // 任务状态
t.WaitDuration = t.StartTs.Sub(t.CreateTs).Seconds() // 等待时长
// 判断爬虫文件是否存在
gfFile := model.GetGridFs(spider.FileId)
if gfFile == nil {
t.Error = "找不到爬虫文件，请重新上传"
t.Status = constants.StatusError
t.FinishTs = time.Now() // 结束时间
t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长
t.TotalDuration = t.FinishTs.Sub(t.CreateTs).Seconds() // 总时长
_ = t.Save()
// 文件检查
if err := SpiderFileCheck(t, spider); err != nil {
log.Errorf("spider file check error: %s", err.Error())
return
}
@@ -534,6 +533,30 @@ func ExecuteTask(id int) {
log.Infof(GetWorkerPrefix(id) + "任务(ID:" + t.Id + ")" + "执行完毕. 消耗时间:" + durationStr + "秒")
}
func SpiderFileCheck(t model.Task, spider model.Spider) error {
// 判断爬虫文件是否存在
gfFile := model.GetGridFs(spider.FileId)
if gfFile == nil {
t.Error = "找不到爬虫文件，请重新上传"
t.Status = constants.StatusError
t.FinishTs = time.Now() // 结束时间
t.RuntimeDuration = t.FinishTs.Sub(t.StartTs).Seconds() // 运行时长
t.TotalDuration = t.FinishTs.Sub(t.CreateTs).Seconds() // 总时长
_ = t.Save()
return errors.New(t.Error)
}
// 判断md5值是否一致
path := filepath.Join(viper.GetString("spider.path"), spider.Name)
md5File := filepath.Join(path, spider_handler.Md5File)
md5 := utils.GetSpiderMd5Str(md5File)
if gfFile.Md5 != md5 {
spiderSync := spider_handler.SpiderSync{Spider: spider}
spiderSync.RemoveDownCreate(gfFile.Md5)
}
return nil
}
func GetTaskLog(id string) (logStr string, err error) {
task, err := model.GetTask(id)
@@ -676,19 +699,6 @@ func AddTask(t model.Task) error {
return nil
}
func HandleTaskError(t model.Task, err error) {
log.Error("handle task error:" + err.Error())
t.Status = constants.StatusError
t.Error = err.Error()
t.FinishTs = time.Now()
if err := t.Save(); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
}
debug.PrintStack()
}
func GetTaskEmailMarkdownContent(t model.Task, s model.Spider) string {
n, _ := model.GetNode(t.NodeId)
errMsg := ""

7
backend/utils/file.go
View File

@@ -33,7 +33,14 @@ func ReadFileOneLine(fileName string) string {
return ""
}
return line
}
func GetSpiderMd5Str(file string) string {
md5Str := ReadFileOneLine(file)
// 去掉空格以及换行符
md5Str = strings.Replace(md5Str, " ", "", -1)
md5Str = strings.Replace(md5Str, "\n", "", -1)
return md5Str
}
// 创建文件