Merge branch 'develop' into develop

2026-01-22 17:31:03 +01:00 · 2019-10-28 11:06:13 +08:00
parent 30929ef9ec 418d728825
commit bcf3978600
18 changed files with 364 additions and 228 deletions
--- a/5
+++ b/5
@@ -14,7 +14,8 @@ ADD ./frontend /app
 WORKDIR /app

 # install frontend
-RUN npm install -g yarn && yarn install
+RUN npm config set unsafe-perm true
+RUN npm install -g yarn && yarn install --registry=https://registry.npm.taobao.org

 RUN npm run build:prod

@@ -56,4 +57,4 @@ EXPOSE 8080
 EXPOSE 8000

 # start backend
-CMD ["/bin/sh", "/app/docker_init.sh"]
+CMD ["/bin/sh", "/app/docker_init.sh"]
--- a/backend/model/log.go
+++ b/backend/model/log.go
@@ -24,7 +24,7 @@ func GetLocalLog(logPath string) (fileBytes []byte, err error) {
 	}
 	defer utils.Close(f)

-	const bufLen = 2 * 1024 * 1024
+	const bufLen = 1 * 1024 * 1024
 	logBuf := make([]byte, bufLen)

 	off := int64(0)
--- a/backend/model/node.go
+++ b/backend/model/node.go
@@ -4,6 +4,7 @@ import (
 	"crawlab/constants"
 	"crawlab/database"
 	"crawlab/services/register"
+	"errors"
 	"github.com/apex/log"
 	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
@@ -156,15 +157,20 @@ func GetNodeList(filter interface{}) ([]Node, error) {
 }

 func GetNode(id bson.ObjectId) (Node, error) {
+	var node Node
+
+	if id.Hex() == "" {
+		log.Infof("id is empty")
+		debug.PrintStack()
+		return node, errors.New("id is empty")
+	}
+
 	s, c := database.GetCol("nodes")
 	defer s.Close()

-	var node Node
 	if err := c.FindId(id).One(&node); err != nil {
-		if err != mgo.ErrNotFound {
-			log.Errorf(err.Error())
-			debug.PrintStack()
-		}
+		log.Errorf(err.Error())
+		debug.PrintStack()
 		return node, err
 	}
 	return node, nil
--- a/backend/model/schedule.go
+++ b/backend/model/schedule.go
@@ -16,6 +16,7 @@ type Schedule struct {
 	Description string        `json:"description" bson:"description"`
 	SpiderId    bson.ObjectId `json:"spider_id" bson:"spider_id"`
 	NodeId      bson.ObjectId `json:"node_id" bson:"node_id"`
+	NodeKey     string        `json:"node_key" bson:"node_key"`
 	Cron        string        `json:"cron" bson:"cron"`
 	EntryId     cron.EntryID  `json:"entry_id" bson:"entry_id"`
 	Param       string        `json:"param" bson:"param"`
@@ -38,6 +39,33 @@ func (sch *Schedule) Save() error {
 	return nil
 }

+func (sch *Schedule) Delete() error {
+	s, c := database.GetCol("schedules")
+	defer s.Close()
+	return c.RemoveId(sch.Id)
+}
+
+func (sch *Schedule) SyncNodeIdAndSpiderId(node Node, spider Spider) {
+	sch.syncNodeId(node)
+	sch.syncSpiderId(spider)
+}
+
+func (sch *Schedule) syncNodeId(node Node) {
+	if node.Id.Hex() == sch.NodeId.Hex() {
+		return
+	}
+	sch.NodeId = node.Id
+	_ = sch.Save()
+}
+
+func (sch *Schedule) syncSpiderId(spider Spider) {
+	if spider.Id.Hex() == sch.SpiderId.Hex() {
+		return
+	}
+	sch.SpiderId = spider.Id
+	_ = sch.Save()
+}
+
 func GetScheduleList(filter interface{}) ([]Schedule, error) {
 	s, c := database.GetCol("schedules")
 	defer s.Close()
@@ -47,11 +75,12 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) {
 		return schedules, err
 	}

-	for i, schedule := range schedules {
+	var schs []Schedule
+	for _, schedule := range schedules {
 		// 获取节点名称
 		if schedule.NodeId == bson.ObjectIdHex(constants.ObjectIdNull) {
 			// 选择所有节点
-			schedules[i].NodeName = "All Nodes"
+			schedule.NodeName = "All Nodes"
 		} else {
 			// 选择单一节点
 			node, err := GetNode(schedule.NodeId)
@@ -59,7 +88,7 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) {
 				log.Errorf(err.Error())
 				continue
 			}
-			schedules[i].NodeName = node.Name
+			schedule.NodeName = node.Name
 		}

 		// 获取爬虫名称
@@ -67,11 +96,13 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) {
 		if err != nil {
 			log.Errorf("get spider by id: %s, error: %s", schedule.SpiderId.Hex(), err.Error())
 			debug.PrintStack()
+			_ = schedule.Delete()
 			continue
 		}
-		schedules[i].SpiderName = spider.Name
+		schedule.SpiderName = spider.Name
+		schs = append(schs, schedule)
 	}
-	return schedules, nil
+	return schs, nil
 }

 func GetSchedule(id bson.ObjectId) (Schedule, error) {
@@ -93,7 +124,12 @@ func UpdateSchedule(id bson.ObjectId, item Schedule) error {
 	if err := c.FindId(id).One(&result); err != nil {
 		return err
 	}
+	node, err := GetNode(item.NodeId)
+	if err != nil {
+		return err
+	}

+	item.NodeKey = node.Key
 	if err := item.Save(); err != nil {
 		return err
 	}
@@ -104,9 +140,15 @@ func AddSchedule(item Schedule) error {
 	s, c := database.GetCol("schedules")
 	defer s.Close()

+	node, err := GetNode(item.NodeId)
+	if err != nil {
+		return err
+	}
+
 	item.Id = bson.NewObjectId()
 	item.CreateTs = time.Now()
 	item.UpdateTs = time.Now()
+	item.NodeKey = node.Key

 	if err := c.Insert(&item); err != nil {
 		debug.PrintStack()
--- a/backend/model/spider.go
+++ b/backend/model/spider.go
@@ -98,6 +98,12 @@ func (spider *Spider) GetLastTask() (Task, error) {
 	return tasks[0], nil
 }

+func (spider *Spider) Delete() error {
+	s, c := database.GetCol("spiders")
+	defer s.Close()
+	return c.RemoveId(spider.Id)
+}
+
 // 爬虫列表
 func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, int, error) {
 	s, c := database.GetCol("spiders")
--- a/backend/model/task.go
+++ b/backend/model/task.go
@@ -4,7 +4,6 @@ import (
 	"crawlab/constants"
 	"crawlab/database"
 	"github.com/apex/log"
-	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
 	"runtime/debug"
 	"time"
@@ -118,20 +117,16 @@ func GetTaskList(filter interface{}, skip int, limit int, sortKey string) ([]Tas
 	for i, task := range tasks {
 		// 获取爬虫名称
 		spider, err := task.GetSpider()
-		if err == mgo.ErrNotFound {
-			// do nothing
-		} else if err != nil {
-			return tasks, err
+		if spider.Id.Hex() == "" || err != nil {
+			_ = spider.Delete()
 		} else {
 			tasks[i].SpiderName = spider.DisplayName
 		}

 		// 获取节点名称
 		node, err := task.GetNode()
-		if err == mgo.ErrNotFound {
-			// do nothing
-		} else if err != nil {
-			return tasks, err
+		if node.Id.Hex() == "" || err != nil {
+			_ = task.Delete()
 		} else {
 			tasks[i].NodeName = node.Name
 		}
--- a/backend/routes/node.go
+++ b/backend/routes/node.go
@@ -15,9 +15,9 @@ func GetNodeList(c *gin.Context) {
 		return
 	}

-	for i, node := range nodes {
-		nodes[i].IsMaster = services.IsMasterNode(node.Id.Hex())
-	}
+	//for i, node := range nodes {
+	//	nodes[i].IsMaster = services.IsMasterNode(node.Id.Hex())
+	//}

 	c.JSON(http.StatusOK, Response{
 		Status:  "ok",
@@ -109,11 +109,11 @@ func GetSystemInfo(c *gin.Context) {
 	})
 }

-func DeleteNode(c *gin.Context)  {
+func DeleteNode(c *gin.Context) {
 	id := c.Param("id")
 	node, err := model.GetNode(bson.ObjectIdHex(id))
 	if err != nil {
-		HandleError(http.StatusInternalServerError, c ,err)
+		HandleError(http.StatusInternalServerError, c, err)
 		return
 	}
 	err = node.Delete()
--- a/backend/routes/schedule.go
+++ b/backend/routes/schedule.go
@@ -1,7 +1,6 @@
 package routes

 import (
-	"crawlab/constants"
 	"crawlab/model"
 	"crawlab/services"
 	"github.com/gin-gonic/gin"
@@ -46,13 +45,14 @@ func PostSchedule(c *gin.Context) {
 		HandleError(http.StatusBadRequest, c, err)
 		return
 	}
-	newItem.Id = bson.ObjectIdHex(id)

-	// 如果node_id为空，则置为空ObjectId
-	if newItem.NodeId == "" {
-		newItem.NodeId = bson.ObjectIdHex(constants.ObjectIdNull)
+	// 验证cron表达式
+	if err := services.ParserCron(newItem.Cron); err != nil {
+		HandleError(http.StatusOK, c, err)
+		return
 	}

+	newItem.Id = bson.ObjectIdHex(id)
 	// 更新数据库
 	if err := model.UpdateSchedule(bson.ObjectIdHex(id), newItem); err != nil {
 		HandleError(http.StatusInternalServerError, c, err)
@@ -80,9 +80,10 @@ func PutSchedule(c *gin.Context) {
 		return
 	}

-	// 如果node_id为空，则置为空ObjectId
-	if item.NodeId == "" {
-		item.NodeId = bson.ObjectIdHex(constants.ObjectIdNull)
+	// 验证cron表达式
+	if err := services.ParserCron(item.Cron); err != nil {
+		HandleError(http.StatusOK, c, err)
+		return
 	}

 	// 更新数据库
--- a/backend/services/log.go
+++ b/backend/services/log.go
@@ -15,6 +15,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime/debug"
+	"time"
 )

 // 任务日志频道映射
@@ -45,8 +46,14 @@ func GetRemoteLog(task model.Task) (logStr string, err error) {
 	// 生成频道，等待获取log
 	ch := TaskLogChanMap.ChanBlocked(task.Id)

-	// 此处阻塞，等待结果
-	logStr = <-ch
+	select {
+	case logStr = <-ch:
+		log.Infof("get remote log")
+		break
+	case <-time.After(30 * time.Second):
+		logStr = "get remote log timeout"
+		break
+	}

 	return logStr, nil
 }
--- a/backend/services/msg_handler/handler.go
+++ b/backend/services/msg_handler/handler.go
@@ -3,6 +3,7 @@ package msg_handler
 import (
 	"crawlab/constants"
 	"crawlab/entity"
+	"github.com/apex/log"
 )

 type Handler interface {
@@ -10,6 +11,7 @@ type Handler interface {
 }

 func GetMsgHandler(msg entity.NodeMessage) Handler {
+	log.Infof("received msg , type is : %s", msg.Type)
 	if msg.Type == constants.MsgTypeGetLog || msg.Type == constants.MsgTypeRemoveLog {
 		// 日志相关
 		return &Log{
--- a/backend/services/msg_handler/msg_log.go
+++ b/backend/services/msg_handler/msg_log.go
@@ -40,9 +40,12 @@ func (g *Log) get() error {
 		msgSd.Log = utils.BytesToString(logStr)
 	}
 	// 发布消息给主节点
-	if err := database.Pub(constants.ChannelMasterNode, msgSd); err != nil {
+	if err := utils.Pub(constants.ChannelMasterNode, msgSd); err != nil {
+		log.Errorf("pub log to master node error: %s", err.Error())
+		debug.PrintStack()
 		return err
 	}
+	log.Infof(msgSd.Log)
 	return nil
 }

--- a/backend/services/node.go
+++ b/backend/services/node.go
@@ -88,6 +88,8 @@ func UpdateNodeStatus() {
 		handleNodeInfo(key, data)
 	}

+	// 重新获取list
+	list, _ = database.RedisClient.HKeys("nodes")
 	// 重置不在redis的key为offline
 	model.ResetNodeStatusToOffline(list)
 }
@@ -110,13 +112,15 @@ func handleNodeInfo(key string, data Data) {
 	if err := c.Find(bson.M{"key": key}).One(&node); err != nil {
 		// 数据库不存在该节点
 		node = model.Node{
-			Key:      key,
-			Name:     data.Ip,
-			Ip:       data.Ip,
-			Port:     "8000",
-			Mac:      data.Mac,
-			Status:   constants.StatusOnline,
-			IsMaster: data.Master,
+			Key:          key,
+			Name:         data.Ip,
+			Ip:           data.Ip,
+			Port:         "8000",
+			Mac:          data.Mac,
+			Status:       constants.StatusOnline,
+			IsMaster:     data.Master,
+			UpdateTs:     time.Now(),
+			UpdateTsUnix: time.Now().Unix(),
 		}
 		if err := node.Add(); err != nil {
 			log.Errorf(err.Error())
@@ -125,6 +129,8 @@ func handleNodeInfo(key string, data Data) {
 	} else {
 		// 数据库存在该节点
 		node.Status = constants.StatusOnline
+		node.UpdateTs = time.Now()
+		node.UpdateTsUnix = time.Now().Unix()
 		if err := node.Save(); err != nil {
 			log.Errorf(err.Error())
 			return
@@ -205,6 +211,8 @@ func WorkerNodeCallback(message redis.Message) (err error) {
 	// 反序列化
 	msg := utils.GetMessage(message)
 	if err := msg_handler.GetMsgHandler(*msg).Handle(); err != nil {
+		log.Errorf("msg handler error: %s", err.Error())
+		debug.PrintStack()
 		return err
 	}
 	return nil
--- a/backend/services/schedule.go
+++ b/backend/services/schedule.go
@@ -5,7 +5,7 @@ import (
 	"crawlab/lib/cron"
 	"crawlab/model"
 	"github.com/apex/log"
-	uuid "github.com/satori/go.uuid"
+	"github.com/satori/go.uuid"
 	"runtime/debug"
 )

@@ -17,7 +17,22 @@ type Scheduler struct {

 func AddTask(s model.Schedule) func() {
 	return func() {
-		nodeId := s.NodeId
+		node, err := model.GetNodeByKey(s.NodeKey)
+		if err != nil || node.Id.Hex() == "" {
+			log.Errorf("get node by key error: %s", err.Error())
+			debug.PrintStack()
+			return
+		}
+
+		spider := model.GetSpiderByName(s.SpiderName)
+		if spider == nil || spider.Id.Hex() == "" {
+			log.Errorf("get spider by name error: %s", err.Error())
+			debug.PrintStack()
+			return
+		}
+
+		// 同步ID到定时任务
+		s.SyncNodeIdAndSpiderId(node, *spider)

 		// 生成任务ID
 		id := uuid.NewV4()
@@ -25,8 +40,8 @@ func AddTask(s model.Schedule) func() {
 		// 生成任务模型
 		t := model.Task{
 			Id:       id.String(),
-			SpiderId: s.SpiderId,
-			NodeId:   nodeId,
+			SpiderId: spider.Id,
+			NodeId:   node.Id,
 			Status:   constants.StatusPending,
 			Param:    s.Param,
 		}
@@ -107,6 +122,18 @@ func (s *Scheduler) RemoveAll() {
 	}
 }

+// 验证cron表达式是否正确
+func ParserCron(spec string) error {
+	parser := cron.NewParser(
+		cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor,
+	)
+
+	if _, err := parser.Parse(spec); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (s *Scheduler) Update() error {
 	// 删除所有定时任务
 	s.RemoveAll()
--- a/backend/services/task.go
+++ b/backend/services/task.go
@@ -100,91 +100,85 @@ func AssignTask(task model.Task) error {
 	return nil
 }

-// 执行shell命令
-func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (err error) {
-	log.Infof("cwd: " + cwd)
-	log.Infof("cmd: " + cmdStr)
+// 设置环境变量
+func SetEnv(cmd *exec.Cmd, envs []model.Env, taskId string, dataCol string) *exec.Cmd {
+	// 默认环境变量
+	cmd.Env = append(os.Environ(), "CRAWLAB_TASK_ID="+taskId)
+	cmd.Env = append(cmd.Env, "CRAWLAB_COLLECTION="+dataCol)
+	cmd.Env = append(cmd.Env, "PYTHONUNBUFFERED=0")
+	cmd.Env = append(cmd.Env, "PYTHONIOENCODING=utf-8")
+	cmd.Env = append(cmd.Env, "TZ=Asia/Shanghai")

-	// 生成执行命令
-	var cmd *exec.Cmd
-	if runtime.GOOS == constants.Windows {
-		cmd = exec.Command("cmd", "/C", cmdStr)
-	} else {
-		cmd = exec.Command("sh", "-c", cmdStr)
-	}
-
-	// 工作目录
-	cmd.Dir = cwd
-
-	// 指定stdout, stderr日志位置
-	fLog, err := os.Create(t.LogPath)
-	if err != nil {
-		HandleTaskError(t, err)
-		return err
-	}
-	defer utils.Close(fLog)
-	cmd.Stdout = fLog
-	cmd.Stderr = fLog
-
-	// 添加默认环境变量
-	cmd.Env = append(cmd.Env, "CRAWLAB_TASK_ID="+t.Id)
-	cmd.Env = append(cmd.Env, "CRAWLAB_COLLECTION="+s.Col)
-
-	// 添加任务环境变量
-	for _, env := range s.Envs {
+	//任务环境变量
+	for _, env := range envs {
 		cmd.Env = append(cmd.Env, env.Name+"="+env.Value)
 	}

-	// 起一个goroutine来监控进程
-	ch := utils.TaskExecChanMap.ChanBlocked(t.Id)
-	go func() {
-		// 传入信号，此处阻塞
-		signal := <-ch
-		log.Infof("cancel process signal: %s", signal)
-		if signal == constants.TaskCancel && cmd.Process != nil {
-			// 取消进程
-			if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil {
-				log.Errorf("process kill error: %s", err.Error())
-				debug.PrintStack()
-			}
-			t.Status = constants.StatusCancelled
-		} else {
-			// 保存任务
-			t.Status = constants.StatusFinished
-		}
-		t.FinishTs = time.Now()
-		t.Error = "user kill the process ..."
-		if err := t.Save(); err != nil {
-			log.Infof("save task error: %s", err.Error())
+	// TODO 全局环境变量
+	return cmd
+}
+
+func SetLogConfig(cmd *exec.Cmd, path string) error {
+	fLog, err := os.Create(path)
+	if err != nil {
+		log.Errorf("create task log file error: %s", path)
+		debug.PrintStack()
+		return err
+	}
+	cmd.Stdout = fLog
+	cmd.Stderr = fLog
+	return nil
+}
+
+func FinishOrCancelTask(ch chan string, cmd *exec.Cmd, t model.Task) {
+	// 传入信号，此处阻塞
+	signal := <-ch
+	log.Infof("process received signal: %s", signal)
+
+	if signal == constants.TaskCancel && cmd.Process != nil {
+		// 取消进程
+		if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil {
+			log.Errorf("process kill error: %s", err.Error())
 			debug.PrintStack()
-			return
+
+			t.Error = "kill process error: " + err.Error()
+			t.Status = constants.StatusError
+		} else {
+			t.Error = "user kill the process ..."
+			t.Status = constants.StatusCancelled
 		}
-	}()
+	} else {
+		// 保存任务
+		t.Status = constants.StatusFinished
+	}

-	// 在选择所有节点执行的时候，实际就是随机一个节点执行的，
-	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	t.FinishTs = time.Now()
+	_ = t.Save()
+}

-	// 异步启动进程
+func StartTaskProcess(cmd *exec.Cmd, t model.Task) error {
 	if err := cmd.Start(); err != nil {
 		log.Errorf("start spider error:{}", err.Error())
 		debug.PrintStack()
-		return err
-	}

-	// 保存pid到task
-	t.Pid = cmd.Process.Pid
-	if err := t.Save(); err != nil {
-		log.Errorf("save task pid error: %s", err.Error())
-		debug.PrintStack()
+		t.Error = "start task error: " + err.Error()
+		t.Status = constants.StatusError
+		t.FinishTs = time.Now()
+		_ = t.Save()
 		return err
 	}
-	// 同步等待进程完成
+	return nil
+}
+
+func WaitTaskProcess(cmd *exec.Cmd, t model.Task) error {
 	if err := cmd.Wait(); err != nil {
 		log.Errorf("wait process finish error: %s", err.Error())
 		debug.PrintStack()
+
 		if exitError, ok := err.(*exec.ExitError); ok {
 			exitCode := exitError.ExitCode()
 			log.Errorf("exit error, exit code: %d", exitCode)
+
 			// 非kill 的错误类型
 			if exitCode != -1 {
 				// 非手动kill保存为错误状态
@@ -194,6 +188,52 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e
 				_ = t.Save()
 			}
 		}
+
+		return err
+	}
+	return nil
+}
+
+// 执行shell命令
+func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (err error) {
+	log.Infof("cwd: %s", cwd)
+	log.Infof("cmd: %s", cmdStr)
+
+	// 生成执行命令
+	var cmd *exec.Cmd
+	if runtime.GOOS == constants.Windows {
+		cmd = exec.Command("cmd", "/C", cmdStr)
+	} else {
+		cmd = exec.Command("")
+		cmd = exec.Command("sh", "-c", cmdStr)
+	}
+
+	// 工作目录
+	cmd.Dir = cwd
+
+	// 日志配置
+	if err := SetLogConfig(cmd, t.LogPath); err != nil {
+		return err
+	}
+
+	// 环境变量配置
+	cmd = SetEnv(cmd, s.Envs, t.Id, s.Col)
+
+	// 起一个goroutine来监控进程
+	ch := utils.TaskExecChanMap.ChanBlocked(t.Id)
+
+	go FinishOrCancelTask(ch, cmd, t)
+
+	// kill的时候，可以kill所有的子进程
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+
+	// 启动进程
+	if err := StartTaskProcess(cmd, t); err != nil {
+		return err
+	}
+
+	// 同步等待进程完成
+	if err := WaitTaskProcess(cmd, t); err != nil {
 		return err
 	}
 	ch <- constants.TaskFinish
@@ -208,6 +248,7 @@ func MakeLogDir(t model.Task) (fileDir string, err error) {
 	// 如果日志目录不存在，生成该目录
 	if !utils.Exists(fileDir) {
 		if err := os.MkdirAll(fileDir, 0777); err != nil {
+			log.Errorf("execute task, make log dir error: %s", err.Error())
 			debug.PrintStack()
 			return "", err
 		}
@@ -272,82 +313,55 @@ func ExecuteTask(id int) {
 	// 获取当前节点
 	node, err := model.GetCurrentNode()
 	if err != nil {
-		log.Errorf(GetWorkerPrefix(id) + err.Error())
+		log.Errorf("execute task get current node error: %s", err.Error())
+		debug.PrintStack()
 		return
 	}

-	// 公共队列
-	queuePub := "tasks:public"
-
 	// 节点队列
 	queueCur := "tasks:node:" + node.Id.Hex()
-
 	// 节点队列任务
 	var msg string
-	msg, err = database.RedisClient.LPop(queueCur)
-	if err != nil {
-		if msg == "" {
-			// 节点队列没有任务，获取公共队列任务
-			msg, err = database.RedisClient.LPop(queuePub)
-			if err != nil {
-				if msg == "" {
-					// 公共队列没有任务
-					log.Debugf(GetWorkerPrefix(id) + "没有任务...")
-					return
-				} else {
-					log.Errorf(GetWorkerPrefix(id) + err.Error())
-					debug.PrintStack()
-					return
-				}
-			}
-		} else {
-			log.Errorf(GetWorkerPrefix(id) + err.Error())
-			debug.PrintStack()
-			return
+	if msg, err = database.RedisClient.LPop(queueCur); err != nil {
+		// 节点队列没有任务，获取公共队列任务
+		queuePub := "tasks:public"
+		if msg, err = database.RedisClient.LPop(queuePub); err != nil {
 		}
 	}

+	if msg == "" {
+		return
+	}
+
 	// 反序列化
 	tMsg := TaskMessage{}
 	if err := json.Unmarshal([]byte(msg), &tMsg); err != nil {
-		log.Errorf(GetWorkerPrefix(id) + err.Error())
-		debug.PrintStack()
+		log.Errorf("json string to struct error: %s", err.Error())
 		return
 	}

 	// 获取任务
 	t, err := model.GetTask(tMsg.Id)
 	if err != nil {
-		log.Errorf(GetWorkerPrefix(id) + err.Error())
+		log.Errorf("execute task, get task error: %s", err.Error())
 		return
 	}

 	// 获取爬虫
 	spider, err := t.GetSpider()
 	if err != nil {
-		log.Errorf(GetWorkerPrefix(id) + err.Error())
+		log.Errorf("execute task, get spider error: %s", err.Error())
 		return
 	}

 	// 创建日志目录
-	fileDir, err := MakeLogDir(t)
-	if err != nil {
-		log.Errorf(GetWorkerPrefix(id) + err.Error())
+	var fileDir string
+	if fileDir, err = MakeLogDir(t); err != nil {
 		return
 	}
-
 	// 获取日志文件路径
 	t.LogPath = GetLogFilePaths(fileDir)

-	// 创建日志目录文件夹
-	fileStdoutDir := filepath.Dir(t.LogPath)
-	if !utils.Exists(fileStdoutDir) {
-		if err := os.MkdirAll(fileStdoutDir, os.ModePerm); err != nil {
-			log.Errorf(GetWorkerPrefix(id) + err.Error())
-			return
-		}
-	}
-
 	// 工作目录
 	cwd := filepath.Join(
 		viper.GetString("spider.path"),
--- a/frontend/src/components/Common/CrawlConfirmDialog.vue
+++ b/frontend/src/components/Common/CrawlConfirmDialog.vue
@@ -9,9 +9,8 @@
    <el-form label-width="80px">
      <el-form-item :label="$t('Node')">
        <el-select v-model="nodeId">
-          <el-option value="" :label="$t('All Nodes')"/>
          <el-option
-            v-for="op in $store.state.node.nodeList"
+            v-for="op in nodeList"
            :key="op._id"
            :value="op._id"
            :disabled="op.status !== 'online'"
@@ -31,6 +30,7 @@
 </template>

 <script>
+import request from '../../api/request'
 export default {
  name: 'CrawlConfirmDialog',
  props: {
@@ -46,7 +46,8 @@ export default {
  data () {
    return {
      nodeId: '',
-      param: ''
+      param: '',
+      nodeList: []
    }
  },
  methods: {
@@ -61,6 +62,20 @@ export default {
      this.$emit('close')
      this.$st.sendEv('爬虫', '运行')
    }
+  },
+  created () {
+    // 节点列表
+    request.get('/nodes', {}).then(response => {
+      this.nodeList = response.data.data.map(d => {
+        d.systemInfo = {
+          os: '',
+          arch: '',
+          num_cpu: '',
+          executables: []
+        }
+        return d
+      })
+    })
  }
 }
 </script>
--- a/frontend/src/views/schedule/ScheduleList.vue
+++ b/frontend/src/views/schedule/ScheduleList.vue
@@ -14,9 +14,9 @@
        <el-form-item :label="$t('Schedule Name')" prop="name" required>
          <el-input v-model="scheduleForm.name" :placeholder="$t('Schedule Name')"></el-input>
        </el-form-item>
-        <el-form-item :label="$t('Node')" prop="node_id">
+        <el-form-item :label="$t('Node')" prop="node_id" required>
          <el-select v-model="scheduleForm.node_id">
-            <el-option :label="$t('All Nodes')" value="000000000000000000000000"></el-option>
+            <!--<el-option :label="$t('All Nodes')" value="000000000000000000000000"></el-option>-->
            <el-option
              v-for="op in nodeList"
              :key="op._id"
@@ -38,21 +38,22 @@
            </el-option>
          </el-select>
        </el-form-item>
-        <el-form-item :label="$t('schedules.cron')" prop="cron" :rules="cronRules" required>
-          <template slot="label">
-            <el-tooltip :content="$t('schedules.cron_format')"
-                        placement="top">
-              <span>
-                {{$t('schedules.cron')}}
-                <i class="fa fa-exclamation-circle"></i>
-              </span>
-            </el-tooltip>
-          </template>
-          <el-input style="width:calc(100% - 100px);padding-right:10px"
+        <!--:rules="cronRules"-->
+        <el-form-item :label="$t('schedules.cron')" prop="cron"  required>
+          <!--<template slot="label">-->
+            <!--<el-tooltip :content="$t('schedules.cron_format')"-->
+                        <!--placement="top">-->
+              <!--<span>-->
+                <!--{{$t('schedules.cron')}}-->
+                <!--<i class="fa fa-exclamation-circle"></i>-->
+              <!--</span>-->
+            <!--</el-tooltip>-->
+          <!--</template>-->
+          <el-input style="padding-right:10px"
                    v-model="scheduleForm.cron"
                    :placeholder="$t('schedules.cron')">
          </el-input>
-          <el-button size="small" style="width:100px" type="primary" @click="onShowCronDialog">{{$t('schedules.add_cron')}}</el-button>
+          <!--<el-button size="small" style="width:100px" type="primary" @click="onShowCronDialog">{{$t('schedules.add_cron')}}</el-button>-->
        </el-form-item>
        <el-form-item :label="$t('Execute Command')" prop="params">
          <el-input v-model="spider.cmd"
@@ -69,6 +70,7 @@
                    :placeholder="$t('Schedule Description')"></el-input>
        </el-form-item>
      </el-form>
+      <!--取消、保存-->
      <span slot="footer" class="dialog-footer">
        <el-button size="small" @click="onCancel">{{$t('Cancel')}}</el-button>
        <el-button size="small" type="primary" @click="onAddSubmit">{{$t('Submit')}}</el-button>
@@ -76,9 +78,9 @@
    </el-dialog>

    <!--cron generation popup-->
-    <el-dialog title="生成 Cron" :visible.sync="showCron">
-      <vcrontab @hide="showCron=false" @fill="onCrontabFill" :expression="expression"></vcrontab>
-    </el-dialog>
+    <!--<el-dialog title="生成 Cron" :visible.sync="showCron">-->
+      <!--<vcrontab @hide="showCron=false" @fill="onCrontabFill" :expression="expression"></vcrontab>-->
+    <!--</el-dialog>-->

    <el-card style="border-radius: 0">
      <!--filter-->
@@ -131,28 +133,15 @@
 </template>

 <script>
-import vcrontab from 'vcrontab'
+// import vcrontab from 'vcrontab'
+import request from '../../api/request'
 import {
  mapState
 } from 'vuex'

 export default {
  name: 'ScheduleList',
-  components: { vcrontab },
  data () {
-    const cronValidator = (rule, value, callback) => {
-      let patArr = []
-      for (let i = 0; i < 6; i++) {
-        patArr.push('[/*,0-9-]+')
-      }
-      const pat = '^' + patArr.join(' ') + '( [/*,0-9-]+)?' + '$'
-      if (!value) {
-        callback(new Error('cron cannot be empty'))
-      } else if (!value.match(pat)) {
-        callback(new Error('cron format is invalid'))
-      }
-      callback()
-    }
    return {
      columns: [
        { name: 'name', label: 'Name', width: '180' },
@@ -165,11 +154,10 @@ export default {
      isEdit: false,
      dialogTitle: '',
      dialogVisible: false,
-      cronRules: [
-        { validator: cronValidator, trigger: 'blur' }
-      ],
      showCron: false,
-      expression: ''
+      expression: '',
+      spiderList: [],
+      nodeList: []
    }
  },
  computed: {
@@ -177,12 +165,6 @@ export default {
      'scheduleList',
      'scheduleForm'
    ]),
-    ...mapState('spider', [
-      'spiderList'
-    ]),
-    ...mapState('node', [
-      'nodeList'
-    ]),
    filteredTableData () {
      return this.scheduleList
    },
@@ -211,19 +193,25 @@ export default {
    onAddSubmit () {
      this.$refs.scheduleForm.validate(res => {
        if (res) {
-          let action
          if (this.isEdit) {
-            action = 'editSchedule'
-          } else {
-            action = 'addSchedule'
-          }
-          this.$store.dispatch('schedule/' + action, this.scheduleForm._id)
-            .then(() => {
+            request.post(`/schedules/${this.scheduleForm._id}`, this.scheduleForm).then(response => {
+              if (response.data.error) {
+                this.$message.error(response.data.error)
+                return
+              }
              this.dialogVisible = false
-              setTimeout(() => {
-                this.$store.dispatch('schedule/getScheduleList')
-              }, 100)
+              this.$store.dispatch('schedule/getScheduleList')
            })
+          } else {
+            request.put('/schedules', this.scheduleForm).then(response => {
+              if (response.data.error) {
+                this.$message.error(response.data.error)
+                return
+              }
+              this.dialogVisible = false
+              this.$store.dispatch('schedule/getScheduleList')
+            })
+          }
        }
      })
      this.$st.sendEv('定时任务', '提交')
@@ -269,8 +257,25 @@ export default {
  },
  created () {
    this.$store.dispatch('schedule/getScheduleList')
-    // this.$store.dispatch('spider/getSpiderList')
-    this.$store.dispatch('node/getNodeList')
+
+    // 节点列表
+    request.get('/nodes', {}).then(response => {
+      this.nodeList = response.data.data.map(d => {
+        d.systemInfo = {
+          os: '',
+          arch: '',
+          num_cpu: '',
+          executables: []
+        }
+        return d
+      })
+    })
+
+    // 爬虫列表
+    request.get('/spiders', {})
+      .then(response => {
+        this.spiderList = response.data.data.list
+      })
  }
 }
 </script>
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -424,12 +424,10 @@ export default {
      this.dialogVisible = true
    },
    isShowRun (row) {
-      if (this.isCustomized(row)) {
-        // customized spider
-        return !!row.cmd
+      if (row.cmd) {
+        return true
      } else {
-        // configurable spider
-        return !!row.fields
+        return false
      }
    },
    isCustomized (row) {
--- a/frontend/src/views/task/TaskDetail.vue
+++ b/frontend/src/views/task/TaskDetail.vue
@@ -35,6 +35,7 @@ import {
 import TaskOverview from '../../components/Overview/TaskOverview'
 import GeneralTableView from '../../components/TableView/GeneralTableView'
 import LogView from '../../components/ScrollView/LogView'
+import request from '../../api/request'

 export default {
  name: 'TaskDetail',
@@ -46,12 +47,12 @@ export default {
  data () {
    return {
      activeTabName: 'overview',
-      handle: undefined
+      handle: undefined,
+      taskLog: ''
    }
  },
  computed: {
    ...mapState('task', [
-      'taskLog',
      'taskResultsData',
      'taskResultsTotalCount'
    ]),
@@ -97,18 +98,23 @@ export default {
    downloadCSV () {
      this.$store.dispatch('task/getTaskResultExcel', this.$route.params.id)
      this.$st.sendEv('任务详情-结果', '下载CSV')
+    },
+    getTaskLog () {
+      if (this.$route.params.id) {
+        request.get(`/tasks/${this.$route.params.id}/log`).then(response => {
+          this.taskLog = response.data.data
+        })
+      }
    }
  },
-  async created () {
-    await this.$store.dispatch('task/getTaskData', this.$route.params.id)
-    this.$store.dispatch('task/getTaskLog', this.$route.params.id)
+  created () {
+    this.$store.dispatch('task/getTaskData', this.$route.params.id)
    this.$store.dispatch('task/getTaskResults', this.$route.params.id)

-    if (this.taskForm && ['running'].includes(this.taskForm.status)) {
-      this.handle = setInterval(() => {
-        this.$store.dispatch('task/getTaskLog', this.$route.params.id)
-      }, 5000)
-    }
+    this.getTaskLog()
+    this.handle = setInterval(() => {
+      this.getTaskLog()
+    }, 5000)
  },
  destroyed () {
    clearInterval(this.handle)