Merge branch 'develop' into develop

backend/model/log.go

@@ -24,7 +24,7 @@ func GetLocalLog(logPath string) (fileBytes []byte, err error) {
 	}
 	defer utils.Close(f)
 
-	const bufLen = 2 * 1024 * 1024
+	const bufLen = 1 * 1024 * 1024
 	logBuf := make([]byte, bufLen)
 
 	off := int64(0)

backend/model/node.go

@@ -4,6 +4,7 @@ import (
 	"crawlab/constants"
 	"crawlab/database"
 	"crawlab/services/register"
+	"errors"
 	"github.com/apex/log"
 	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
@@ -156,15 +157,20 @@ func GetNodeList(filter interface{}) ([]Node, error) {
 }
 
 func GetNode(id bson.ObjectId) (Node, error) {
+	var node Node
+
+	if id.Hex() == "" {
+		log.Infof("id is empty")
+		debug.PrintStack()
+		return node, errors.New("id is empty")
+	}
+
 	s, c := database.GetCol("nodes")
 	defer s.Close()
 
-	var node Node
 	if err := c.FindId(id).One(&node); err != nil {
-		if err != mgo.ErrNotFound {
-			log.Errorf(err.Error())
-			debug.PrintStack()
-		}
+		log.Errorf(err.Error())
+		debug.PrintStack()
 		return node, err
 	}
 	return node, nil

backend/model/schedule.go

@@ -16,6 +16,7 @@ type Schedule struct {
 	Description string        `json:"description" bson:"description"`
 	SpiderId    bson.ObjectId `json:"spider_id" bson:"spider_id"`
 	NodeId      bson.ObjectId `json:"node_id" bson:"node_id"`
+	NodeKey     string        `json:"node_key" bson:"node_key"`
 	Cron        string        `json:"cron" bson:"cron"`
 	EntryId     cron.EntryID  `json:"entry_id" bson:"entry_id"`
 	Param       string        `json:"param" bson:"param"`
@@ -38,6 +39,33 @@ func (sch *Schedule) Save() error {
 	return nil
 }
 
+func (sch *Schedule) Delete() error {
+	s, c := database.GetCol("schedules")
+	defer s.Close()
+	return c.RemoveId(sch.Id)
+}
+
+func (sch *Schedule) SyncNodeIdAndSpiderId(node Node, spider Spider) {
+	sch.syncNodeId(node)
+	sch.syncSpiderId(spider)
+}
+
+func (sch *Schedule) syncNodeId(node Node) {
+	if node.Id.Hex() == sch.NodeId.Hex() {
+		return
+	}
+	sch.NodeId = node.Id
+	_ = sch.Save()
+}
+
+func (sch *Schedule) syncSpiderId(spider Spider) {
+	if spider.Id.Hex() == sch.SpiderId.Hex() {
+		return
+	}
+	sch.SpiderId = spider.Id
+	_ = sch.Save()
+}
+
 func GetScheduleList(filter interface{}) ([]Schedule, error) {
 	s, c := database.GetCol("schedules")
 	defer s.Close()
@@ -47,11 +75,12 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) {
 		return schedules, err
 	}
 
-	for i, schedule := range schedules {
+	var schs []Schedule
+	for _, schedule := range schedules {
 		// 获取节点名称
 		if schedule.NodeId == bson.ObjectIdHex(constants.ObjectIdNull) {
 			// 选择所有节点
-			schedules[i].NodeName = "All Nodes"
+			schedule.NodeName = "All Nodes"
 		} else {
 			// 选择单一节点
 			node, err := GetNode(schedule.NodeId)
@@ -59,7 +88,7 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) {
 				log.Errorf(err.Error())
 				continue
 			}
-			schedules[i].NodeName = node.Name
+			schedule.NodeName = node.Name
 		}
 
 		// 获取爬虫名称
@@ -67,11 +96,13 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) {
 		if err != nil {
 			log.Errorf("get spider by id: %s, error: %s", schedule.SpiderId.Hex(), err.Error())
 			debug.PrintStack()
+			_ = schedule.Delete()
 			continue
 		}
-		schedules[i].SpiderName = spider.Name
+		schedule.SpiderName = spider.Name
+		schs = append(schs, schedule)
 	}
-	return schedules, nil
+	return schs, nil
 }
 
 func GetSchedule(id bson.ObjectId) (Schedule, error) {
@@ -93,7 +124,12 @@ func UpdateSchedule(id bson.ObjectId, item Schedule) error {
 	if err := c.FindId(id).One(&result); err != nil {
 		return err
 	}
+	node, err := GetNode(item.NodeId)
+	if err != nil {
+		return err
+	}
 
+	item.NodeKey = node.Key
 	if err := item.Save(); err != nil {
 		return err
 	}
@@ -104,9 +140,15 @@ func AddSchedule(item Schedule) error {
 	s, c := database.GetCol("schedules")
 	defer s.Close()
 
+	node, err := GetNode(item.NodeId)
+	if err != nil {
+		return err
+	}
+
 	item.Id = bson.NewObjectId()
 	item.CreateTs = time.Now()
 	item.UpdateTs = time.Now()
+	item.NodeKey = node.Key
 
 	if err := c.Insert(&item); err != nil {
 		debug.PrintStack()

backend/model/spider.go

@@ -98,6 +98,12 @@ func (spider *Spider) GetLastTask() (Task, error) {
 	return tasks[0], nil
 }
 
+func (spider *Spider) Delete() error {
+	s, c := database.GetCol("spiders")
+	defer s.Close()
+	return c.RemoveId(spider.Id)
+}
+
 // 爬虫列表
 func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, int, error) {
 	s, c := database.GetCol("spiders")

backend/model/task.go

@@ -4,7 +4,6 @@ import (
 	"crawlab/constants"
 	"crawlab/database"
 	"github.com/apex/log"
-	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
 	"runtime/debug"
 	"time"
@@ -118,20 +117,16 @@ func GetTaskList(filter interface{}, skip int, limit int, sortKey string) ([]Tas
 	for i, task := range tasks {
 		// 获取爬虫名称
 		spider, err := task.GetSpider()
-		if err == mgo.ErrNotFound {
-			// do nothing
-		} else if err != nil {
-			return tasks, err
+		if spider.Id.Hex() == "" || err != nil {
+			_ = spider.Delete()
 		} else {
 			tasks[i].SpiderName = spider.DisplayName
 		}
 
 		// 获取节点名称
 		node, err := task.GetNode()
-		if err == mgo.ErrNotFound {
-			// do nothing
-		} else if err != nil {
-			return tasks, err
+		if node.Id.Hex() == "" || err != nil {
+			_ = task.Delete()
 		} else {
 			tasks[i].NodeName = node.Name
 		}