updated Dockerfile

backend/conf/config.yml

@@ -1,7 +1,11 @@
+api:
+  address: "localhost:8000"
 mongo:
   host: localhost
   port: 27017
   db: crawlab_test
+  username: ""
+  password: ""
 redis:
   network: tcp
   address: "localhost:6379"
@@ -11,10 +15,10 @@ log:
 server:
   host: 0.0.0.0
   port: 8000
-  master: "Y"
+  master: "N"
   secret: "crawlab"
 spider:
-  path: "/Users/yeqing/projects/crawlab/spiders"
+  path: "/app/spiders"
 task:
   workers: 4
 other:

backend/model/node.go

@@ -21,8 +21,9 @@ type Node struct {
 	// 前端展示
 	IsMaster bool `json:"is_master"`
 
-	UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
-	CreateTs time.Time `json:"create_ts" bson:"create_ts"`
+	UpdateTs     time.Time `json:"update_ts" bson:"update_ts"`
+	CreateTs     time.Time `json:"create_ts" bson:"create_ts"`
+	UpdateTsUnix int64       `json:"update_ts_unix" bson:"update_ts_unix"`
 }
 
 func (n *Node) Save() error {
@@ -40,6 +41,7 @@ func (n *Node) Add() error {
 	defer s.Close()
 	n.Id = bson.NewObjectId()
 	n.UpdateTs = time.Now()
+	n.UpdateTsUnix = time.Now().Unix()
 	n.CreateTs = time.Now()
 	if err := c.Insert(&n); err != nil {
 		debug.PrintStack()

backend/services/node.go

@@ -16,10 +16,11 @@ import (
 )
 
 type Data struct {
-	Mac      string    `json:"mac"`
-	Ip       string    `json:"ip"`
-	Master   bool      `json:"master"`
-	UpdateTs time.Time `json:"update_ts"`
+	Mac          string    `json:"mac"`
+	Ip           string    `json:"ip"`
+	Master       bool      `json:"master"`
+	UpdateTs     time.Time `json:"update_ts"`
+	UpdateTsUnix int64     `json:"update_ts_unix"`
 }
 
 type NodeMessage struct {
@@ -193,9 +194,8 @@ func UpdateNodeStatus() {
 		}
 
 		// 如果记录的更新时间超过60秒，该节点被认为离线
-		if time.Now().Sub(data.UpdateTs) > 60*time.Second {
+		if time.Now().Unix()-data.UpdateTsUnix > 60 {
 			// 在Redis中删除该节点
-
 			if err := database.RedisClient.HDel("nodes", data.Mac); err != nil {
 				log.Errorf(err.Error())
 				return
@@ -284,10 +284,11 @@ func UpdateNodeData() {
 
 	// 构造节点数据
 	data := Data{
-		Mac:      mac,
-		Ip:       ip,
-		Master:   IsMaster(),
-		UpdateTs: time.Now(),
+		Mac:          mac,
+		Ip:           ip,
+		Master:       IsMaster(),
+		UpdateTs:     time.Now(),
+		UpdateTsUnix: time.Now().Unix(),
 	}
 
 	// 注册节点到Redis

backend/services/task.go

@@ -124,10 +124,15 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e
 	cmd.Stdout = fLog
 	cmd.Stderr = fLog
 
-	// 添加环境变量
+	// 添加默认环境变量
 	cmd.Env = append(cmd.Env, "CRAWLAB_TASK_ID="+t.Id)
 	cmd.Env = append(cmd.Env, "CRAWLAB_COLLECTION="+s.Col)
 
+	// 添加任务环境变量
+	for _, env := range s.Envs {
+		cmd.Env = append(cmd.Env, env.Name + "=" + env.Value)
+	}
+
 	// 起一个goroutine来监控进程
 	ch := TaskExecChanMap.ChanBlocked(t.Id)
 	go func() {
@@ -393,7 +398,7 @@ func GetTaskLog(id string) (logStr string, err error) {
 	}
 
 	logStr = ""
-	if IsMaster() {
+	if IsMasterNode(task.NodeId.Hex()) {
 		// 若为主节点，获取本机日志
 		logBytes, err := GetLocalLog(task.LogPath)
 		logStr = string(logBytes)