Merge pull request #160 from tikazyq/develop

Develop

Jenkinsfile

@@ -46,15 +46,10 @@ pipeline {
             steps {
                 echo 'Deploying....'
                 sh """
-                if [[ ${ENV:GIT_BRANCH} = master ]] || [[ ${ENV:GIT_BRANCH} = develop ]]; then
-                    # 重启docker compose
-                    cd ./jenkins/${ENV:GIT_BRANCH}
-                    docker-compose stop | true
-                    docker-compose up -d
-                else
-                    # 发布到Dockerhub
-                    docker push tikazyq/crawlab:${ENV:TAG}
-                fi
+                # 重启docker compose
+                cd ./jenkins/${ENV:GIT_BRANCH}
+                docker-compose stop | true
+                docker-compose up -d
                 """
             }
         }

README-zh.md

@@ -37,22 +37,7 @@
 
 ### Docker
 
-运行主节点示例。`192.168.99.1`是在Docker Machine网络中的宿主机IP地址。`192.168.99.100`是Docker主节点的IP地址。
-
-```bash
-docker run -d --rm --name crawlab \
-        -e CRAWLAB_REDIS_ADDRESS=192.168.99.1 \
-        -e CRAWLAB_MONGO_HOST=192.168.99.1 \
-        -e CRAWLAB_SERVER_MASTER=Y \
-        -e CRAWLAB_API_ADDRESS=192.168.99.100:8000 \
-        -e CRAWLAB_SPIDER_PATH=/app/spiders \
-        -p 8080:8080 \
-        -p 8000:8000 \
-        -v /var/logs/crawlab:/var/logs/crawlab \
-        tikazyq/crawlab:0.3.0
-```
-
-当然也可以用`docker-compose`来一键启动，甚至不用配置MongoDB和Redis数据库，**当然我们推荐这样做**。在当前目录中创建`docker-compose.yml`文件，输入以下内容。
+请用`docker-compose`来一键启动，甚至不用配置MongoDB和Redis数据库，**当然我们推荐这样做**。在当前目录中创建`docker-compose.yml`文件，输入以下内容。
 
 ```yaml
 version: '3.3'

README.md

@@ -37,22 +37,7 @@ Two methods:
 
 ### Docker
 
-Run Master Node for example. `192.168.99.1` is the host machine IP address in Docker Machine network. `192.168.99.100` is the Master Node's IP address. 
-
-```bash
-docker run -d --rm --name crawlab \
-        -e CRAWLAB_REDIS_ADDRESS=192.168.99.1 \
-        -e CRAWLAB_MONGO_HOST=192.168.99.1 \
-        -e CRAWLAB_SERVER_MASTER=Y \
-        -e CRAWLAB_API_ADDRESS=192.168.99.100:8000 \
-        -e CRAWLAB_SPIDER_PATH=/app/spiders \
-        -p 8080:8080 \
-        -p 8000:8000 \
-        -v /var/logs/crawlab:/var/logs/crawlab \
-        tikazyq/crawlab:0.3.0
-```
-
-Surely you can use `docker-compose` to one-click to start up. By doing so, you don't even have to configure MongoDB and Redis databases. Create a file named `docker-compose.yml` and input the code below.
+Please use `docker-compose` to one-click to start up. By doing so, you don't even have to configure MongoDB and Redis databases. Create a file named `docker-compose.yml` and input the code below.
 
 
 ```yaml

backend/database/mongo_test.go

@@ -1,19 +1,70 @@
 package database
 
 import (
+	"crawlab/config"
+	"github.com/apex/log"
+	"github.com/globalsign/mgo"
 	. "github.com/smartystreets/goconvey/convey"
 	"github.com/spf13/viper"
+	"reflect"
 	"testing"
 )
 
+func init() {
+	if err := config.InitConfig("../conf/config.yml"); err != nil {
+		log.Fatal("Init config failed")
+	}
+	log.Infof("初始化配置成功")
+	err := InitMongo()
+	if err != nil {
+		log.Fatal("Init mongodb failed")
+	}
+
+}
+
 func TestGetDb(t *testing.T) {
 	Convey("Test GetDb", t, func() {
+		if err := config.InitConfig("../conf/config.yml"); err != nil {
+			t.Fatal("Init config failed")
+		}
+		t.Log("初始化配置成功")
+		err := InitMongo()
+		if err != nil {
+			t.Fatal("Init mongodb failed")
+		}
 		s, db := GetDb()
 		Convey("The value should be Session.Copy", func() {
-			So(s, ShouldEqual, Session.Copy())
+			So(s, ShouldResemble, Session.Copy())
 		})
 		Convey("The value should be reference of database", func() {
-			So(db, ShouldEqual, s.DB(viper.GetString("mongo.db")))
+			So(db, ShouldResemble, s.DB(viper.GetString("mongo.db")))
+		})
+	})
+}
+
+func TestGetCol(t *testing.T) {
+	var c = "nodes"
+	var colActual *mgo.Collection
+	Convey("Test GetCol", t, func() {
+		s, col := GetCol(c)
+		Convey("s should resemble Session.Copy", func() {
+			So(s, ShouldResemble, Session.Copy())
+			So(reflect.TypeOf(col), ShouldResemble, reflect.TypeOf(colActual))
+		})
+	})
+}
+
+func TestGetGridFs(t *testing.T) {
+	var prefix = "files"
+	var gfActual *mgo.GridFS
+
+	Convey("Test GetGridFs", t, func() {
+		s, gf := GetGridFs(prefix)
+		Convey("s should be session.copy", func() {
+			So(s, ShouldResemble, Session.Copy())
+		})
+		Convey("gf should be *mgo.GridFS", func() {
+			So(reflect.TypeOf(gf), ShouldResemble, reflect.TypeOf(gfActual))
 		})
 	})
 }

backend/mock/base.go

@@ -0,0 +1,16 @@
+package mock
+
+type Response struct {
+	Status  string      `json:"status"`
+	Message string      `json:"message"`
+	Data    interface{} `json:"data"`
+	Error   string      `json:"error"`
+}
+
+type ListResponse struct {
+	Status  string      `json:"status"`
+	Message string      `json:"message"`
+	Total   int         `json:"total"`
+	Data    interface{} `json:"data"`
+	Error   string      `json:"error"`
+}

backend/mock/file.go

@@ -0,0 +1,8 @@
+package mock
+
+type File struct {
+	Name  string `json:"name"`
+	Path  string `json:"path"`
+	IsDir bool   `json:"is_dir"`
+	Size  int64  `json:"size"`
+}

backend/mock/node.go

@@ -0,0 +1,210 @@
+package mock
+
+import (
+	"crawlab/model"
+	"crawlab/services"
+	"github.com/apex/log"
+	"github.com/gin-gonic/gin"
+	"github.com/globalsign/mgo/bson"
+	"net/http"
+	"time"
+)
+
+var NodeList = []model.Node{
+	{
+		Id:           bson.ObjectId("5d429e6c19f7abede924fee2"),
+		Ip:           "10.32.35.15",
+		Name:         "test1",
+		Status:       "online",
+		Port:         "8081",
+		Mac:          "ac:12:df:12:fd",
+		Description:  "For test1",
+		IsMaster:     true,
+		UpdateTs:     time.Now(),
+		CreateTs:     time.Now(),
+		UpdateTsUnix: time.Now().Unix(),
+	},
+	{
+		Id:           bson.ObjectId("5d429e6c19f7abede924fe22"),
+		Ip:           "10.32.35.12",
+		Name:         "test2",
+		Status:       "online",
+		Port:         "8082",
+		Mac:          "ac:12:df:12:vh",
+		Description:  "For test2",
+		IsMaster:     true,
+		UpdateTs:     time.Now(),
+		CreateTs:     time.Now(),
+		UpdateTsUnix: time.Now().Unix(),
+	},
+}
+
+var TaskList = []model.Task{
+	{
+		Id:              "1234",
+		SpiderId:        bson.ObjectId("xx429e6c19f7abede924fee2"),
+		StartTs:         time.Now(),
+		FinishTs:        time.Now(),
+		Status:          "进行中",
+		NodeId:          bson.ObjectId("5d429e6c19f7abede924fee2"),
+		LogPath:         "./log",
+		Cmd:             "scrapy crawl test",
+		Error:           "",
+		ResultCount:     0,
+		WaitDuration:    10.0,
+		RuntimeDuration: 10,
+		TotalDuration:   20,
+		SpiderName:      "test",
+		NodeName:        "test",
+		CreateTs:        time.Now(),
+		UpdateTs:        time.Now(),
+	},
+	{
+		Id:              "5678",
+		SpiderId:        bson.ObjectId("xx429e6c19f7abede924fddf"),
+		StartTs:         time.Now(),
+		FinishTs:        time.Now(),
+		Status:          "进行中",
+		NodeId:          bson.ObjectId("5d429e6c19f7abede924fee2"),
+		LogPath:         "./log",
+		Cmd:             "scrapy crawl test2",
+		Error:           "",
+		ResultCount:     0,
+		WaitDuration:    10.0,
+		RuntimeDuration: 10,
+		TotalDuration:   20,
+		SpiderName:      "test",
+		NodeName:        "test",
+		CreateTs:        time.Now(),
+		UpdateTs:        time.Now(),
+	},
+}
+
+var dataList = []services.Data{
+	{
+		Mac:          "ac:12:fc:fd:ds:dd",
+		Ip:           "192.10.2.1",
+		Master:       true,
+		UpdateTs:     time.Now(),
+		UpdateTsUnix: time.Now().Unix(),
+	},
+	{
+		Mac:          "22:12:fc:fd:ds:dd",
+		Ip:           "182.10.2.2",
+		Master:       true,
+		UpdateTs:     time.Now(),
+		UpdateTsUnix: time.Now().Unix(),
+	},
+}
+
+var executeble = []model.Executable{
+	{
+		Path:        "/test",
+		FileName:    "test.py",
+		DisplayName: "test.py",
+	},
+}
+var systemInfo = model.SystemInfo{ARCH: "x86",
+	OS:          "linux",
+	Hostname:    "test",
+	NumCpu:      4,
+	Executables: executeble,
+}
+
+func GetNodeList(c *gin.Context) {
+	nodes := NodeList
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    nodes,
+	})
+}
+
+func GetNode(c *gin.Context) {
+	var result model.Node
+	id := c.Param("id")
+	for _, node := range NodeList {
+		if node.Id == bson.ObjectId(id) {
+			result = node
+		}
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    result,
+	})
+}
+
+func Ping(c *gin.Context) {
+	data := dataList[0]
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    data,
+	})
+}
+
+func PostNode(c *gin.Context) {
+	id := c.Param("id")
+	var oldItem model.Node
+	for _, node := range NodeList {
+		if node.Id == bson.ObjectId(id) {
+			oldItem = node
+		}
+
+	}
+	log.Info(id)
+	var newItem model.Node
+	if err := c.ShouldBindJSON(&newItem); err != nil {
+		HandleError(http.StatusBadRequest, c, err)
+		return
+	}
+	newItem.Id = oldItem.Id
+
+	log.Info("Post Node success")
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}
+
+func GetNodeTaskList(c *gin.Context) {
+
+	tasks := TaskList
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    tasks,
+	})
+}
+
+func DeleteNode(c *gin.Context) {
+	id := bson.ObjectId("5d429e6c19f7abede924fee2")
+
+	for _, node := range NodeList {
+		if node.Id == bson.ObjectId(id) {
+			log.Infof("Delete a node")
+		}
+	}
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}
+
+func GetSystemInfo(c *gin.Context) {
+	id := c.Param("id")
+	log.Info(id)
+	sysInfo := systemInfo
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    sysInfo,
+	})
+}

backend/mock/node_test.go

@@ -0,0 +1,187 @@
+package mock
+
+import (
+	"crawlab/model"
+	"encoding/json"
+	"github.com/gin-gonic/gin"
+	"github.com/globalsign/mgo/bson"
+	. "github.com/smartystreets/goconvey/convey"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+	"ucloudBilling/ucloud/log"
+)
+
+var app *gin.Engine
+// 本测试依赖MongoDB的服务，所以在测试之前需要启动MongoDB及相关服务
+func init() {
+	app = gin.Default()
+
+	// mock Test
+	// 节点相关的API
+	app.GET("/ping", Ping)
+	app.GET("/nodes", GetNodeList)               // 节点列表
+	app.GET("/nodes/:id", GetNode)               // 节点详情
+	app.POST("/nodes/:id", PostNode)             // 修改节点
+	app.GET("/nodes/:id/tasks", GetNodeTaskList) // 节点任务列表
+	app.GET("/nodes/:id/system", GetSystemInfo)  // 节点任务列表
+	app.DELETE("/nodes/:id", DeleteNode)         // 删除节点
+	//// 爬虫
+	// 定时任务
+	app.GET("/schedules", GetScheduleList)       // 定时任务列表
+	app.GET("/schedules/:id", GetSchedule)       // 定时任务详情
+	app.PUT("/schedules", PutSchedule)           // 创建定时任务
+	app.POST("/schedules/:id", PostSchedule)     // 修改定时任务
+	app.DELETE("/schedules/:id", DeleteSchedule) // 删除定时任务
+}
+
+//mock test, test data in ./mock
+func TestGetNodeList(t *testing.T) {
+	var resp Response
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "/nodes", nil)
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	t.Log(resp.Data)
+	if err != nil {
+		t.Fatal("Unmarshal resp failed")
+	}
+
+	Convey("Test API GetNodeList", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+}
+
+func TestGetNode(t *testing.T) {
+	var resp Response
+	var mongoId = "5d429e6c19f7abede924fee2"
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "/nodes/"+mongoId, nil)
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	if err != nil {
+		t.Fatal("Unmarshal resp failed")
+	}
+	t.Log(resp.Data)
+	Convey("Test API GetNode", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+			So(resp.Data.(map[string]interface{})["_id"], ShouldEqual, bson.ObjectId(mongoId).Hex())
+		})
+	})
+}
+
+func TestPing(t *testing.T) {
+	var resp Response
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "/ping", nil)
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	if err != nil {
+		t.Fatal("Unmarshal resp failed")
+	}
+	Convey("Test API ping", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+}
+
+func TestGetNodeTaskList(t *testing.T) {
+	var resp Response
+	var mongoId = "5d429e6c19f7abede924fee2"
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/tasks", nil)
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	if err != nil {
+		t.Fatal("Unmarshal resp failed")
+	}
+	Convey("Test API GetNodeTaskList", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+}
+
+func TestDeleteNode(t *testing.T) {
+	var resp Response
+
+	var mongoId = "5d429e6c19f7abede924fee2"
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("DELETE", "nodes/"+mongoId, nil)
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	if err != nil {
+		t.Fatal("Unmarshal resp failed")
+	}
+	Convey("Test API DeleteNode", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+}
+
+func TestPostNode(t *testing.T) {
+	var newItem = model.Node{
+		Id:           bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
+		Ip:           "10.32.35.15",
+		Name:         "test1",
+		Status:       "online",
+		Port:         "8081",
+		Mac:          "ac:12:df:12:fd",
+		Description:  "For test1",
+		IsMaster:     true,
+		UpdateTs:     time.Now(),
+		CreateTs:     time.Now(),
+		UpdateTsUnix: time.Now().Unix(),
+	}
+
+	var resp Response
+	body, _ := json.Marshal(newItem)
+	log.Info(strings.NewReader(string(body)))
+
+	var mongoId = "5d429e6c19f7abede924fee2"
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("POST", "nodes/"+mongoId, strings.NewReader(string(body)))
+
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	t.Log(resp)
+	if err != nil {
+		t.Fatal("Unmarshal resp failed")
+	}
+	Convey("Test API PostNode", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+}
+
+func TestGetSystemInfo(t *testing.T) {
+	var resp Response
+	var mongoId = "5d429e6c19f7abede924fee2"
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/system", nil)
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	if err != nil {
+		t.Fatal("Unmarshal resp failed")
+	}
+	Convey("Test API GetSystemInfo", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+}

backend/mock/schedule.go

@@ -0,0 +1,126 @@
+package mock
+
+import (
+	"crawlab/constants"
+	"crawlab/model"
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/globalsign/mgo/bson"
+	"net/http"
+	"time"
+)
+
+var scheduleList = []model.Schedule{
+	{
+		Id:       bson.ObjectId("5d429e6c19f7abede924fee2"),
+		Name:     "test schedule",
+		SpiderId: "123",
+		NodeId:   bson.ObjectId("5d429e6c19f7abede924fee2"),
+		Cron:     "***1*",
+		EntryId:  10,
+		// 前端展示
+		SpiderName: "test scedule",
+		NodeName:   "测试节点",
+
+		CreateTs: time.Now(),
+		UpdateTs: time.Now(),
+	},
+	{
+		Id:       bson.ObjectId("xx429e6c19f7abede924fee2"),
+		Name:     "test schedule2",
+		SpiderId: "234",
+		NodeId:   bson.ObjectId("5d429e6c19f7abede924fee2"),
+		Cron:     "***1*",
+		EntryId:  10,
+		// 前端展示
+		SpiderName: "test scedule2",
+		NodeName:   "测试节点",
+
+		CreateTs: time.Now(),
+		UpdateTs: time.Now(),
+	},
+}
+
+func GetScheduleList(c *gin.Context) {
+	results := scheduleList
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    results,
+	})
+}
+
+func GetSchedule(c *gin.Context) {
+	id := c.Param("id")
+
+	var result model.Schedule
+	for _, sch := range scheduleList {
+		if sch.Id == bson.ObjectId(id) {
+			result = sch
+		}
+	}
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    result,
+	})
+}
+
+func PostSchedule(c *gin.Context) {
+	id := c.Param("id")
+	var oldItem model.Schedule
+	for _, sch := range scheduleList {
+		if sch.Id == bson.ObjectId(id) {
+			oldItem = sch
+		}
+
+	}
+
+	var newItem model.Schedule
+	if err := c.ShouldBindJSON(&newItem); err != nil {
+		HandleError(http.StatusBadRequest, c, err)
+		return
+	}
+	newItem.Id = oldItem.Id
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}
+
+func PutSchedule(c *gin.Context) {
+	var item model.Schedule
+
+	// 绑定数据模型
+	if err := c.ShouldBindJSON(&item); err != nil {
+		HandleError(http.StatusBadRequest, c, err)
+		return
+	}
+
+	// 如果node_id为空，则置为空ObjectId
+	if item.NodeId == "" {
+		item.NodeId = bson.ObjectIdHex(constants.ObjectIdNull)
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}
+
+func DeleteSchedule(c *gin.Context) {
+	id := bson.ObjectIdHex("5d429e6c19f7abede924fee2")
+	for _, sch := range scheduleList {
+		if sch.Id == bson.ObjectId(id) {
+			fmt.Println("delete a schedule")
+		}
+	}
+	fmt.Println(id)
+	fmt.Println("update schedule")
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}

backend/mock/schedule_test.go

@@ -0,0 +1,144 @@
+package mock
+
+import (
+	"crawlab/model"
+	"encoding/json"
+	"github.com/globalsign/mgo/bson"
+	. "github.com/smartystreets/goconvey/convey"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+	"ucloudBilling/ucloud/log"
+)
+
+func TestGetScheduleList(t *testing.T) {
+	var resp Response
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "/schedules", nil)
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	if err != nil {
+		t.Fatal("Unmarshal resp failed")
+	}
+	t.Log(resp.Data)
+	Convey("Test API GetScheduleList", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+}
+
+func TestGetSchedule(t *testing.T) {
+	var mongoId = "5d429e6c19f7abede924fee2"
+	var resp Response
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "/schedules/"+mongoId, nil)
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	if err != nil {
+		t.Fatal("Unmarshal resp failed")
+	}
+	Convey("Test API GetSchedule", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+			So(resp.Data.(map[string]interface{})["_id"], ShouldEqual, bson.ObjectId(mongoId).Hex())
+		})
+	})
+}
+
+func TestDeleteSchedule(t *testing.T) {
+	var mongoId = "5d429e6c19f7abede924fee2"
+	var resp Response
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("DELETE", "/schedules/"+mongoId, nil)
+	app.ServeHTTP(w, req)
+
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	log.Info(w.Body.String())
+	if err != nil {
+		t.Fatal("Unmarshal resp failed")
+	}
+
+	Convey("Test DeleteSchedule", t, func() {
+		Convey("Test resp status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+		})
+	})
+}
+
+func TestPostSchedule(t *testing.T) {
+	var newItem = model.Schedule{
+		Id:       bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
+		Name:     "test schedule",
+		SpiderId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
+		NodeId:   bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
+		Cron:     "***1*",
+		EntryId:  10,
+		// 前端展示
+		SpiderName: "test scedule",
+		NodeName:   "测试节点",
+
+		CreateTs: time.Now(),
+		UpdateTs: time.Now(),
+	}
+
+	var resp Response
+	var mongoId = "5d429e6c19f7abede924fee2"
+	body,_ := json.Marshal(newItem)
+	log.Info(strings.NewReader(string(body)))
+	w := httptest.NewRecorder()
+	req,_ := http.NewRequest("POST", "/schedules/"+mongoId,strings.NewReader(string(body)))
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()),&resp)
+	t.Log(resp)
+	if err != nil {
+		t.Fatal("unmarshal resp failed")
+	}
+	Convey("Test API PostSchedule", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+
+}
+
+func TestPutSchedule(t *testing.T) {
+	var newItem = model.Schedule{
+		Id:       bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
+		Name:     "test schedule",
+		SpiderId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
+		NodeId:   bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
+		Cron:     "***1*",
+		EntryId:  10,
+		// 前端展示
+		SpiderName: "test scedule",
+		NodeName:   "测试节点",
+
+		CreateTs: time.Now(),
+		UpdateTs: time.Now(),
+	}
+
+	var resp Response
+	body,_ := json.Marshal(newItem)
+	log.Info(strings.NewReader(string(body)))
+	w := httptest.NewRecorder()
+	req,_ := http.NewRequest("PUT", "/schedules",strings.NewReader(string(body)))
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()),&resp)
+	t.Log(resp)
+	if err != nil {
+		t.Fatal("unmarshal resp failed")
+	}
+	Convey("Test API PutSchedule", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+
+}

backend/mock/spider.go

@@ -0,0 +1 @@
+package mock

backend/mock/system.go

@@ -0,0 +1 @@
+package mock

backend/mock/task.go

@@ -0,0 +1 @@
+package mock

backend/mock/user.go

@@ -0,0 +1 @@
+package mock

backend/mock/utils.go

@@ -0,0 +1,24 @@
+package mock
+
+import (
+	"github.com/gin-gonic/gin"
+	"runtime/debug"
+)
+
+func HandleError(statusCode int, c *gin.Context, err error) {
+	debug.PrintStack()
+	c.JSON(statusCode, Response{
+		Status:  "ok",
+		Message: "error",
+		Error:   err.Error(),
+	})
+}
+
+func HandleErrorF(statusCode int, c *gin.Context, err string) {
+	debug.PrintStack()
+	c.JSON(statusCode, Response{
+		Status:  "ok",
+		Message: "error",
+		Error:   err,
+	})
+}

backend/model/node.go

@@ -17,13 +17,15 @@ type Node struct {
 	Port        string        `json:"port" bson:"port"`
 	Mac         string        `json:"mac" bson:"mac"`
 	Description string        `json:"description" bson:"description"`
+	// 用于唯一标识节点，可能是mac地址，可能是ip地址
+	Key string `json:"key" bson:"key"`
 
 	// 前端展示
 	IsMaster bool `json:"is_master"`
 
 	UpdateTs     time.Time `json:"update_ts" bson:"update_ts"`
 	CreateTs     time.Time `json:"create_ts" bson:"create_ts"`
-	UpdateTsUnix int64       `json:"update_ts_unix" bson:"update_ts_unix"`
+	UpdateTsUnix int64     `json:"update_ts_unix" bson:"update_ts_unix"`
 }
 
 func (n *Node) Save() error {
@@ -98,12 +100,12 @@ func GetNode(id bson.ObjectId) (Node, error) {
 	return node, nil
 }
 
-func GetNodeByMac(mac string) (Node, error) {
+func GetNodeByKey(key string) (Node, error) {
 	s, c := database.GetCol("nodes")
 	defer s.Close()
 
 	var node Node
-	if err := c.Find(bson.M{"mac": mac}).One(&node); err != nil {
+	if err := c.Find(bson.M{"key": key}).One(&node); err != nil {
 		if err != mgo.ErrNotFound {
 			log.Errorf(err.Error())
 			debug.PrintStack()

backend/services/node.go

@@ -16,6 +16,7 @@ import (
 )
 
 type Data struct {
+	Key          string    `json:"key"`
 	Mac          string    `json:"mac"`
 	Ip           string    `json:"ip"`
 	Master       bool      `json:"master"`
@@ -51,10 +52,9 @@ const (
 
 // 获取本机节点
 func GetCurrentNode() (model.Node, error) {
-	// 获取本机MAC地址
-	value, err := register.GetRegister().GetValue()
+	// 获得注册的key值
+	key, err := register.GetRegister().GetKey()
 	if err != nil {
-		debug.PrintStack()
 		return model.Node{}, err
 	}
 
@@ -68,8 +68,7 @@ func GetCurrentNode() (model.Node, error) {
 		}
 
 		// 尝试获取节点
-		node, err = model.GetNodeByMac(value)
-
+		node, err = model.GetNodeByKey(key)
 		// 如果获取失败
 		if err != nil {
 			// 如果为主节点，表示为第一次注册，插入节点信息
@@ -80,12 +79,26 @@ func GetCurrentNode() (model.Node, error) {
 					debug.PrintStack()
 					return model.Node{}, err
 				}
+
+				mac, err := register.GetRegister().GetMac()
+				if err != nil {
+					debug.PrintStack()
+					return model.Node{}, err
+				}
+
+				key, err := register.GetRegister().GetKey()
+				if err != nil {
+					debug.PrintStack()
+					return model.Node{}, err
+				}
+
 				// 生成节点
 				node = model.Node{
+					Key:      key,
 					Id:       bson.NewObjectId(),
 					Ip:       ip,
-					Name:     value,
-					Mac:      value,
+					Name:     key,
+					Mac:      mac,
 					IsMaster: true,
 				}
 				if err := node.Add(); err != nil {
@@ -100,11 +113,9 @@ func GetCurrentNode() (model.Node, error) {
 			time.Sleep(5 * time.Second)
 			continue
 		}
-
 		// 跳出循环
 		break
 	}
-
 	return node, nil
 }
 
@@ -122,12 +133,12 @@ func IsMasterNode(id string) bool {
 
 // 获取节点数据
 func GetNodeData() (Data, error) {
-	val, err := register.GetRegister().GetValue()
-	if err != nil {
+	key, err := register.GetRegister().GetKey()
+	if key == "" {
 		return Data{}, err
 	}
 
-	value, err := database.RedisClient.HGet("nodes", val)
+	value, err := database.RedisClient.HGet("nodes", key)
 	data := Data{}
 	if err := json.Unmarshal([]byte(value), &data); err != nil {
 		return data, err
@@ -145,9 +156,9 @@ func UpdateNodeStatus() {
 	}
 
 	// 遍历节点keys
-	for _, mac := range list {
+	for _, key := range list {
 		// 获取节点数据
-		value, err := database.RedisClient.HGet("nodes", mac)
+		value, err := database.RedisClient.HGet("nodes", key)
 		if err != nil {
 			log.Errorf(err.Error())
 			return
@@ -163,7 +174,7 @@ func UpdateNodeStatus() {
 		// 如果记录的更新时间超过60秒，该节点被认为离线
 		if time.Now().Unix()-data.UpdateTsUnix > 60 {
 			// 在Redis中删除该节点
-			if err := database.RedisClient.HDel("nodes", data.Mac); err != nil {
+			if err := database.RedisClient.HDel("nodes", data.Key); err != nil {
 				log.Errorf(err.Error())
 				return
 			}
@@ -172,7 +183,7 @@ func UpdateNodeStatus() {
 			s, c := database.GetCol("nodes")
 			defer s.Close()
 			var node model.Node
-			if err := c.Find(bson.M{"mac": mac}).One(&node); err != nil {
+			if err := c.Find(bson.M{"key": key}).One(&node); err != nil {
 				log.Errorf(err.Error())
 				debug.PrintStack()
 				return
@@ -189,14 +200,16 @@ func UpdateNodeStatus() {
 		s, c := database.GetCol("nodes")
 		defer s.Close()
 		var node model.Node
-		if err := c.Find(bson.M{"mac": mac}).One(&node); err != nil {
+		if err := c.Find(bson.M{"key": key}).One(&node); err != nil {
 			// 数据库不存在该节点
 			node = model.Node{
-				Name:   data.Mac,
-				Ip:     data.Ip,
-				Port:   "8000",
-				Mac:    data.Mac,
-				Status: constants.StatusOnline,
+				Key:      key,
+				Name:     key,
+				Ip:       data.Ip,
+				Port:     "8000",
+				Mac:      data.Mac,
+				Status:   constants.StatusOnline,
+				IsMaster: data.Master,
 			}
 			if err := node.Add(); err != nil {
 				log.Errorf(err.Error())
@@ -216,8 +229,8 @@ func UpdateNodeStatus() {
 	nodes, err := model.GetNodeList(nil)
 	for _, node := range nodes {
 		hasNode := false
-		for _, mac := range list {
-			if mac == node.Mac {
+		for _, key := range list {
+			if key == node.Key {
 				hasNode = true
 				break
 			}
@@ -236,7 +249,7 @@ func UpdateNodeStatus() {
 // 更新节点数据
 func UpdateNodeData() {
 	// 获取MAC地址
-	val, err := register.GetRegister().GetValue()
+	mac, err := register.GetRegister().GetMac()
 	if err != nil {
 		log.Errorf(err.Error())
 		return
@@ -248,10 +261,13 @@ func UpdateNodeData() {
 		log.Errorf(err.Error())
 		return
 	}
+	// 获取redis的key
+	key, err := register.GetRegister().GetKey()
 
 	// 构造节点数据
 	data := Data{
-		Mac:          val,
+		Key:          key,
+		Mac:          mac,
 		Ip:           ip,
 		Master:       IsMaster(),
 		UpdateTs:     time.Now(),
@@ -265,7 +281,7 @@ func UpdateNodeData() {
 		debug.PrintStack()
 		return
 	}
-	if err := database.RedisClient.HSet("nodes", val, string(dataBytes)); err != nil {
+	if err := database.RedisClient.HSet("nodes", key, string(dataBytes)); err != nil {
 		log.Errorf(err.Error())
 		return
 	}

backend/services/register/register.go

@@ -10,50 +10,44 @@ import (
 
 type Register interface {
 	// 注册的key类型
-	GetKey() string
+	GetType() string
 	// 注册的key的值，唯一标识节点
-	GetValue() (string, error)
+	GetKey() (string, error)
 	// 注册的节点IP
 	GetIp() (string, error)
+	// 注册节点的mac地址
+	GetMac() (string, error)
 }
 
-// mac 地址注册
+// ===================== mac 地址注册 =====================
 type MacRegister struct{}
 
-func (mac *MacRegister) GetKey() string {
+func (mac *MacRegister) GetType() string {
 	return "mac"
 }
 
-func (mac *MacRegister) GetValue() (string, error) {
-	interfaces, err := net.Interfaces()
-	if err != nil {
-		log.Errorf("get interfaces error:" + err.Error())
-		debug.PrintStack()
-		return "", err
-	}
-	for _, inter := range interfaces {
-		if inter.HardwareAddr != nil {
-			mac := inter.HardwareAddr.String()
-			return mac, nil
-		}
-	}
-	return "", nil
+func (mac *MacRegister) GetKey() (string, error) {
+	return mac.GetMac()
+}
+
+func (mac *MacRegister) GetMac() (string, error) {
+	return getMac()
 }
 
 func (mac *MacRegister) GetIp() (string, error) {
 	return getIp()
 }
 
-// ip 注册
+// ===================== ip 地址注册 =====================
 type IpRegister struct {
 	Ip string
 }
 
-func (ip *IpRegister) GetKey() string {
+func (ip *IpRegister) GetType() string {
 	return "ip"
 }
 
-func (ip *IpRegister) GetValue() (string, error) {
+func (ip *IpRegister) GetKey() (string, error) {
 	return ip.Ip, nil
 }
 
@@ -61,6 +55,11 @@ func (ip *IpRegister) GetIp() (string, error) {
 	return ip.Ip, nil
 }
 
+func (ip *IpRegister) GetMac() (string, error) {
+	return getMac()
+}
+
+// ===================== 公共方法 =====================
 // 获取本机的IP地址
 // TODO: 考虑多个IP地址的情况
 func getIp() (string, error) {
@@ -78,6 +77,23 @@ func getIp() (string, error) {
 	return "", nil
 }
 
+func getMac() (string, error) {
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		log.Errorf("get interfaces error:" + err.Error())
+		debug.PrintStack()
+		return "", err
+	}
+	for _, inter := range interfaces {
+		if inter.HardwareAddr != nil {
+			mac := inter.HardwareAddr.String()
+			return mac, nil
+		}
+	}
+	return "", nil
+}
+
+// ===================== 获得注册简单工厂 =====================
 var register Register
 
 // 获得注册器

backend/services/spider.go

@@ -7,10 +7,12 @@ import (
 	"crawlab/model"
 	"crawlab/utils"
 	"encoding/json"
+	"fmt"
 	"github.com/apex/log"
+	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
 	"github.com/pkg/errors"
-	uuid "github.com/satori/go.uuid"
+	"github.com/satori/go.uuid"
 	"github.com/spf13/viper"
 	"io"
 	"io/ioutil"
@@ -162,24 +164,6 @@ func ZipSpider(spider model.Spider) (filePath string, err error) {
 func UploadToGridFs(spider model.Spider, fileName string, filePath string) (fid bson.ObjectId, err error) {
 	fid = ""
 
-	// 读取zip文件
-	file, err := os.OpenFile(filePath, os.O_RDONLY, 0777)
-	fileBytes, err := ioutil.ReadAll(file)
-	if err != nil {
-		debug.PrintStack()
-		return
-	}
-	if err = file.Close(); err != nil {
-		debug.PrintStack()
-		return
-	}
-
-	// 删除zip文件
-	if err = os.Remove(filePath); err != nil {
-		debug.PrintStack()
-		return
-	}
-
 	// 获取MongoDB GridFS连接
 	s, gf := database.GetGridFs("files")
 	defer s.Close()
@@ -198,23 +182,58 @@ func UploadToGridFs(spider model.Spider, fileName string, filePath string) (fid
 		return
 	}
 
-	// 将文件写入到GridFS
-	if _, err = f.Write(fileBytes); err != nil {
+	//分片读取爬虫zip文件
+	err = ReadFileByStep(filePath, WriteToGridFS, f)
+	if err != nil {
+		debug.PrintStack()
+		return "", err
+	}
+
+	// 删除zip文件
+	if err = os.Remove(filePath); err != nil {
 		debug.PrintStack()
 		return
 	}
-
 	// 关闭文件，提交写入
 	if err = f.Close(); err != nil {
 		return "", err
 	}
-
 	// 文件ID
 	fid = f.Id().(bson.ObjectId)
 
 	return fid, nil
 }
 
+func WriteToGridFS(content []byte, f *mgo.GridFile) {
+	if _, err := f.Write(content); err != nil {
+		debug.PrintStack()
+		return
+	}
+}
+
+//分片读取大文件
+func ReadFileByStep(filePath string, handle func([]byte, *mgo.GridFile), fileCreate *mgo.GridFile) error {
+	f, err := os.OpenFile(filePath, os.O_RDONLY, 0777)
+	if err != nil {
+		log.Infof("can't opened this file")
+		return err
+	}
+	defer f.Close()
+	s := make([]byte, 4096)
+	for {
+		switch nr, err := f.Read(s[:]); true {
+		case nr < 0:
+			fmt.Fprintf(os.Stderr, "cat: error reading: %s\n", err.Error())
+			debug.PrintStack()
+		case nr == 0: // EOF
+			return nil
+		case nr > 0:
+			handle(s[0:nr], fileCreate)
+		}
+	}
+	return nil
+}
+
 // 发布所有爬虫
 func PublishAllSpiders() error {
 	// 获取爬虫列表

frontend/src/views/layout/components/TagsView.vue

@@ -110,20 +110,22 @@ export default {
     },
     moveToCurrentTag () {
       const tags = this.$refs.tag
-      this.$nextTick(() => {
-        for (const tag of tags) {
-          if (tag.to.path === this.$route.path) {
-            this.$refs.scrollPane.moveToTarget(tag)
+      if (tags) {
+        this.$nextTick(() => {
+          for (const tag of tags) {
+            if (tag.to.path === this.$route.path) {
+              this.$refs.scrollPane.moveToTarget(tag)
 
-            // when query is different then update
-            if (tag.to.fullPath !== this.$route.fullPath) {
-              this.$store.dispatch('updateVisitedView', this.$route)
+              // when query is different then update
+              if (tag.to.fullPath !== this.$route.fullPath) {
+                this.$store.dispatch('updateVisitedView', this.$route)
+              }
+
+              break
             }
-
-            break
           }
-        }
-      })
+        })
+      }
     },
     refreshSelectedTag (view) {
       this.$store.dispatch('delCachedView', view).then(() => {