Merge branch 'develop' of https://github.com/crawlab-team/crawlab into v0.4.0

backend/config/config_test.go

@@ -7,7 +7,7 @@ import (
 
 func TestInitConfig(t *testing.T) {
 	Convey("Test InitConfig func", t, func() {
-		x := InitConfig("")
+		x := InitConfig("../conf/config.yml")
 
 		Convey("The value should be nil", func() {
 			So(x, ShouldEqual, nil)

backend/database/pubsub.go

@@ -46,12 +46,12 @@ func (c *Subscriber) Connect() {
 					panic(errors.New("redis connection failed too many times, panic"))
 				}
 				con, err := GetRedisConn()
+				i += 1
 				if err != nil {
 					log.Error("redis dial failed")
 					continue
 				}
 				c.client = redis.PubSubConn{Conn: con}
-				i += 1
 
 				continue
 			}

backend/mock/spider.go

@@ -1 +1,178 @@
-package mock
+package mock
+
+import (
+	"crawlab/model"
+	"github.com/apex/log"
+	"github.com/gin-gonic/gin"
+	"github.com/globalsign/mgo/bson"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+var SpiderList = []model.Spider{
+	{
+		Id:          bson.ObjectId("5d429e6c19f7abede924fee2"),
+		Name:        "For test",
+		DisplayName: "test",
+		Type:        "test",
+		Col:         "test",
+		Site:        "www.baidu.com",
+		Envs:        nil,
+		Src:         "../app/spiders",
+		Cmd:         "scrapy crawl test",
+		LastRunTs:   time.Now(),
+		CreateTs:    time.Now(),
+		UpdateTs:    time.Now(),
+	},
+}
+
+func GetSpiderList(c *gin.Context) {
+
+	// mock get spider list from database
+	results := SpiderList
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    results,
+	})
+}
+
+func GetSpider(c *gin.Context) {
+	id := c.Param("id")
+	var result model.Spider
+
+	if !bson.IsObjectIdHex(id) {
+		HandleErrorF(http.StatusBadRequest, c, "invalid id")
+	}
+
+	for _, spider := range SpiderList {
+		if spider.Id == bson.ObjectId(id) {
+			result = spider
+		}
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    result,
+	})
+}
+
+func PostSpider(c *gin.Context) {
+	id := c.Param("id")
+
+	if !bson.IsObjectIdHex(id) {
+		HandleErrorF(http.StatusBadRequest, c, "invalid id")
+	}
+
+	var item model.Spider
+	if err := c.ShouldBindJSON(&item); err != nil {
+		HandleError(http.StatusBadRequest, c, err)
+		return
+	}
+
+	log.Info("modify the item")
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}
+func GetSpiderDir(c *gin.Context) {
+	// 爬虫ID
+	id := c.Param("id")
+
+	// 目录相对路径
+	path := c.Query("path")
+	var spi model.Spider
+
+	// 获取爬虫
+	for _, spider := range SpiderList {
+		if spider.Id == bson.ObjectId(id) {
+			spi = spider
+		}
+	}
+
+	// 获取目录下文件列表
+	f, err := ioutil.ReadDir(filepath.Join(spi.Src, path))
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 遍历文件列表
+	var fileList []model.File
+	for _, file := range f {
+		fileList = append(fileList, model.File{
+			Name:  file.Name(),
+			IsDir: file.IsDir(),
+			Size:  file.Size(),
+			Path:  filepath.Join(path, file.Name()),
+		})
+	}
+
+	// 返回结果
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    fileList,
+	})
+}
+
+func GetSpiderTasks(c *gin.Context) {
+	id := c.Param("id")
+
+	var spider model.Spider
+	for _, spi := range SpiderList {
+		if spi.Id == bson.ObjectId(id) {
+			spider = spi
+		}
+	}
+
+	var tasks model.Task
+	for _, task := range TaskList {
+		if task.SpiderId == spider.Id {
+			tasks = task
+		}
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    tasks,
+	})
+}
+
+func DeleteSpider(c *gin.Context) {
+	id := c.Param("id")
+
+	if !bson.IsObjectIdHex(id) {
+		HandleErrorF(http.StatusBadRequest, c, "invalid id")
+		return
+	}
+
+	// 获取该爬虫,get this spider
+	var spider model.Spider
+	for _, spi := range SpiderList {
+		if spi.Id == bson.ObjectId(id) {
+			spider = spi
+		}
+	}
+
+	// 删除爬虫文件目录,delete the spider dir
+	if err := os.RemoveAll(spider.Src); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 从数据库中删除该爬虫,delete this spider from database
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}

backend/mock/spider_test.go

@@ -0,0 +1,137 @@
+package mock
+
+import (
+	"crawlab/model"
+	"encoding/json"
+	"github.com/globalsign/mgo/bson"
+	. "github.com/smartystreets/goconvey/convey"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestGetSpiderList(t *testing.T) {
+	var resp Response
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "/spiders", nil)
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	if err != nil {
+		t.Fatal("unmarshal resp faild")
+	}
+	Convey("Test API GetSpiderList", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+}
+
+func TestGetSpider(t *testing.T) {
+	var resp Response
+	var spiderId = "5d429e6c19f7abede924fee2"
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "/spiders/"+spiderId, nil)
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	if err != nil {
+		t.Fatal("unmarshal resp failed")
+	}
+	Convey("Test API GetSpider", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+}
+
+func TestPostSpider(t *testing.T) {
+	var spider = model.Spider{
+		Id:          bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
+		Name:        "For test",
+		DisplayName: "test",
+		Type:        "test",
+		Col:         "test",
+		Site:        "www.baidu.com",
+		Envs:        nil,
+		Src:         "/app/spider",
+		Cmd:         "scrapy crawl test",
+		LastRunTs:   time.Now(),
+		CreateTs:    time.Now(),
+		UpdateTs:    time.Now(),
+	}
+	var resp Response
+	var spiderId = "5d429e6c19f7abede924fee2"
+	w := httptest.NewRecorder()
+	body, _ := json.Marshal(spider)
+	req, _ := http.NewRequest("POST", "/spiders/"+spiderId, strings.NewReader(string(body)))
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	if err != nil {
+		t.Fatal("unmarshal resp failed")
+	}
+	Convey("Test API PostSpider", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+
+}
+
+func TestGetSpiderDir(t *testing.T) {
+	var spiderId = "5d429e6c19f7abede924fee2"
+	var resp Response
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/dir", nil)
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	if err != nil {
+		t.Fatal("unmarshal resp failed")
+	}
+	Convey("Test API GetSpiderDir", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+
+}
+
+func TestGetSpiderTasks(t *testing.T) {
+	var spiderId = "5d429e6c19f7abede924fee2"
+	var resp Response
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/tasks", nil)
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	if err != nil {
+		t.Fatal("unmarshal resp failed")
+	}
+	Convey("Test API GetSpiderTasks", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+}
+
+func TestDeleteSpider(t *testing.T) {
+	var spiderId = "5d429e6c19f7abede924fee2"
+	var resp Response
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("DELETE", "/spiders/"+spiderId, nil)
+	app.ServeHTTP(w, req)
+	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	if err != nil {
+		t.Fatal("unmarshal resp failed")
+	}
+	Convey("Test API DeleteSpider", t, func() {
+		Convey("Test response status", func() {
+			So(resp.Status, ShouldEqual, "ok")
+			So(resp.Message, ShouldEqual, "success")
+		})
+	})
+}

backend/routes/utils.go

@@ -9,7 +9,7 @@ import (
 func HandleError(statusCode int, c *gin.Context, err error) {
 	log.Errorf("handle error:" + err.Error())
 	debug.PrintStack()
-	c.JSON(statusCode, Response{
+	c.AbortWithStatusJSON(statusCode, Response{
 		Status:  "ok",
 		Message: "error",
 		Error:   err.Error(),
@@ -18,7 +18,7 @@ func HandleError(statusCode int, c *gin.Context, err error) {
 
 func HandleErrorF(statusCode int, c *gin.Context, err string) {
 	debug.PrintStack()
-	c.JSON(statusCode, Response{
+	c.AbortWithStatusJSON(statusCode, Response{
 		Status:  "ok",
 		Message: "error",
 		Error:   err,

backend/services/spider.go

@@ -39,6 +39,8 @@ func GetSpidersFromDir() ([]model.Spider, error) {
 
 	// 如果爬虫项目目录不存在，则创建一个
 	if !utils.Exists(srcPath) {
+        mask := syscall.Umask(0)  // 改为 0000 八进制
+		defer syscall.Umask(mask) // 改为原来的 umask
 		if err := os.MkdirAll(srcPath, 0666); err != nil {
 			debug.PrintStack()
 			return []model.Spider{}, err