mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-22 17:31:03 +01:00
Merge pull request #175 from hantmac/master
add unit test for routes/spider.go
This commit is contained in:
Marvin Zhang
@@ -1 +1,178 @@
|
||||
package mock
|
||||
package mock
|
||||
|
||||
import (
|
||||
"crawlab/model"
|
||||
"github.com/apex/log"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/globalsign/mgo/bson"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
var SpiderList = []model.Spider{
|
||||
{
|
||||
Id: bson.ObjectId("5d429e6c19f7abede924fee2"),
|
||||
Name: "For test",
|
||||
DisplayName: "test",
|
||||
Type: "test",
|
||||
Col: "test",
|
||||
Site: "www.baidu.com",
|
||||
Envs: nil,
|
||||
Src: "../app/spiders",
|
||||
Cmd: "scrapy crawl test",
|
||||
LastRunTs: time.Now(),
|
||||
CreateTs: time.Now(),
|
||||
UpdateTs: time.Now(),
|
||||
},
|
||||
}
|
||||
|
||||
func GetSpiderList(c *gin.Context) {
|
||||
|
||||
// mock get spider list from database
|
||||
results := SpiderList
|
||||
|
||||
c.JSON(http.StatusOK, Response{
|
||||
Status: "ok",
|
||||
Message: "success",
|
||||
Data: results,
|
||||
})
|
||||
}
|
||||
|
||||
func GetSpider(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
var result model.Spider
|
||||
|
||||
if !bson.IsObjectIdHex(id) {
|
||||
HandleErrorF(http.StatusBadRequest, c, "invalid id")
|
||||
}
|
||||
|
||||
for _, spider := range SpiderList {
|
||||
if spider.Id == bson.ObjectId(id) {
|
||||
result = spider
|
||||
}
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, Response{
|
||||
Status: "ok",
|
||||
Message: "success",
|
||||
Data: result,
|
||||
})
|
||||
}
|
||||
|
||||
func PostSpider(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
if !bson.IsObjectIdHex(id) {
|
||||
HandleErrorF(http.StatusBadRequest, c, "invalid id")
|
||||
}
|
||||
|
||||
var item model.Spider
|
||||
if err := c.ShouldBindJSON(&item); err != nil {
|
||||
HandleError(http.StatusBadRequest, c, err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("modify the item")
|
||||
|
||||
c.JSON(http.StatusOK, Response{
|
||||
Status: "ok",
|
||||
Message: "success",
|
||||
})
|
||||
}
|
||||
func GetSpiderDir(c *gin.Context) {
|
||||
// 爬虫ID
|
||||
id := c.Param("id")
|
||||
|
||||
// 目录相对路径
|
||||
path := c.Query("path")
|
||||
var spi model.Spider
|
||||
|
||||
// 获取爬虫
|
||||
for _, spider := range SpiderList {
|
||||
if spider.Id == bson.ObjectId(id) {
|
||||
spi = spider
|
||||
}
|
||||
}
|
||||
|
||||
// 获取目录下文件列表
|
||||
f, err := ioutil.ReadDir(filepath.Join(spi.Src, path))
|
||||
if err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// 遍历文件列表
|
||||
var fileList []model.File
|
||||
for _, file := range f {
|
||||
fileList = append(fileList, model.File{
|
||||
Name: file.Name(),
|
||||
IsDir: file.IsDir(),
|
||||
Size: file.Size(),
|
||||
Path: filepath.Join(path, file.Name()),
|
||||
})
|
||||
}
|
||||
|
||||
// 返回结果
|
||||
c.JSON(http.StatusOK, Response{
|
||||
Status: "ok",
|
||||
Message: "success",
|
||||
Data: fileList,
|
||||
})
|
||||
}
|
||||
|
||||
func GetSpiderTasks(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
var spider model.Spider
|
||||
for _, spi := range SpiderList {
|
||||
if spi.Id == bson.ObjectId(id) {
|
||||
spider = spi
|
||||
}
|
||||
}
|
||||
|
||||
var tasks model.Task
|
||||
for _, task := range TaskList {
|
||||
if task.SpiderId == spider.Id {
|
||||
tasks = task
|
||||
}
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, Response{
|
||||
Status: "ok",
|
||||
Message: "success",
|
||||
Data: tasks,
|
||||
})
|
||||
}
|
||||
|
||||
func DeleteSpider(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
if !bson.IsObjectIdHex(id) {
|
||||
HandleErrorF(http.StatusBadRequest, c, "invalid id")
|
||||
return
|
||||
}
|
||||
|
||||
// 获取该爬虫,get this spider
|
||||
var spider model.Spider
|
||||
for _, spi := range SpiderList {
|
||||
if spi.Id == bson.ObjectId(id) {
|
||||
spider = spi
|
||||
}
|
||||
}
|
||||
|
||||
// 删除爬虫文件目录,delete the spider dir
|
||||
if err := os.RemoveAll(spider.Src); err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// 从数据库中删除该爬虫,delete this spider from database
|
||||
|
||||
c.JSON(http.StatusOK, Response{
|
||||
Status: "ok",
|
||||
Message: "success",
|
||||
})
|
||||
}
|
||||
|
||||
137
backend/mock/spider_test.go
Normal file
137
backend/mock/spider_test.go
Normal file
@@ -0,0 +1,137 @@
|
||||
package mock
|
||||
|
||||
import (
|
||||
"crawlab/model"
|
||||
"encoding/json"
|
||||
"github.com/globalsign/mgo/bson"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGetSpiderList(t *testing.T) {
|
||||
var resp Response
|
||||
w := httptest.NewRecorder()
|
||||
req, _ := http.NewRequest("GET", "/spiders", nil)
|
||||
app.ServeHTTP(w, req)
|
||||
err := json.Unmarshal([]byte(w.Body.String()), &resp)
|
||||
if err != nil {
|
||||
t.Fatal("unmarshal resp faild")
|
||||
}
|
||||
Convey("Test API GetSpiderList", t, func() {
|
||||
Convey("Test response status", func() {
|
||||
So(resp.Status, ShouldEqual, "ok")
|
||||
So(resp.Message, ShouldEqual, "success")
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetSpider(t *testing.T) {
|
||||
var resp Response
|
||||
var spiderId = "5d429e6c19f7abede924fee2"
|
||||
w := httptest.NewRecorder()
|
||||
req, _ := http.NewRequest("GET", "/spiders/"+spiderId, nil)
|
||||
app.ServeHTTP(w, req)
|
||||
err := json.Unmarshal([]byte(w.Body.String()), &resp)
|
||||
if err != nil {
|
||||
t.Fatal("unmarshal resp failed")
|
||||
}
|
||||
Convey("Test API GetSpider", t, func() {
|
||||
Convey("Test response status", func() {
|
||||
So(resp.Status, ShouldEqual, "ok")
|
||||
So(resp.Message, ShouldEqual, "success")
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestPostSpider(t *testing.T) {
|
||||
var spider = model.Spider{
|
||||
Id: bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
|
||||
Name: "For test",
|
||||
DisplayName: "test",
|
||||
Type: "test",
|
||||
Col: "test",
|
||||
Site: "www.baidu.com",
|
||||
Envs: nil,
|
||||
Src: "/app/spider",
|
||||
Cmd: "scrapy crawl test",
|
||||
LastRunTs: time.Now(),
|
||||
CreateTs: time.Now(),
|
||||
UpdateTs: time.Now(),
|
||||
}
|
||||
var resp Response
|
||||
var spiderId = "5d429e6c19f7abede924fee2"
|
||||
w := httptest.NewRecorder()
|
||||
body, _ := json.Marshal(spider)
|
||||
req, _ := http.NewRequest("POST", "/spiders/"+spiderId, strings.NewReader(string(body)))
|
||||
app.ServeHTTP(w, req)
|
||||
err := json.Unmarshal([]byte(w.Body.String()), &resp)
|
||||
if err != nil {
|
||||
t.Fatal("unmarshal resp failed")
|
||||
}
|
||||
Convey("Test API PostSpider", t, func() {
|
||||
Convey("Test response status", func() {
|
||||
So(resp.Status, ShouldEqual, "ok")
|
||||
So(resp.Message, ShouldEqual, "success")
|
||||
})
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func TestGetSpiderDir(t *testing.T) {
|
||||
var spiderId = "5d429e6c19f7abede924fee2"
|
||||
var resp Response
|
||||
w := httptest.NewRecorder()
|
||||
req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/dir", nil)
|
||||
app.ServeHTTP(w, req)
|
||||
err := json.Unmarshal([]byte(w.Body.String()), &resp)
|
||||
if err != nil {
|
||||
t.Fatal("unmarshal resp failed")
|
||||
}
|
||||
Convey("Test API GetSpiderDir", t, func() {
|
||||
Convey("Test response status", func() {
|
||||
So(resp.Status, ShouldEqual, "ok")
|
||||
So(resp.Message, ShouldEqual, "success")
|
||||
})
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func TestGetSpiderTasks(t *testing.T) {
|
||||
var spiderId = "5d429e6c19f7abede924fee2"
|
||||
var resp Response
|
||||
w := httptest.NewRecorder()
|
||||
req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/tasks", nil)
|
||||
app.ServeHTTP(w, req)
|
||||
err := json.Unmarshal([]byte(w.Body.String()), &resp)
|
||||
if err != nil {
|
||||
t.Fatal("unmarshal resp failed")
|
||||
}
|
||||
Convey("Test API GetSpiderTasks", t, func() {
|
||||
Convey("Test response status", func() {
|
||||
So(resp.Status, ShouldEqual, "ok")
|
||||
So(resp.Message, ShouldEqual, "success")
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestDeleteSpider(t *testing.T) {
|
||||
var spiderId = "5d429e6c19f7abede924fee2"
|
||||
var resp Response
|
||||
w := httptest.NewRecorder()
|
||||
req, _ := http.NewRequest("DELETE", "/spiders/"+spiderId, nil)
|
||||
app.ServeHTTP(w, req)
|
||||
err := json.Unmarshal([]byte(w.Body.String()), &resp)
|
||||
if err != nil {
|
||||
t.Fatal("unmarshal resp failed")
|
||||
}
|
||||
Convey("Test API DeleteSpider", t, func() {
|
||||
Convey("Test response status", func() {
|
||||
So(resp.Status, ShouldEqual, "ok")
|
||||
So(resp.Message, ShouldEqual, "success")
|
||||
})
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user