diff --git a/backend/mock/base.go b/backend/mock/base.go new file mode 100644 index 00000000..d8b11eb9 --- /dev/null +++ b/backend/mock/base.go @@ -0,0 +1,16 @@ +package mock + +type Response struct { + Status string `json:"status"` + Message string `json:"message"` + Data interface{} `json:"data"` + Error string `json:"error"` +} + +type ListResponse struct { + Status string `json:"status"` + Message string `json:"message"` + Total int `json:"total"` + Data interface{} `json:"data"` + Error string `json:"error"` +} diff --git a/backend/mock/file.go b/backend/mock/file.go new file mode 100644 index 00000000..addd771a --- /dev/null +++ b/backend/mock/file.go @@ -0,0 +1,8 @@ +package mock + +type File struct { + Name string `json:"name"` + Path string `json:"path"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size"` +} diff --git a/backend/mock/node.go b/backend/mock/node.go new file mode 100644 index 00000000..878dbcfa --- /dev/null +++ b/backend/mock/node.go @@ -0,0 +1,210 @@ +package mock + +import ( + "crawlab/model" + "crawlab/services" + "github.com/apex/log" + "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" + "net/http" + "time" +) + +var NodeList = []model.Node{ + { + Id: bson.ObjectId("5d429e6c19f7abede924fee2"), + Ip: "10.32.35.15", + Name: "test1", + Status: "online", + Port: "8081", + Mac: "ac:12:df:12:fd", + Description: "For test1", + IsMaster: true, + UpdateTs: time.Now(), + CreateTs: time.Now(), + UpdateTsUnix: time.Now().Unix(), + }, + { + Id: bson.ObjectId("5d429e6c19f7abede924fe22"), + Ip: "10.32.35.12", + Name: "test2", + Status: "online", + Port: "8082", + Mac: "ac:12:df:12:vh", + Description: "For test2", + IsMaster: true, + UpdateTs: time.Now(), + CreateTs: time.Now(), + UpdateTsUnix: time.Now().Unix(), + }, +} + +var TaskList = []model.Task{ + { + Id: "1234", + SpiderId: bson.ObjectId("xx429e6c19f7abede924fee2"), + StartTs: time.Now(), + FinishTs: time.Now(), + Status: "进行中", + NodeId: bson.ObjectId("5d429e6c19f7abede924fee2"), + LogPath: "./log", + Cmd: "scrapy crawl test", + Error: "", + ResultCount: 0, + WaitDuration: 10.0, + RuntimeDuration: 10, + TotalDuration: 20, + SpiderName: "test", + NodeName: "test", + CreateTs: time.Now(), + UpdateTs: time.Now(), + }, + { + Id: "5678", + SpiderId: bson.ObjectId("xx429e6c19f7abede924fddf"), + StartTs: time.Now(), + FinishTs: time.Now(), + Status: "进行中", + NodeId: bson.ObjectId("5d429e6c19f7abede924fee2"), + LogPath: "./log", + Cmd: "scrapy crawl test2", + Error: "", + ResultCount: 0, + WaitDuration: 10.0, + RuntimeDuration: 10, + TotalDuration: 20, + SpiderName: "test", + NodeName: "test", + CreateTs: time.Now(), + UpdateTs: time.Now(), + }, +} + +var dataList = []services.Data{ + { + Mac: "ac:12:fc:fd:ds:dd", + Ip: "192.10.2.1", + Master: true, + UpdateTs: time.Now(), + UpdateTsUnix: time.Now().Unix(), + }, + { + Mac: "22:12:fc:fd:ds:dd", + Ip: "182.10.2.2", + Master: true, + UpdateTs: time.Now(), + UpdateTsUnix: time.Now().Unix(), + }, +} + +var executeble = []model.Executable{ + { + Path: "/test", + FileName: "test.py", + DisplayName: "test.py", + }, +} +var systemInfo = model.SystemInfo{ARCH: "x86", + OS: "linux", + Hostname: "test", + NumCpu: 4, + Executables: executeble, +} + +func GetNodeList(c *gin.Context) { + nodes := NodeList + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: nodes, + }) +} + +func GetNode(c *gin.Context) { + var result model.Node + id := c.Param("id") + for _, node := range NodeList { + if node.Id == bson.ObjectId(id) { + result = node + } + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: result, + }) +} + +func Ping(c *gin.Context) { + data := dataList[0] + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: data, + }) +} + +func PostNode(c *gin.Context) { + id := c.Param("id") + var oldItem model.Node + for _, node := range NodeList { + if node.Id == bson.ObjectId(id) { + oldItem = node + } + + } + log.Info(id) + var newItem model.Node + if err := c.ShouldBindJSON(&newItem); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + newItem.Id = oldItem.Id + + log.Info("Post Node success") + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func GetNodeTaskList(c *gin.Context) { + + tasks := TaskList + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: tasks, + }) +} + +func DeleteNode(c *gin.Context) { + id := bson.ObjectId("5d429e6c19f7abede924fee2") + + for _, node := range NodeList { + if node.Id == bson.ObjectId(id) { + log.Infof("Delete a node") + } + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func GetSystemInfo(c *gin.Context) { + id := c.Param("id") + log.Info(id) + sysInfo := systemInfo + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: sysInfo, + }) +} diff --git a/backend/routes/node_test.go b/backend/mock/node_test.go similarity index 57% rename from backend/routes/node_test.go rename to backend/mock/node_test.go index ee3570c3..9d7096b3 100644 --- a/backend/routes/node_test.go +++ b/backend/mock/node_test.go @@ -1,17 +1,17 @@ -package routes +package mock import ( - "crawlab/config" - "crawlab/database" + "crawlab/model" "encoding/json" - "github.com/apex/log" "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" . "github.com/smartystreets/goconvey/convey" - "github.com/spf13/viper" "net/http" "net/http/httptest" - "runtime/debug" + "strings" "testing" + "time" + "ucloudBilling/ucloud/log" ) var app *gin.Engine @@ -19,34 +19,7 @@ var app *gin.Engine func init() { app = gin.Default() - // 初始化配置 - if err := config.InitConfig("../conf/config.yml"); err != nil { - panic(err) - } - log.Info("初始化配置成功") - - // 初始化日志设置 - logLevel := viper.GetString("log.level") - if logLevel != "" { - log.SetLevelFromString(logLevel) - } - log.Info("初始化日志设置成功") - - // 初始化Mongodb数据库 - if err := database.InitMongo(); err != nil { - debug.PrintStack() - panic(err) - } - log.Info("初始化Mongodb数据库成功") - - // 初始化Redis数据库 - if err := database.InitRedis(); err != nil { - debug.PrintStack() - panic(err) - } - log.Info("初始化Redis数据库成功") - - // 路由 + // mock Test // 节点相关的API app.GET("/ping", Ping) app.GET("/nodes", GetNodeList) // 节点列表 @@ -55,20 +28,27 @@ func init() { app.GET("/nodes/:id/tasks", GetNodeTaskList) // 节点任务列表 app.GET("/nodes/:id/system", GetSystemInfo) // 节点任务列表 app.DELETE("/nodes/:id", DeleteNode) // 删除节点 - // 爬虫 + //// 爬虫 + // 定时任务 + app.GET("/schedules", GetScheduleList) // 定时任务列表 + app.GET("/schedules/:id", GetSchedule) // 定时任务详情 + app.PUT("/schedules", PutSchedule) // 创建定时任务 + app.POST("/schedules/:id", PostSchedule) // 修改定时任务 + app.DELETE("/schedules/:id", DeleteSchedule) // 删除定时任务 } -//先测试GetNodeList得到一个节点的ID，再继续测试下面的API +//mock test, test data in ./mock func TestGetNodeList(t *testing.T) { var resp Response w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/nodes", nil) app.ServeHTTP(w, req) err := json.Unmarshal([]byte(w.Body.String()), &resp) + t.Log(resp.Data) if err != nil { t.Fatal("Unmarshal resp failed") } - t.Log(resp.Data) + Convey("Test API GetNodeList", t, func() { Convey("Test response status", func() { So(resp.Status, ShouldEqual, "ok") @@ -77,10 +57,9 @@ func TestGetNodeList(t *testing.T) { }) } -//依赖MongoDB中的数据,_id=5d429e6c19f7abede924fee2,实际测试时需替换 func TestGetNode(t *testing.T) { var resp Response - var mongoId = "5d5a658319f7ab423585b0b0" + var mongoId = "5d429e6c19f7abede924fee2" w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/nodes/"+mongoId, nil) app.ServeHTTP(w, req) @@ -93,7 +72,7 @@ func TestGetNode(t *testing.T) { Convey("Test response status", func() { So(resp.Status, ShouldEqual, "ok") So(resp.Message, ShouldEqual, "success") - So(resp.Data.(map[string]interface{})["_id"], ShouldEqual, mongoId) + So(resp.Data.(map[string]interface{})["_id"], ShouldEqual, bson.ObjectId(mongoId).Hex()) }) }) } @@ -117,7 +96,7 @@ func TestPing(t *testing.T) { func TestGetNodeTaskList(t *testing.T) { var resp Response - var mongoId = "5d5a658319f7ab423585b0b0" + var mongoId = "5d429e6c19f7abede924fee2" w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/tasks", nil) app.ServeHTTP(w, req) @@ -135,7 +114,8 @@ func TestGetNodeTaskList(t *testing.T) { func TestDeleteNode(t *testing.T) { var resp Response - var mongoId = "5d5a658319f7ab423585b0b0" + + var mongoId = "5d429e6c19f7abede924fee2" w := httptest.NewRecorder() req, _ := http.NewRequest("DELETE", "nodes/"+mongoId, nil) app.ServeHTTP(w, req) @@ -150,3 +130,58 @@ func TestDeleteNode(t *testing.T) { }) }) } + +func TestPostNode(t *testing.T) { + var newItem = model.Node{ + Id: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + Ip: "10.32.35.15", + Name: "test1", + Status: "online", + Port: "8081", + Mac: "ac:12:df:12:fd", + Description: "For test1", + IsMaster: true, + UpdateTs: time.Now(), + CreateTs: time.Now(), + UpdateTsUnix: time.Now().Unix(), + } + + var resp Response + body, _ := json.Marshal(newItem) + log.Info(strings.NewReader(string(body))) + + var mongoId = "5d429e6c19f7abede924fee2" + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", "nodes/"+mongoId, strings.NewReader(string(body))) + + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + t.Log(resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + Convey("Test API PostNode", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} + +func TestGetSystemInfo(t *testing.T) { + var resp Response + var mongoId = "5d429e6c19f7abede924fee2" + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/system", nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + Convey("Test API GetSystemInfo", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} diff --git a/backend/mock/schedule.go b/backend/mock/schedule.go new file mode 100644 index 00000000..ae982ca6 --- /dev/null +++ b/backend/mock/schedule.go @@ -0,0 +1,126 @@ +package mock + +import ( + "crawlab/constants" + "crawlab/model" + "fmt" + "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" + "net/http" + "time" +) + +var scheduleList = []model.Schedule{ + { + Id: bson.ObjectId("5d429e6c19f7abede924fee2"), + Name: "test schedule", + SpiderId: "123", + NodeId: bson.ObjectId("5d429e6c19f7abede924fee2"), + Cron: "***1*", + EntryId: 10, + // 前端展示 + SpiderName: "test scedule", + NodeName: "测试节点", + + CreateTs: time.Now(), + UpdateTs: time.Now(), + }, + { + Id: bson.ObjectId("xx429e6c19f7abede924fee2"), + Name: "test schedule2", + SpiderId: "234", + NodeId: bson.ObjectId("5d429e6c19f7abede924fee2"), + Cron: "***1*", + EntryId: 10, + // 前端展示 + SpiderName: "test scedule2", + NodeName: "测试节点", + + CreateTs: time.Now(), + UpdateTs: time.Now(), + }, +} + +func GetScheduleList(c *gin.Context) { + results := scheduleList + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: results, + }) +} + +func GetSchedule(c *gin.Context) { + id := c.Param("id") + + var result model.Schedule + for _, sch := range scheduleList { + if sch.Id == bson.ObjectId(id) { + result = sch + } + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: result, + }) +} + +func PostSchedule(c *gin.Context) { + id := c.Param("id") + var oldItem model.Schedule + for _, sch := range scheduleList { + if sch.Id == bson.ObjectId(id) { + oldItem = sch + } + + } + + var newItem model.Schedule + if err := c.ShouldBindJSON(&newItem); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + newItem.Id = oldItem.Id + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func PutSchedule(c *gin.Context) { + var item model.Schedule + + // 绑定数据模型 + if err := c.ShouldBindJSON(&item); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + // 如果node_id为空，则置为空ObjectId + if item.NodeId == "" { + item.NodeId = bson.ObjectIdHex(constants.ObjectIdNull) + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func DeleteSchedule(c *gin.Context) { + id := bson.ObjectIdHex("5d429e6c19f7abede924fee2") + for _, sch := range scheduleList { + if sch.Id == bson.ObjectId(id) { + fmt.Println("delete a schedule") + } + } + fmt.Println(id) + fmt.Println("update schedule") + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} diff --git a/backend/mock/schedule_test.go b/backend/mock/schedule_test.go new file mode 100644 index 00000000..d26a08d8 --- /dev/null +++ b/backend/mock/schedule_test.go @@ -0,0 +1,144 @@ +package mock + +import ( + "crawlab/model" + "encoding/json" + "github.com/globalsign/mgo/bson" + . "github.com/smartystreets/goconvey/convey" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + "ucloudBilling/ucloud/log" +) + +func TestGetScheduleList(t *testing.T) { + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/schedules", nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + t.Log(resp.Data) + Convey("Test API GetScheduleList", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} + +func TestGetSchedule(t *testing.T) { + var mongoId = "5d429e6c19f7abede924fee2" + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/schedules/"+mongoId, nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + Convey("Test API GetSchedule", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + So(resp.Data.(map[string]interface{})["_id"], ShouldEqual, bson.ObjectId(mongoId).Hex()) + }) + }) +} + +func TestDeleteSchedule(t *testing.T) { + var mongoId = "5d429e6c19f7abede924fee2" + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("DELETE", "/schedules/"+mongoId, nil) + app.ServeHTTP(w, req) + + err := json.Unmarshal([]byte(w.Body.String()), &resp) + log.Info(w.Body.String()) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + + Convey("Test DeleteSchedule", t, func() { + Convey("Test resp status", func() { + So(resp.Status, ShouldEqual, "ok") + }) + }) +} + +func TestPostSchedule(t *testing.T) { + var newItem = model.Schedule{ + Id: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + Name: "test schedule", + SpiderId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + NodeId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + Cron: "***1*", + EntryId: 10, + // 前端展示 + SpiderName: "test scedule", + NodeName: "测试节点", + + CreateTs: time.Now(), + UpdateTs: time.Now(), + } + + var resp Response + var mongoId = "5d429e6c19f7abede924fee2" + body,_ := json.Marshal(newItem) + log.Info(strings.NewReader(string(body))) + w := httptest.NewRecorder() + req,_ := http.NewRequest("POST", "/schedules/"+mongoId,strings.NewReader(string(body))) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()),&resp) + t.Log(resp) + if err != nil { + t.Fatal("unmarshal resp failed") + } + Convey("Test API PostSchedule", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) + +} + +func TestPutSchedule(t *testing.T) { + var newItem = model.Schedule{ + Id: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + Name: "test schedule", + SpiderId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + NodeId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + Cron: "***1*", + EntryId: 10, + // 前端展示 + SpiderName: "test scedule", + NodeName: "测试节点", + + CreateTs: time.Now(), + UpdateTs: time.Now(), + } + + var resp Response + body,_ := json.Marshal(newItem) + log.Info(strings.NewReader(string(body))) + w := httptest.NewRecorder() + req,_ := http.NewRequest("PUT", "/schedules",strings.NewReader(string(body))) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()),&resp) + t.Log(resp) + if err != nil { + t.Fatal("unmarshal resp failed") + } + Convey("Test API PutSchedule", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) + +} diff --git a/backend/mock/spider.go b/backend/mock/spider.go new file mode 100644 index 00000000..c4807247 --- /dev/null +++ b/backend/mock/spider.go @@ -0,0 +1 @@ +package mock \ No newline at end of file diff --git a/backend/mock/system.go b/backend/mock/system.go new file mode 100644 index 00000000..c4807247 --- /dev/null +++ b/backend/mock/system.go @@ -0,0 +1 @@ +package mock \ No newline at end of file diff --git a/backend/mock/task.go b/backend/mock/task.go new file mode 100644 index 00000000..c4807247 --- /dev/null +++ b/backend/mock/task.go @@ -0,0 +1 @@ +package mock \ No newline at end of file diff --git a/backend/mock/user.go b/backend/mock/user.go new file mode 100644 index 00000000..c4807247 --- /dev/null +++ b/backend/mock/user.go @@ -0,0 +1 @@ +package mock \ No newline at end of file diff --git a/backend/mock/utils.go b/backend/mock/utils.go new file mode 100644 index 00000000..fd7d4efd --- /dev/null +++ b/backend/mock/utils.go @@ -0,0 +1,24 @@ +package mock + +import ( + "github.com/gin-gonic/gin" + "runtime/debug" +) + +func HandleError(statusCode int, c *gin.Context, err error) { + debug.PrintStack() + c.JSON(statusCode, Response{ + Status: "ok", + Message: "error", + Error: err.Error(), + }) +} + +func HandleErrorF(statusCode int, c *gin.Context, err string) { + debug.PrintStack() + c.JSON(statusCode, Response{ + Status: "ok", + Message: "error", + Error: err, + }) +}