diff --git a/backend/mock/spider.go b/backend/mock/spider.go index c4807247..ef3e6104 100644 --- a/backend/mock/spider.go +++ b/backend/mock/spider.go @@ -1 +1,178 @@ -package mock \ No newline at end of file +package mock + +import ( + "crawlab/model" + "github.com/apex/log" + "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "time" +) + +var SpiderList = []model.Spider{ + { + Id: bson.ObjectId("5d429e6c19f7abede924fee2"), + Name: "For test", + DisplayName: "test", + Type: "test", + Col: "test", + Site: "www.baidu.com", + Envs: nil, + Src: "../app/spiders", + Cmd: "scrapy crawl test", + LastRunTs: time.Now(), + CreateTs: time.Now(), + UpdateTs: time.Now(), + }, +} + +func GetSpiderList(c *gin.Context) { + + // mock get spider list from database + results := SpiderList + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: results, + }) +} + +func GetSpider(c *gin.Context) { + id := c.Param("id") + var result model.Spider + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + } + + for _, spider := range SpiderList { + if spider.Id == bson.ObjectId(id) { + result = spider + } + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: result, + }) +} + +func PostSpider(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + } + + var item model.Spider + if err := c.ShouldBindJSON(&item); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + log.Info("modify the item") + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} +func GetSpiderDir(c *gin.Context) { + // 爬虫ID + id := c.Param("id") + + // 目录相对路径 + path := c.Query("path") + var spi model.Spider + + // 获取爬虫 + for _, spider := range SpiderList { + if spider.Id == bson.ObjectId(id) { + spi = spider + } + } + + // 获取目录下文件列表 + f, err := ioutil.ReadDir(filepath.Join(spi.Src, path)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 遍历文件列表 + var fileList []model.File + for _, file := range f { + fileList = append(fileList, model.File{ + Name: file.Name(), + IsDir: file.IsDir(), + Size: file.Size(), + Path: filepath.Join(path, file.Name()), + }) + } + + // 返回结果 + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: fileList, + }) +} + +func GetSpiderTasks(c *gin.Context) { + id := c.Param("id") + + var spider model.Spider + for _, spi := range SpiderList { + if spi.Id == bson.ObjectId(id) { + spider = spi + } + } + + var tasks model.Task + for _, task := range TaskList { + if task.SpiderId == spider.Id { + tasks = task + } + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: tasks, + }) +} + +func DeleteSpider(c *gin.Context) { + id := c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } + + // 获取该爬虫,get this spider + var spider model.Spider + for _, spi := range SpiderList { + if spi.Id == bson.ObjectId(id) { + spider = spi + } + } + + // 删除爬虫文件目录,delete the spider dir + if err := os.RemoveAll(spider.Src); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 从数据库中删除该爬虫,delete this spider from database + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} diff --git a/backend/mock/spider_test.go b/backend/mock/spider_test.go new file mode 100644 index 00000000..87634ff7 --- /dev/null +++ b/backend/mock/spider_test.go @@ -0,0 +1,137 @@ +package mock + +import ( + "crawlab/model" + "encoding/json" + "github.com/globalsign/mgo/bson" + . "github.com/smartystreets/goconvey/convey" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestGetSpiderList(t *testing.T) { + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/spiders", nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("unmarshal resp faild") + } + Convey("Test API GetSpiderList", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} + +func TestGetSpider(t *testing.T) { + var resp Response + var spiderId = "5d429e6c19f7abede924fee2" + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/spiders/"+spiderId, nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("unmarshal resp failed") + } + Convey("Test API GetSpider", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} + +func TestPostSpider(t *testing.T) { + var spider = model.Spider{ + Id: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + Name: "For test", + DisplayName: "test", + Type: "test", + Col: "test", + Site: "www.baidu.com", + Envs: nil, + Src: "/app/spider", + Cmd: "scrapy crawl test", + LastRunTs: time.Now(), + CreateTs: time.Now(), + UpdateTs: time.Now(), + } + var resp Response + var spiderId = "5d429e6c19f7abede924fee2" + w := httptest.NewRecorder() + body, _ := json.Marshal(spider) + req, _ := http.NewRequest("POST", "/spiders/"+spiderId, strings.NewReader(string(body))) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("unmarshal resp failed") + } + Convey("Test API PostSpider", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) + +} + +func TestGetSpiderDir(t *testing.T) { + var spiderId = "5d429e6c19f7abede924fee2" + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/dir", nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("unmarshal resp failed") + } + Convey("Test API GetSpiderDir", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) + +} + +func TestGetSpiderTasks(t *testing.T) { + var spiderId = "5d429e6c19f7abede924fee2" + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/tasks", nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("unmarshal resp failed") + } + Convey("Test API GetSpiderTasks", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} + +func TestDeleteSpider(t *testing.T) { + var spiderId = "5d429e6c19f7abede924fee2" + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("DELETE", "/spiders/"+spiderId, nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("unmarshal resp failed") + } + Convey("Test API DeleteSpider", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} \ No newline at end of file