From 099eb863400c4a3dc2823fd3dbd8db1ca75da26f Mon Sep 17 00:00:00 2001 From: hantmac Date: Sun, 18 Aug 2019 14:20:37 +0800 Subject: [PATCH 01/11] Add Unit Test for backend/utils/chan.go --- backend/utils/chan_test.go | 76 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 backend/utils/chan_test.go diff --git a/backend/utils/chan_test.go b/backend/utils/chan_test.go new file mode 100644 index 00000000..7b5f1bba --- /dev/null +++ b/backend/utils/chan_test.go @@ -0,0 +1,76 @@ +package utils + +import ( + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestNewChanMap(t *testing.T) { + mapTest := make(map[string]chan string) + chanTest := make(chan string) + test := "test" + + Convey("Call NewChanMap to generate ChanMap", t, func() { + mapTest[test] = chanTest + chanMapTest := ChanMap{mapTest} + chanMap := NewChanMap() + chanMap.m[test] = chanTest + + Convey(test, func() { + So(chanMap, ShouldResemble, &chanMapTest) + }) + + }) +} + +func TestChan(t *testing.T) { + mapTest := make(map[string]chan string) + chanTest := make(chan string) + mapTest["test"] = chanTest + chanMapTest := ChanMap{mapTest} + + Convey("Test Chan use exist key", t, func() { + ch1 := chanMapTest.Chan( + "test") + Convey("ch1 should equal chanTest", func() { + So(ch1, ShouldEqual, chanTest) + }) + + }) + Convey("Test Chan use no-exist key", t, func() { + ch2 := chanMapTest.Chan("test2") + Convey("ch2 should equal chanMapTest.m[test2]", func() { + + So(chanMapTest.m["test2"], ShouldEqual, ch2) + }) + Convey("Cap of chanMapTest.m[test2] should equal 10", func() { + So(10, ShouldEqual, cap(chanMapTest.m["test2"])) + }) + }) +} + +func TestChanBlocked(t *testing.T) { + mapTest := make(map[string]chan string) + chanTest := make(chan string) + mapTest["test"] = chanTest + chanMapTest := ChanMap{mapTest} + + Convey("Test Chan use exist key", t, func() { + ch1 := chanMapTest.ChanBlocked( + "test") + Convey("ch1 should equal chanTest", func() { + So(ch1, ShouldEqual, chanTest) + }) + + }) + Convey("Test Chan use no-exist key", t, func() { + ch2 := chanMapTest.ChanBlocked("test2") + Convey("ch2 should equal chanMapTest.m[test2]", func() { + + So(chanMapTest.m["test2"], ShouldEqual, ch2) + }) + Convey("Cap of chanMapTest.m[test2] should equal 10", func() { + So(0, ShouldEqual, cap(chanMapTest.m["test2"])) + }) + }) +} From 1b70d45303d7dd7f32d84f1f43b9e5a64fefbcde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sun, 18 Aug 2019 18:45:19 +0800 Subject: [PATCH 02/11] =?UTF-8?q?fix=20tmp=20=E7=9B=AE=E5=BD=95=E4=B8=8D?= =?UTF-8?q?=E4=BC=9A=E8=87=AA=E5=8A=A8=E5=88=9B=E5=BB=BA=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/go.sum | 5 +++++ backend/routes/spider.go | 14 +++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/backend/go.sum b/backend/go.sum index 82fc37b5..910e18be 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -53,6 +53,7 @@ github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -65,6 +66,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -117,8 +119,10 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -184,6 +188,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= diff --git a/backend/routes/spider.go b/backend/routes/spider.go index e83d7ed8..dceb2651 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -131,10 +131,22 @@ func PutSpider(c *gin.Context) { return } + // 以防tmp目录不存在 + tmpPath := viper.GetString("other.tmppath") + if !utils.Exists(tmpPath) { + if err := os.Mkdir(tmpPath, os.ModePerm); err != nil { + log.Error("mkdir other.tmppath dir error:" + err.Error()) + debug.PrintStack() + HandleError(http.StatusBadRequest, c, errors.New("Mkdir other.tmppath dir error")) + return + } + } + // 保存到本地临时文件 randomId := uuid.NewV4() - tmpFilePath := filepath.Join(viper.GetString("other.tmppath"), randomId.String()+".zip") + tmpFilePath := filepath.Join(tmpPath, randomId.String()+".zip") if err := c.SaveUploadedFile(file, tmpFilePath); err != nil { + log.Error("save upload file error: " + err.Error()) debug.PrintStack() HandleError(http.StatusInternalServerError, c, err) return From 6601ba46d8c580cf61f6277d7ae9bc9adde6294f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sun, 18 Aug 2019 18:48:38 +0800 Subject: [PATCH 03/11] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=BF=BD=E7=95=A5backe?= =?UTF-8?q?nd/spiders=E7=9B=AE=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 13c696a5..2c38a7c2 100644 --- a/.gitignore +++ b/.gitignore @@ -119,4 +119,6 @@ logs/ tmp/ _book/ .idea -*.lock \ No newline at end of file +*.lock + +backend/spiders \ No newline at end of file From 419c667e831880b0334dfd911458f8c808d2c498 Mon Sep 17 00:00:00 2001 From: hantmac Date: Sun, 18 Aug 2019 19:07:59 +0800 Subject: [PATCH 04/11] Complete Unit Test for package utils --- backend/utils/file_test.go | 72 +++++++++++++++++++++++++++++++++++++ backend/utils/model_test.go | 49 +++++++++++++++++++++++++ backend/utils/user_test.go | 14 ++++++++ 3 files changed, 135 insertions(+) create mode 100644 backend/utils/file_test.go create mode 100644 backend/utils/model_test.go create mode 100644 backend/utils/user_test.go diff --git a/backend/utils/file_test.go b/backend/utils/file_test.go new file mode 100644 index 00000000..484366f5 --- /dev/null +++ b/backend/utils/file_test.go @@ -0,0 +1,72 @@ +package utils + +import ( + . "github.com/smartystreets/goconvey/convey" + "os" + "testing" +) + +func TestExists(t *testing.T) { + var pathString = "../config" + var wrongPathString = "test" + + Convey("Test path or file is Exists or not", t, func() { + res := Exists(pathString) + Convey("The result should be true", func() { + So(res, ShouldEqual, true) + }) + wrongRes := Exists(wrongPathString) + Convey("The result should be false", func() { + So(wrongRes, ShouldEqual, false) + }) + }) +} + +func TestIsDir(t *testing.T) { + var pathString = "../config" + var fileString = "../config/config.go" + var wrongString = "test" + + Convey("Test path is folder or not", t, func() { + res := IsDir(pathString) + So(res, ShouldEqual, true) + fileRes := IsDir(fileString) + So(fileRes, ShouldEqual, false) + wrongRes := IsDir(wrongString) + So(wrongRes, ShouldEqual, false) + }) +} + +func TestCompress(t *testing.T) { + var pathString = "../utils" + var files []*os.File + var disPath = "../utils/test" + file, err := os.Open(pathString) + if err != nil { + t.Error("open source path failed") + } + files = append(files, file) + Convey("Verify dispath is valid path", t, func() { + er := Compress(files, disPath) + Convey("err should be nil", func() { + So(er, ShouldEqual, nil) + }) + }) + +} + +// 测试之前需存在有效的test(.zip)文件 +func TestDeCompress(t *testing.T) { + var tmpFilePath = "./test" + tmpFile, err := os.OpenFile(tmpFilePath, os.O_RDONLY, 0777) + if err != nil { + t.Fatal("open zip file failed") + } + var dstPath = "./testDeCompress" + Convey("Test DeCopmress func", t, func() { + + err := DeCompress(tmpFile, dstPath) + So(err, ShouldEqual, nil) + }) + +} diff --git a/backend/utils/model_test.go b/backend/utils/model_test.go new file mode 100644 index 00000000..d641865c --- /dev/null +++ b/backend/utils/model_test.go @@ -0,0 +1,49 @@ +package utils + +import ( + "github.com/globalsign/mgo/bson" + . "github.com/smartystreets/goconvey/convey" + "strconv" + "testing" + "time" +) + +func TestIsObjectIdNull(t *testing.T) { + var id bson.ObjectId = "123455" + Convey("Test Object ID is null or not", t, func() { + res := IsObjectIdNull(id) + So(res, ShouldEqual, false) + }) +} + +func TestInterfaceToString(t *testing.T) { + var valueBson bson.ObjectId = "12345" + var valueString = "12345" + var valueInt = 12345 + var valueTime = time.Now().Add(60 * time.Second) + var valueOther = []string{"a", "b"} + + Convey("Test InterfaceToString", t, func() { + resBson := InterfaceToString(valueBson) + Convey("resBson should be string value", func() { + So(resBson, ShouldEqual, valueBson.Hex()) + }) + resString := InterfaceToString(valueString) + Convey("resString should be string value", func() { + So(resString, ShouldEqual, valueString) + }) + resInt := InterfaceToString(valueInt) + Convey("resInt should be string value", func() { + So(resInt, ShouldEqual, strconv.Itoa(valueInt)) + }) + resTime := InterfaceToString(valueTime) + Convey("resTime should be string value", func() { + So(resTime, ShouldEqual, valueTime.String()) + }) + resOther := InterfaceToString(valueOther) + Convey("resOther should be empty string", func() { + So(resOther, ShouldEqual, "") + }) + }) + +} diff --git a/backend/utils/user_test.go b/backend/utils/user_test.go new file mode 100644 index 00000000..68cf4d65 --- /dev/null +++ b/backend/utils/user_test.go @@ -0,0 +1,14 @@ +package utils + +import ( + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestEncryptPassword(t *testing.T) { + var passwd = "test" + Convey("Test EncryptPassword", t, func() { + res := EncryptPassword(passwd) + t.Log(res) + }) +} From 3fa7b535f4c190a1e386d297b6782e10ca5fd6b9 Mon Sep 17 00:00:00 2001 From: hantmac Date: Mon, 19 Aug 2019 09:56:16 +0800 Subject: [PATCH 05/11] bug fix: fixed mongo.go unit test --- backend/database/mongo_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/backend/database/mongo_test.go b/backend/database/mongo_test.go index a73ae6df..b875b91a 100644 --- a/backend/database/mongo_test.go +++ b/backend/database/mongo_test.go @@ -1,6 +1,7 @@ package database import ( + "crawlab/config" . "github.com/smartystreets/goconvey/convey" "github.com/spf13/viper" "testing" @@ -8,12 +9,20 @@ import ( func TestGetDb(t *testing.T) { Convey("Test GetDb", t, func() { + if err := config.InitConfig(""); err != nil { + t.Fatal("Init config failed") + } + t.Log("初始化配置成功") + err := InitMongo() + if err != nil { + t.Fatal("Init mongodb failed") + } s, db := GetDb() Convey("The value should be Session.Copy", func() { - So(s, ShouldEqual, Session.Copy()) + So(s, ShouldResemble, Session.Copy()) }) Convey("The value should be reference of database", func() { - So(db, ShouldEqual, s.DB(viper.GetString("mongo.db"))) + So(db, ShouldResemble, s.DB(viper.GetString("mongo.db"))) }) }) } From 22c599c9ddd3da0c45540b44ee306963229e712e Mon Sep 17 00:00:00 2001 From: hantmac Date: Mon, 19 Aug 2019 18:06:49 +0800 Subject: [PATCH 06/11] Add unit test for routes/node --- backend/routes/node_test.go | 152 ++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 backend/routes/node_test.go diff --git a/backend/routes/node_test.go b/backend/routes/node_test.go new file mode 100644 index 00000000..ee3570c3 --- /dev/null +++ b/backend/routes/node_test.go @@ -0,0 +1,152 @@ +package routes + +import ( + "crawlab/config" + "crawlab/database" + "encoding/json" + "github.com/apex/log" + "github.com/gin-gonic/gin" + . "github.com/smartystreets/goconvey/convey" + "github.com/spf13/viper" + "net/http" + "net/http/httptest" + "runtime/debug" + "testing" +) + +var app *gin.Engine +// 本测试依赖MongoDB的服务，所以在测试之前需要启动MongoDB及相关服务 +func init() { + app = gin.Default() + + // 初始化配置 + if err := config.InitConfig("../conf/config.yml"); err != nil { + panic(err) + } + log.Info("初始化配置成功") + + // 初始化日志设置 + logLevel := viper.GetString("log.level") + if logLevel != "" { + log.SetLevelFromString(logLevel) + } + log.Info("初始化日志设置成功") + + // 初始化Mongodb数据库 + if err := database.InitMongo(); err != nil { + debug.PrintStack() + panic(err) + } + log.Info("初始化Mongodb数据库成功") + + // 初始化Redis数据库 + if err := database.InitRedis(); err != nil { + debug.PrintStack() + panic(err) + } + log.Info("初始化Redis数据库成功") + + // 路由 + // 节点相关的API + app.GET("/ping", Ping) + app.GET("/nodes", GetNodeList) // 节点列表 + app.GET("/nodes/:id", GetNode) // 节点详情 + app.POST("/nodes/:id", PostNode) // 修改节点 + app.GET("/nodes/:id/tasks", GetNodeTaskList) // 节点任务列表 + app.GET("/nodes/:id/system", GetSystemInfo) // 节点任务列表 + app.DELETE("/nodes/:id", DeleteNode) // 删除节点 + // 爬虫 +} + +//先测试GetNodeList得到一个节点的ID，再继续测试下面的API +func TestGetNodeList(t *testing.T) { + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/nodes", nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + t.Log(resp.Data) + Convey("Test API GetNodeList", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} + +//依赖MongoDB中的数据,_id=5d429e6c19f7abede924fee2,实际测试时需替换 +func TestGetNode(t *testing.T) { + var resp Response + var mongoId = "5d5a658319f7ab423585b0b0" + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/nodes/"+mongoId, nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + t.Log(resp.Data) + Convey("Test API GetNode", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + So(resp.Data.(map[string]interface{})["_id"], ShouldEqual, mongoId) + }) + }) +} + +func TestPing(t *testing.T) { + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/ping", nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + Convey("Test API ping", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} + +func TestGetNodeTaskList(t *testing.T) { + var resp Response + var mongoId = "5d5a658319f7ab423585b0b0" + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/tasks", nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + Convey("Test API GetNodeTaskList", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} + +func TestDeleteNode(t *testing.T) { + var resp Response + var mongoId = "5d5a658319f7ab423585b0b0" + w := httptest.NewRecorder() + req, _ := http.NewRequest("DELETE", "nodes/"+mongoId, nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + Convey("Test API DeleteNode", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} From be9def35dc36034f71bbd8055d02832ab4539444 Mon Sep 17 00:00:00 2001 From: hantmac Date: Mon, 19 Aug 2019 18:48:25 +0800 Subject: [PATCH 07/11] Complete unite test for backend/database/mongo.go --- backend/database/mongo_test.go | 44 +++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/backend/database/mongo_test.go b/backend/database/mongo_test.go index b875b91a..ed6044ee 100644 --- a/backend/database/mongo_test.go +++ b/backend/database/mongo_test.go @@ -2,14 +2,29 @@ package database import ( "crawlab/config" + "github.com/apex/log" + "github.com/globalsign/mgo" . "github.com/smartystreets/goconvey/convey" "github.com/spf13/viper" + "reflect" "testing" ) +func init() { + if err := config.InitConfig("../conf/config.yml"); err != nil { + log.Fatal("Init config failed") + } + log.Infof("初始化配置成功") + err := InitMongo() + if err != nil { + log.Fatal("Init mongodb failed") + } + +} + func TestGetDb(t *testing.T) { Convey("Test GetDb", t, func() { - if err := config.InitConfig(""); err != nil { + if err := config.InitConfig("../conf/config.yml"); err != nil { t.Fatal("Init config failed") } t.Log("初始化配置成功") @@ -26,3 +41,30 @@ func TestGetDb(t *testing.T) { }) }) } + +func TestGetCol(t *testing.T) { + var c = "nodes" + var colActual *mgo.Collection + Convey("Test GetCol", t, func() { + s, col := GetCol(c) + Convey("s should resemble Session.Copy", func() { + So(s, ShouldResemble, Session.Copy()) + So(reflect.TypeOf(col), ShouldResemble, reflect.TypeOf(colActual)) + }) + }) +} + +func TestGetGridFs(t *testing.T) { + var prefix = "files" + var gfActual *mgo.GridFS + + Convey("Test GetGridFs", t, func() { + s, gf := GetGridFs(prefix) + Convey("s should be session.copy", func() { + So(s, ShouldResemble, Session.Copy()) + }) + Convey("gf should be *mgo.GridFS", func() { + So(reflect.TypeOf(gf), ShouldResemble, reflect.TypeOf(gfActual)) + }) + }) +} From a599db1810ee0c828fb29a177697537be4cf087c Mon Sep 17 00:00:00 2001 From: hantmac Date: Tue, 20 Aug 2019 16:59:53 +0800 Subject: [PATCH 08/11] Code optimization:change the crawler zip file to slice read --- backend/services/spider.go | 61 ++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/backend/services/spider.go b/backend/services/spider.go index bf144959..9404c63a 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -7,10 +7,12 @@ import ( "crawlab/model" "crawlab/utils" "encoding/json" + "fmt" "github.com/apex/log" + "github.com/globalsign/mgo" "github.com/globalsign/mgo/bson" "github.com/pkg/errors" - uuid "github.com/satori/go.uuid" + "github.com/satori/go.uuid" "github.com/spf13/viper" "io" "io/ioutil" @@ -162,24 +164,6 @@ func ZipSpider(spider model.Spider) (filePath string, err error) { func UploadToGridFs(spider model.Spider, fileName string, filePath string) (fid bson.ObjectId, err error) { fid = "" - // 读取zip文件 - file, err := os.OpenFile(filePath, os.O_RDONLY, 0777) - fileBytes, err := ioutil.ReadAll(file) - if err != nil { - debug.PrintStack() - return - } - if err = file.Close(); err != nil { - debug.PrintStack() - return - } - - // 删除zip文件 - if err = os.Remove(filePath); err != nil { - debug.PrintStack() - return - } - // 获取MongoDB GridFS连接 s, gf := database.GetGridFs("files") defer s.Close() @@ -198,23 +182,54 @@ func UploadToGridFs(spider model.Spider, fileName string, filePath string) (fid return } - // 将文件写入到GridFS - if _, err = f.Write(fileBytes); err != nil { + //分片读取爬虫zip文件 + ReadFileByStep(filePath, WriteToGridFS, f) + + // 删除zip文件 + if err = os.Remove(filePath); err != nil { debug.PrintStack() return } - // 关闭文件，提交写入 if err = f.Close(); err != nil { return "", err } - // 文件ID fid = f.Id().(bson.ObjectId) return fid, nil } +func WriteToGridFS(content []byte, f *mgo.GridFile) { + if _, err := f.Write(content); err != nil { + debug.PrintStack() + return + } +} + +//分片读取大文件 +func ReadFileByStep(filePath string, handle func([]byte, *mgo.GridFile), fileCreate *mgo.GridFile) error { + f, err := os.OpenFile(filePath, os.O_RDONLY, 0777) + if err != nil { + log.Infof("can't opened this file") + return err + } + defer f.Close() + s := make([]byte, 4096) + for { + switch nr, err := f.Read(s[:]); true { + case nr < 0: + fmt.Fprintf(os.Stderr, "cat: error reading: %s\n", err.Error()) + debug.PrintStack() + case nr == 0: // EOF + return nil + case nr > 0: + handle(s[0:nr], fileCreate) + } + } + return nil +} + // 发布所有爬虫 func PublishAllSpiders() error { // 获取爬虫列表 From 4446f8704abb145deb24b202495df2c78f6dbe48 Mon Sep 17 00:00:00 2001 From: hantmac Date: Tue, 20 Aug 2019 17:09:43 +0800 Subject: [PATCH 09/11] Code optimization:change the crawler zip file to slice read --- backend/services/spider.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/services/spider.go b/backend/services/spider.go index 9404c63a..f4f856e6 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -183,7 +183,11 @@ func UploadToGridFs(spider model.Spider, fileName string, filePath string) (fid } //分片读取爬虫zip文件 - ReadFileByStep(filePath, WriteToGridFS, f) + err = ReadFileByStep(filePath, WriteToGridFS, f) + if err != nil { + debug.PrintStack() + return "", err + } // 删除zip文件 if err = os.Remove(filePath); err != nil { From 3643895f048086a158a9f127550f45f502eb577f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Wed, 21 Aug 2019 14:37:55 +0800 Subject: [PATCH 10/11] =?UTF-8?q?fix=20TagsView.vue=20=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E5=89=8D=E7=AB=AF=E6=8A=A5=E9=94=99=E7=9A=84=E9=97=AE=E9=A2=98?= =?UTF-8?q?=20update=20=E8=8A=82=E7=82=B9=E6=B3=A8=E5=86=8C=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E7=8B=AC=E7=AB=8B=E7=9A=84key=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=EF=BC=8C=E8=80=8C=E4=B8=8D=E6=98=AF=E4=BD=BF=E7=94=A8mac?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/node.go | 8 ++- backend/services/node.go | 70 ++++++++++++------- backend/services/register/register.go | 58 +++++++++------ .../src/views/layout/components/TagsView.vue | 24 ++++--- 4 files changed, 98 insertions(+), 62 deletions(-) diff --git a/backend/model/node.go b/backend/model/node.go index 01bbd3f7..61c20473 100644 --- a/backend/model/node.go +++ b/backend/model/node.go @@ -17,13 +17,15 @@ type Node struct { Port string `json:"port" bson:"port"` Mac string `json:"mac" bson:"mac"` Description string `json:"description" bson:"description"` + // 用于唯一标识节点，可能是mac地址，可能是ip地址 + Key string `json:"key" bson:"key"` // 前端展示 IsMaster bool `json:"is_master"` UpdateTs time.Time `json:"update_ts" bson:"update_ts"` CreateTs time.Time `json:"create_ts" bson:"create_ts"` - UpdateTsUnix int64 `json:"update_ts_unix" bson:"update_ts_unix"` + UpdateTsUnix int64 `json:"update_ts_unix" bson:"update_ts_unix"` } func (n *Node) Save() error { @@ -98,12 +100,12 @@ func GetNode(id bson.ObjectId) (Node, error) { return node, nil } -func GetNodeByMac(mac string) (Node, error) { +func GetNodeByKey(key string) (Node, error) { s, c := database.GetCol("nodes") defer s.Close() var node Node - if err := c.Find(bson.M{"mac": mac}).One(&node); err != nil { + if err := c.Find(bson.M{"key": key}).One(&node); err != nil { if err != mgo.ErrNotFound { log.Errorf(err.Error()) debug.PrintStack() diff --git a/backend/services/node.go b/backend/services/node.go index e5c8062a..1fa2370c 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -16,6 +16,7 @@ import ( ) type Data struct { + Key string `json:"key"` Mac string `json:"mac"` Ip string `json:"ip"` Master bool `json:"master"` @@ -51,10 +52,9 @@ const ( // 获取本机节点 func GetCurrentNode() (model.Node, error) { - // 获取本机MAC地址 - value, err := register.GetRegister().GetValue() + // 获得注册的key值 + key, err := register.GetRegister().GetKey() if err != nil { - debug.PrintStack() return model.Node{}, err } @@ -68,8 +68,7 @@ func GetCurrentNode() (model.Node, error) { } // 尝试获取节点 - node, err = model.GetNodeByMac(value) - + node, err = model.GetNodeByKey(key) // 如果获取失败 if err != nil { // 如果为主节点，表示为第一次注册，插入节点信息 @@ -80,12 +79,26 @@ func GetCurrentNode() (model.Node, error) { debug.PrintStack() return model.Node{}, err } + + mac, err := register.GetRegister().GetMac() + if err != nil { + debug.PrintStack() + return model.Node{}, err + } + + key, err := register.GetRegister().GetKey() + if err != nil { + debug.PrintStack() + return model.Node{}, err + } + // 生成节点 node = model.Node{ + Key: key, Id: bson.NewObjectId(), Ip: ip, - Name: value, - Mac: value, + Name: key, + Mac: mac, IsMaster: true, } if err := node.Add(); err != nil { @@ -100,11 +113,9 @@ func GetCurrentNode() (model.Node, error) { time.Sleep(5 * time.Second) continue } - // 跳出循环 break } - return node, nil } @@ -122,12 +133,12 @@ func IsMasterNode(id string) bool { // 获取节点数据 func GetNodeData() (Data, error) { - val, err := register.GetRegister().GetValue() - if err != nil { + key, err := register.GetRegister().GetKey() + if key == "" { return Data{}, err } - value, err := database.RedisClient.HGet("nodes", val) + value, err := database.RedisClient.HGet("nodes", key) data := Data{} if err := json.Unmarshal([]byte(value), &data); err != nil { return data, err @@ -145,9 +156,9 @@ func UpdateNodeStatus() { } // 遍历节点keys - for _, mac := range list { + for _, key := range list { // 获取节点数据 - value, err := database.RedisClient.HGet("nodes", mac) + value, err := database.RedisClient.HGet("nodes", key) if err != nil { log.Errorf(err.Error()) return @@ -163,7 +174,7 @@ func UpdateNodeStatus() { // 如果记录的更新时间超过60秒，该节点被认为离线 if time.Now().Unix()-data.UpdateTsUnix > 60 { // 在Redis中删除该节点 - if err := database.RedisClient.HDel("nodes", data.Mac); err != nil { + if err := database.RedisClient.HDel("nodes", data.Key); err != nil { log.Errorf(err.Error()) return } @@ -172,7 +183,7 @@ func UpdateNodeStatus() { s, c := database.GetCol("nodes") defer s.Close() var node model.Node - if err := c.Find(bson.M{"mac": mac}).One(&node); err != nil { + if err := c.Find(bson.M{"key": key}).One(&node); err != nil { log.Errorf(err.Error()) debug.PrintStack() return @@ -189,14 +200,16 @@ func UpdateNodeStatus() { s, c := database.GetCol("nodes") defer s.Close() var node model.Node - if err := c.Find(bson.M{"mac": mac}).One(&node); err != nil { + if err := c.Find(bson.M{"key": key}).One(&node); err != nil { // 数据库不存在该节点 node = model.Node{ - Name: data.Mac, - Ip: data.Ip, - Port: "8000", - Mac: data.Mac, - Status: constants.StatusOnline, + Key: key, + Name: key, + Ip: data.Ip, + Port: "8000", + Mac: data.Mac, + Status: constants.StatusOnline, + IsMaster: data.Master, } if err := node.Add(); err != nil { log.Errorf(err.Error()) @@ -216,8 +229,8 @@ func UpdateNodeStatus() { nodes, err := model.GetNodeList(nil) for _, node := range nodes { hasNode := false - for _, mac := range list { - if mac == node.Mac { + for _, key := range list { + if key == node.Key { hasNode = true break } @@ -236,7 +249,7 @@ func UpdateNodeStatus() { // 更新节点数据 func UpdateNodeData() { // 获取MAC地址 - val, err := register.GetRegister().GetValue() + mac, err := register.GetRegister().GetMac() if err != nil { log.Errorf(err.Error()) return @@ -248,10 +261,13 @@ func UpdateNodeData() { log.Errorf(err.Error()) return } + // 获取redis的key + key, err := register.GetRegister().GetKey() // 构造节点数据 data := Data{ - Mac: val, + Key: key, + Mac: mac, Ip: ip, Master: IsMaster(), UpdateTs: time.Now(), @@ -265,7 +281,7 @@ func UpdateNodeData() { debug.PrintStack() return } - if err := database.RedisClient.HSet("nodes", val, string(dataBytes)); err != nil { + if err := database.RedisClient.HSet("nodes", key, string(dataBytes)); err != nil { log.Errorf(err.Error()) return } diff --git a/backend/services/register/register.go b/backend/services/register/register.go index 8f0d169e..ccd8b67d 100644 --- a/backend/services/register/register.go +++ b/backend/services/register/register.go @@ -10,50 +10,44 @@ import ( type Register interface { // 注册的key类型 - GetKey() string + GetType() string // 注册的key的值，唯一标识节点 - GetValue() (string, error) + GetKey() (string, error) // 注册的节点IP GetIp() (string, error) + // 注册节点的mac地址 + GetMac() (string, error) } -// mac 地址注册 +// ===================== mac 地址注册 ===================== type MacRegister struct{} -func (mac *MacRegister) GetKey() string { +func (mac *MacRegister) GetType() string { return "mac" } -func (mac *MacRegister) GetValue() (string, error) { - interfaces, err := net.Interfaces() - if err != nil { - log.Errorf("get interfaces error:" + err.Error()) - debug.PrintStack() - return "", err - } - for _, inter := range interfaces { - if inter.HardwareAddr != nil { - mac := inter.HardwareAddr.String() - return mac, nil - } - } - return "", nil +func (mac *MacRegister) GetKey() (string, error) { + return mac.GetMac() +} + +func (mac *MacRegister) GetMac() (string, error) { + return getMac() } func (mac *MacRegister) GetIp() (string, error) { return getIp() } -// ip 注册 +// ===================== ip 地址注册 ===================== type IpRegister struct { Ip string } -func (ip *IpRegister) GetKey() string { +func (ip *IpRegister) GetType() string { return "ip" } -func (ip *IpRegister) GetValue() (string, error) { +func (ip *IpRegister) GetKey() (string, error) { return ip.Ip, nil } @@ -61,6 +55,11 @@ func (ip *IpRegister) GetIp() (string, error) { return ip.Ip, nil } +func (ip *IpRegister) GetMac() (string, error) { + return getMac() +} + +// ===================== 公共方法 ===================== // 获取本机的IP地址 // TODO: 考虑多个IP地址的情况 func getIp() (string, error) { @@ -78,6 +77,23 @@ func getIp() (string, error) { return "", nil } +func getMac() (string, error) { + interfaces, err := net.Interfaces() + if err != nil { + log.Errorf("get interfaces error:" + err.Error()) + debug.PrintStack() + return "", err + } + for _, inter := range interfaces { + if inter.HardwareAddr != nil { + mac := inter.HardwareAddr.String() + return mac, nil + } + } + return "", nil +} + +// ===================== 获得注册简单工厂 ===================== var register Register // 获得注册器 diff --git a/frontend/src/views/layout/components/TagsView.vue b/frontend/src/views/layout/components/TagsView.vue index e97c8dc8..231b44f0 100644 --- a/frontend/src/views/layout/components/TagsView.vue +++ b/frontend/src/views/layout/components/TagsView.vue @@ -110,20 +110,22 @@ export default { }, moveToCurrentTag () { const tags = this.$refs.tag - this.$nextTick(() => { - for (const tag of tags) { - if (tag.to.path === this.$route.path) { - this.$refs.scrollPane.moveToTarget(tag) + if (tags) { + this.$nextTick(() => { + for (const tag of tags) { + if (tag.to.path === this.$route.path) { + this.$refs.scrollPane.moveToTarget(tag) - // when query is different then update - if (tag.to.fullPath !== this.$route.fullPath) { - this.$store.dispatch('updateVisitedView', this.$route) + // when query is different then update + if (tag.to.fullPath !== this.$route.fullPath) { + this.$store.dispatch('updateVisitedView', this.$route) + } + + break } - - break } - } - }) + }) + } }, refreshSelectedTag (view) { this.$store.dispatch('delCachedView', view).then(() => { From 87f1cebc3baa984ebd4531ed72e7cf0e8868c549 Mon Sep 17 00:00:00 2001 From: hantmac Date: Thu, 22 Aug 2019 16:20:30 +0800 Subject: [PATCH 11/11] modify unit test to use mock --- backend/mock/base.go | 16 ++ backend/mock/file.go | 8 + backend/mock/node.go | 210 ++++++++++++++++++++++++++ backend/{routes => mock}/node_test.go | 119 +++++++++------ backend/mock/schedule.go | 126 ++++++++++++++++ backend/mock/schedule_test.go | 144 ++++++++++++++++++ backend/mock/spider.go | 1 + backend/mock/system.go | 1 + backend/mock/task.go | 1 + backend/mock/user.go | 1 + backend/mock/utils.go | 24 +++ 11 files changed, 609 insertions(+), 42 deletions(-) create mode 100644 backend/mock/base.go create mode 100644 backend/mock/file.go create mode 100644 backend/mock/node.go rename backend/{routes => mock}/node_test.go (57%) create mode 100644 backend/mock/schedule.go create mode 100644 backend/mock/schedule_test.go create mode 100644 backend/mock/spider.go create mode 100644 backend/mock/system.go create mode 100644 backend/mock/task.go create mode 100644 backend/mock/user.go create mode 100644 backend/mock/utils.go diff --git a/backend/mock/base.go b/backend/mock/base.go new file mode 100644 index 00000000..d8b11eb9 --- /dev/null +++ b/backend/mock/base.go @@ -0,0 +1,16 @@ +package mock + +type Response struct { + Status string `json:"status"` + Message string `json:"message"` + Data interface{} `json:"data"` + Error string `json:"error"` +} + +type ListResponse struct { + Status string `json:"status"` + Message string `json:"message"` + Total int `json:"total"` + Data interface{} `json:"data"` + Error string `json:"error"` +} diff --git a/backend/mock/file.go b/backend/mock/file.go new file mode 100644 index 00000000..addd771a --- /dev/null +++ b/backend/mock/file.go @@ -0,0 +1,8 @@ +package mock + +type File struct { + Name string `json:"name"` + Path string `json:"path"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size"` +} diff --git a/backend/mock/node.go b/backend/mock/node.go new file mode 100644 index 00000000..878dbcfa --- /dev/null +++ b/backend/mock/node.go @@ -0,0 +1,210 @@ +package mock + +import ( + "crawlab/model" + "crawlab/services" + "github.com/apex/log" + "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" + "net/http" + "time" +) + +var NodeList = []model.Node{ + { + Id: bson.ObjectId("5d429e6c19f7abede924fee2"), + Ip: "10.32.35.15", + Name: "test1", + Status: "online", + Port: "8081", + Mac: "ac:12:df:12:fd", + Description: "For test1", + IsMaster: true, + UpdateTs: time.Now(), + CreateTs: time.Now(), + UpdateTsUnix: time.Now().Unix(), + }, + { + Id: bson.ObjectId("5d429e6c19f7abede924fe22"), + Ip: "10.32.35.12", + Name: "test2", + Status: "online", + Port: "8082", + Mac: "ac:12:df:12:vh", + Description: "For test2", + IsMaster: true, + UpdateTs: time.Now(), + CreateTs: time.Now(), + UpdateTsUnix: time.Now().Unix(), + }, +} + +var TaskList = []model.Task{ + { + Id: "1234", + SpiderId: bson.ObjectId("xx429e6c19f7abede924fee2"), + StartTs: time.Now(), + FinishTs: time.Now(), + Status: "进行中", + NodeId: bson.ObjectId("5d429e6c19f7abede924fee2"), + LogPath: "./log", + Cmd: "scrapy crawl test", + Error: "", + ResultCount: 0, + WaitDuration: 10.0, + RuntimeDuration: 10, + TotalDuration: 20, + SpiderName: "test", + NodeName: "test", + CreateTs: time.Now(), + UpdateTs: time.Now(), + }, + { + Id: "5678", + SpiderId: bson.ObjectId("xx429e6c19f7abede924fddf"), + StartTs: time.Now(), + FinishTs: time.Now(), + Status: "进行中", + NodeId: bson.ObjectId("5d429e6c19f7abede924fee2"), + LogPath: "./log", + Cmd: "scrapy crawl test2", + Error: "", + ResultCount: 0, + WaitDuration: 10.0, + RuntimeDuration: 10, + TotalDuration: 20, + SpiderName: "test", + NodeName: "test", + CreateTs: time.Now(), + UpdateTs: time.Now(), + }, +} + +var dataList = []services.Data{ + { + Mac: "ac:12:fc:fd:ds:dd", + Ip: "192.10.2.1", + Master: true, + UpdateTs: time.Now(), + UpdateTsUnix: time.Now().Unix(), + }, + { + Mac: "22:12:fc:fd:ds:dd", + Ip: "182.10.2.2", + Master: true, + UpdateTs: time.Now(), + UpdateTsUnix: time.Now().Unix(), + }, +} + +var executeble = []model.Executable{ + { + Path: "/test", + FileName: "test.py", + DisplayName: "test.py", + }, +} +var systemInfo = model.SystemInfo{ARCH: "x86", + OS: "linux", + Hostname: "test", + NumCpu: 4, + Executables: executeble, +} + +func GetNodeList(c *gin.Context) { + nodes := NodeList + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: nodes, + }) +} + +func GetNode(c *gin.Context) { + var result model.Node + id := c.Param("id") + for _, node := range NodeList { + if node.Id == bson.ObjectId(id) { + result = node + } + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: result, + }) +} + +func Ping(c *gin.Context) { + data := dataList[0] + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: data, + }) +} + +func PostNode(c *gin.Context) { + id := c.Param("id") + var oldItem model.Node + for _, node := range NodeList { + if node.Id == bson.ObjectId(id) { + oldItem = node + } + + } + log.Info(id) + var newItem model.Node + if err := c.ShouldBindJSON(&newItem); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + newItem.Id = oldItem.Id + + log.Info("Post Node success") + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func GetNodeTaskList(c *gin.Context) { + + tasks := TaskList + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: tasks, + }) +} + +func DeleteNode(c *gin.Context) { + id := bson.ObjectId("5d429e6c19f7abede924fee2") + + for _, node := range NodeList { + if node.Id == bson.ObjectId(id) { + log.Infof("Delete a node") + } + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func GetSystemInfo(c *gin.Context) { + id := c.Param("id") + log.Info(id) + sysInfo := systemInfo + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: sysInfo, + }) +} diff --git a/backend/routes/node_test.go b/backend/mock/node_test.go similarity index 57% rename from backend/routes/node_test.go rename to backend/mock/node_test.go index ee3570c3..9d7096b3 100644 --- a/backend/routes/node_test.go +++ b/backend/mock/node_test.go @@ -1,17 +1,17 @@ -package routes +package mock import ( - "crawlab/config" - "crawlab/database" + "crawlab/model" "encoding/json" - "github.com/apex/log" "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" . "github.com/smartystreets/goconvey/convey" - "github.com/spf13/viper" "net/http" "net/http/httptest" - "runtime/debug" + "strings" "testing" + "time" + "ucloudBilling/ucloud/log" ) var app *gin.Engine @@ -19,34 +19,7 @@ var app *gin.Engine func init() { app = gin.Default() - // 初始化配置 - if err := config.InitConfig("../conf/config.yml"); err != nil { - panic(err) - } - log.Info("初始化配置成功") - - // 初始化日志设置 - logLevel := viper.GetString("log.level") - if logLevel != "" { - log.SetLevelFromString(logLevel) - } - log.Info("初始化日志设置成功") - - // 初始化Mongodb数据库 - if err := database.InitMongo(); err != nil { - debug.PrintStack() - panic(err) - } - log.Info("初始化Mongodb数据库成功") - - // 初始化Redis数据库 - if err := database.InitRedis(); err != nil { - debug.PrintStack() - panic(err) - } - log.Info("初始化Redis数据库成功") - - // 路由 + // mock Test // 节点相关的API app.GET("/ping", Ping) app.GET("/nodes", GetNodeList) // 节点列表 @@ -55,20 +28,27 @@ func init() { app.GET("/nodes/:id/tasks", GetNodeTaskList) // 节点任务列表 app.GET("/nodes/:id/system", GetSystemInfo) // 节点任务列表 app.DELETE("/nodes/:id", DeleteNode) // 删除节点 - // 爬虫 + //// 爬虫 + // 定时任务 + app.GET("/schedules", GetScheduleList) // 定时任务列表 + app.GET("/schedules/:id", GetSchedule) // 定时任务详情 + app.PUT("/schedules", PutSchedule) // 创建定时任务 + app.POST("/schedules/:id", PostSchedule) // 修改定时任务 + app.DELETE("/schedules/:id", DeleteSchedule) // 删除定时任务 } -//先测试GetNodeList得到一个节点的ID，再继续测试下面的API +//mock test, test data in ./mock func TestGetNodeList(t *testing.T) { var resp Response w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/nodes", nil) app.ServeHTTP(w, req) err := json.Unmarshal([]byte(w.Body.String()), &resp) + t.Log(resp.Data) if err != nil { t.Fatal("Unmarshal resp failed") } - t.Log(resp.Data) + Convey("Test API GetNodeList", t, func() { Convey("Test response status", func() { So(resp.Status, ShouldEqual, "ok") @@ -77,10 +57,9 @@ func TestGetNodeList(t *testing.T) { }) } -//依赖MongoDB中的数据,_id=5d429e6c19f7abede924fee2,实际测试时需替换 func TestGetNode(t *testing.T) { var resp Response - var mongoId = "5d5a658319f7ab423585b0b0" + var mongoId = "5d429e6c19f7abede924fee2" w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/nodes/"+mongoId, nil) app.ServeHTTP(w, req) @@ -93,7 +72,7 @@ func TestGetNode(t *testing.T) { Convey("Test response status", func() { So(resp.Status, ShouldEqual, "ok") So(resp.Message, ShouldEqual, "success") - So(resp.Data.(map[string]interface{})["_id"], ShouldEqual, mongoId) + So(resp.Data.(map[string]interface{})["_id"], ShouldEqual, bson.ObjectId(mongoId).Hex()) }) }) } @@ -117,7 +96,7 @@ func TestPing(t *testing.T) { func TestGetNodeTaskList(t *testing.T) { var resp Response - var mongoId = "5d5a658319f7ab423585b0b0" + var mongoId = "5d429e6c19f7abede924fee2" w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/tasks", nil) app.ServeHTTP(w, req) @@ -135,7 +114,8 @@ func TestGetNodeTaskList(t *testing.T) { func TestDeleteNode(t *testing.T) { var resp Response - var mongoId = "5d5a658319f7ab423585b0b0" + + var mongoId = "5d429e6c19f7abede924fee2" w := httptest.NewRecorder() req, _ := http.NewRequest("DELETE", "nodes/"+mongoId, nil) app.ServeHTTP(w, req) @@ -150,3 +130,58 @@ func TestDeleteNode(t *testing.T) { }) }) } + +func TestPostNode(t *testing.T) { + var newItem = model.Node{ + Id: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + Ip: "10.32.35.15", + Name: "test1", + Status: "online", + Port: "8081", + Mac: "ac:12:df:12:fd", + Description: "For test1", + IsMaster: true, + UpdateTs: time.Now(), + CreateTs: time.Now(), + UpdateTsUnix: time.Now().Unix(), + } + + var resp Response + body, _ := json.Marshal(newItem) + log.Info(strings.NewReader(string(body))) + + var mongoId = "5d429e6c19f7abede924fee2" + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", "nodes/"+mongoId, strings.NewReader(string(body))) + + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + t.Log(resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + Convey("Test API PostNode", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} + +func TestGetSystemInfo(t *testing.T) { + var resp Response + var mongoId = "5d429e6c19f7abede924fee2" + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/system", nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + Convey("Test API GetSystemInfo", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} diff --git a/backend/mock/schedule.go b/backend/mock/schedule.go new file mode 100644 index 00000000..ae982ca6 --- /dev/null +++ b/backend/mock/schedule.go @@ -0,0 +1,126 @@ +package mock + +import ( + "crawlab/constants" + "crawlab/model" + "fmt" + "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" + "net/http" + "time" +) + +var scheduleList = []model.Schedule{ + { + Id: bson.ObjectId("5d429e6c19f7abede924fee2"), + Name: "test schedule", + SpiderId: "123", + NodeId: bson.ObjectId("5d429e6c19f7abede924fee2"), + Cron: "***1*", + EntryId: 10, + // 前端展示 + SpiderName: "test scedule", + NodeName: "测试节点", + + CreateTs: time.Now(), + UpdateTs: time.Now(), + }, + { + Id: bson.ObjectId("xx429e6c19f7abede924fee2"), + Name: "test schedule2", + SpiderId: "234", + NodeId: bson.ObjectId("5d429e6c19f7abede924fee2"), + Cron: "***1*", + EntryId: 10, + // 前端展示 + SpiderName: "test scedule2", + NodeName: "测试节点", + + CreateTs: time.Now(), + UpdateTs: time.Now(), + }, +} + +func GetScheduleList(c *gin.Context) { + results := scheduleList + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: results, + }) +} + +func GetSchedule(c *gin.Context) { + id := c.Param("id") + + var result model.Schedule + for _, sch := range scheduleList { + if sch.Id == bson.ObjectId(id) { + result = sch + } + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: result, + }) +} + +func PostSchedule(c *gin.Context) { + id := c.Param("id") + var oldItem model.Schedule + for _, sch := range scheduleList { + if sch.Id == bson.ObjectId(id) { + oldItem = sch + } + + } + + var newItem model.Schedule + if err := c.ShouldBindJSON(&newItem); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + newItem.Id = oldItem.Id + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func PutSchedule(c *gin.Context) { + var item model.Schedule + + // 绑定数据模型 + if err := c.ShouldBindJSON(&item); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + // 如果node_id为空，则置为空ObjectId + if item.NodeId == "" { + item.NodeId = bson.ObjectIdHex(constants.ObjectIdNull) + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func DeleteSchedule(c *gin.Context) { + id := bson.ObjectIdHex("5d429e6c19f7abede924fee2") + for _, sch := range scheduleList { + if sch.Id == bson.ObjectId(id) { + fmt.Println("delete a schedule") + } + } + fmt.Println(id) + fmt.Println("update schedule") + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} diff --git a/backend/mock/schedule_test.go b/backend/mock/schedule_test.go new file mode 100644 index 00000000..d26a08d8 --- /dev/null +++ b/backend/mock/schedule_test.go @@ -0,0 +1,144 @@ +package mock + +import ( + "crawlab/model" + "encoding/json" + "github.com/globalsign/mgo/bson" + . "github.com/smartystreets/goconvey/convey" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + "ucloudBilling/ucloud/log" +) + +func TestGetScheduleList(t *testing.T) { + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/schedules", nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + t.Log(resp.Data) + Convey("Test API GetScheduleList", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} + +func TestGetSchedule(t *testing.T) { + var mongoId = "5d429e6c19f7abede924fee2" + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/schedules/"+mongoId, nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + Convey("Test API GetSchedule", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + So(resp.Data.(map[string]interface{})["_id"], ShouldEqual, bson.ObjectId(mongoId).Hex()) + }) + }) +} + +func TestDeleteSchedule(t *testing.T) { + var mongoId = "5d429e6c19f7abede924fee2" + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("DELETE", "/schedules/"+mongoId, nil) + app.ServeHTTP(w, req) + + err := json.Unmarshal([]byte(w.Body.String()), &resp) + log.Info(w.Body.String()) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + + Convey("Test DeleteSchedule", t, func() { + Convey("Test resp status", func() { + So(resp.Status, ShouldEqual, "ok") + }) + }) +} + +func TestPostSchedule(t *testing.T) { + var newItem = model.Schedule{ + Id: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + Name: "test schedule", + SpiderId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + NodeId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + Cron: "***1*", + EntryId: 10, + // 前端展示 + SpiderName: "test scedule", + NodeName: "测试节点", + + CreateTs: time.Now(), + UpdateTs: time.Now(), + } + + var resp Response + var mongoId = "5d429e6c19f7abede924fee2" + body,_ := json.Marshal(newItem) + log.Info(strings.NewReader(string(body))) + w := httptest.NewRecorder() + req,_ := http.NewRequest("POST", "/schedules/"+mongoId,strings.NewReader(string(body))) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()),&resp) + t.Log(resp) + if err != nil { + t.Fatal("unmarshal resp failed") + } + Convey("Test API PostSchedule", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) + +} + +func TestPutSchedule(t *testing.T) { + var newItem = model.Schedule{ + Id: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + Name: "test schedule", + SpiderId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + NodeId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + Cron: "***1*", + EntryId: 10, + // 前端展示 + SpiderName: "test scedule", + NodeName: "测试节点", + + CreateTs: time.Now(), + UpdateTs: time.Now(), + } + + var resp Response + body,_ := json.Marshal(newItem) + log.Info(strings.NewReader(string(body))) + w := httptest.NewRecorder() + req,_ := http.NewRequest("PUT", "/schedules",strings.NewReader(string(body))) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()),&resp) + t.Log(resp) + if err != nil { + t.Fatal("unmarshal resp failed") + } + Convey("Test API PutSchedule", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) + +} diff --git a/backend/mock/spider.go b/backend/mock/spider.go new file mode 100644 index 00000000..c4807247 --- /dev/null +++ b/backend/mock/spider.go @@ -0,0 +1 @@ +package mock \ No newline at end of file diff --git a/backend/mock/system.go b/backend/mock/system.go new file mode 100644 index 00000000..c4807247 --- /dev/null +++ b/backend/mock/system.go @@ -0,0 +1 @@ +package mock \ No newline at end of file diff --git a/backend/mock/task.go b/backend/mock/task.go new file mode 100644 index 00000000..c4807247 --- /dev/null +++ b/backend/mock/task.go @@ -0,0 +1 @@ +package mock \ No newline at end of file diff --git a/backend/mock/user.go b/backend/mock/user.go new file mode 100644 index 00000000..c4807247 --- /dev/null +++ b/backend/mock/user.go @@ -0,0 +1 @@ +package mock \ No newline at end of file diff --git a/backend/mock/utils.go b/backend/mock/utils.go new file mode 100644 index 00000000..fd7d4efd --- /dev/null +++ b/backend/mock/utils.go @@ -0,0 +1,24 @@ +package mock + +import ( + "github.com/gin-gonic/gin" + "runtime/debug" +) + +func HandleError(statusCode int, c *gin.Context, err error) { + debug.PrintStack() + c.JSON(statusCode, Response{ + Status: "ok", + Message: "error", + Error: err.Error(), + }) +} + +func HandleErrorF(statusCode int, c *gin.Context, err string) { + debug.PrintStack() + c.JSON(statusCode, Response{ + Status: "ok", + Message: "error", + Error: err, + }) +}