From f694aea59084ab65df31ed2efe801f042e2c55cb Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 18 Aug 2019 13:09:37 +0800 Subject: [PATCH 1/6] updated Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a6f5f9ab..774975db 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,7 +46,7 @@ pipeline { steps { echo 'Deploying....' sh """ - if [ ${ENV:GIT_BRANCH} -eq master ] || [ ${ENV:GIT_BRANCH} -eq develop ]; then + if [[ ${ENV:GIT_BRANCH} = master ]] || [[ ${ENV:GIT_BRANCH} = develop ]]; then # 重启docker compose cd ./jenkins/${ENV:GIT_BRANCH} docker-compose stop | true From c516596e1d4c577183fefb5cfe5877080f9bdd89 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 18 Aug 2019 13:12:53 +0800 Subject: [PATCH 2/6] updated README --- README-zh.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README-zh.md b/README-zh.md index fa9d45be..8b47f402 100644 --- a/README-zh.md +++ b/README-zh.md @@ -4,7 +4,7 @@ ![](https://img.shields.io/github/release/tikazyq/crawlab.svg) ![](https://img.shields.io/github/last-commit/tikazyq/crawlab.svg) ![](https://img.shields.io/github/issues/tikazyq/crawlab.svg) -![](https://img.shields.io/github/contributers/tikazyq/crawlab.svg) +![](https://img.shields.io/github/contributors/tikazyq/crawlab.svg) ![](https://img.shields.io/docker/pulls/tikazyq/crawlab) ![](https://img.shields.io/github/license/tikazyq/crawlab.svg) diff --git a/README.md b/README.md index 689a224d..027c8bd7 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ![](https://img.shields.io/github/release/tikazyq/crawlab.svg) ![](https://img.shields.io/github/last-commit/tikazyq/crawlab.svg) ![](https://img.shields.io/github/issues/tikazyq/crawlab.svg) -![](https://img.shields.io/github/contributers/tikazyq/crawlab.svg) +![](https://img.shields.io/github/contributors/tikazyq/crawlab.svg) ![](https://img.shields.io/docker/pulls/tikazyq/crawlab) ![](https://img.shields.io/github/license/tikazyq/crawlab.svg) From 98470b87e3e11af8930d759b50ee9b1bc44b09cf Mon Sep 17 00:00:00 2001 From: hantmac Date: Sun, 18 Aug 2019 14:20:37 +0800 Subject: [PATCH 3/6] Add Unit Test for backend/utils/chan.go --- backend/utils/chan_test.go | 76 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 backend/utils/chan_test.go diff --git a/backend/utils/chan_test.go b/backend/utils/chan_test.go new file mode 100644 index 00000000..7b5f1bba --- /dev/null +++ b/backend/utils/chan_test.go @@ -0,0 +1,76 @@ +package utils + +import ( + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestNewChanMap(t *testing.T) { + mapTest := make(map[string]chan string) + chanTest := make(chan string) + test := "test" + + Convey("Call NewChanMap to generate ChanMap", t, func() { + mapTest[test] = chanTest + chanMapTest := ChanMap{mapTest} + chanMap := NewChanMap() + chanMap.m[test] = chanTest + + Convey(test, func() { + So(chanMap, ShouldResemble, &chanMapTest) + }) + + }) +} + +func TestChan(t *testing.T) { + mapTest := make(map[string]chan string) + chanTest := make(chan string) + mapTest["test"] = chanTest + chanMapTest := ChanMap{mapTest} + + Convey("Test Chan use exist key", t, func() { + ch1 := chanMapTest.Chan( + "test") + Convey("ch1 should equal chanTest", func() { + So(ch1, ShouldEqual, chanTest) + }) + + }) + Convey("Test Chan use no-exist key", t, func() { + ch2 := chanMapTest.Chan("test2") + Convey("ch2 should equal chanMapTest.m[test2]", func() { + + So(chanMapTest.m["test2"], ShouldEqual, ch2) + }) + Convey("Cap of chanMapTest.m[test2] should equal 10", func() { + So(10, ShouldEqual, cap(chanMapTest.m["test2"])) + }) + }) +} + +func TestChanBlocked(t *testing.T) { + mapTest := make(map[string]chan string) + chanTest := make(chan string) + mapTest["test"] = chanTest + chanMapTest := ChanMap{mapTest} + + Convey("Test Chan use exist key", t, func() { + ch1 := chanMapTest.ChanBlocked( + "test") + Convey("ch1 should equal chanTest", func() { + So(ch1, ShouldEqual, chanTest) + }) + + }) + Convey("Test Chan use no-exist key", t, func() { + ch2 := chanMapTest.ChanBlocked("test2") + Convey("ch2 should equal chanMapTest.m[test2]", func() { + + So(chanMapTest.m["test2"], ShouldEqual, ch2) + }) + Convey("Cap of chanMapTest.m[test2] should equal 10", func() { + So(0, ShouldEqual, cap(chanMapTest.m["test2"])) + }) + }) +} From 122919658bfdd4c422dd2503128528aeaf86b10a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sun, 18 Aug 2019 18:45:19 +0800 Subject: [PATCH 4/6] =?UTF-8?q?fix=20tmp=20=E7=9B=AE=E5=BD=95=E4=B8=8D?= =?UTF-8?q?=E4=BC=9A=E8=87=AA=E5=8A=A8=E5=88=9B=E5=BB=BA=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/go.sum | 5 +++++ backend/routes/spider.go | 14 +++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/backend/go.sum b/backend/go.sum index 82fc37b5..910e18be 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -53,6 +53,7 @@ github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -65,6 +66,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -117,8 +119,10 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -184,6 +188,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= diff --git a/backend/routes/spider.go b/backend/routes/spider.go index e83d7ed8..dceb2651 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -131,10 +131,22 @@ func PutSpider(c *gin.Context) { return } + // 以防tmp目录不存在 + tmpPath := viper.GetString("other.tmppath") + if !utils.Exists(tmpPath) { + if err := os.Mkdir(tmpPath, os.ModePerm); err != nil { + log.Error("mkdir other.tmppath dir error:" + err.Error()) + debug.PrintStack() + HandleError(http.StatusBadRequest, c, errors.New("Mkdir other.tmppath dir error")) + return + } + } + // 保存到本地临时文件 randomId := uuid.NewV4() - tmpFilePath := filepath.Join(viper.GetString("other.tmppath"), randomId.String()+".zip") + tmpFilePath := filepath.Join(tmpPath, randomId.String()+".zip") if err := c.SaveUploadedFile(file, tmpFilePath); err != nil { + log.Error("save upload file error: " + err.Error()) debug.PrintStack() HandleError(http.StatusInternalServerError, c, err) return From 57a8cf9dfc02ed774d1894b95df26fbbf396af3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sun, 18 Aug 2019 18:48:38 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=BF=BD=E7=95=A5backend?= =?UTF-8?q?/spiders=E7=9B=AE=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 13c696a5..2c38a7c2 100644 --- a/.gitignore +++ b/.gitignore @@ -119,4 +119,6 @@ logs/ tmp/ _book/ .idea -*.lock \ No newline at end of file +*.lock + +backend/spiders \ No newline at end of file From 0fe20c07f45a1d4d4a2fd5fb5fa4d5e796ae2cfa Mon Sep 17 00:00:00 2001 From: hantmac Date: Sun, 18 Aug 2019 19:07:59 +0800 Subject: [PATCH 6/6] Complete Unit Test for package utils --- backend/utils/file_test.go | 72 +++++++++++++++++++++++++++++++++++++ backend/utils/model_test.go | 49 +++++++++++++++++++++++++ backend/utils/user_test.go | 14 ++++++++ 3 files changed, 135 insertions(+) create mode 100644 backend/utils/file_test.go create mode 100644 backend/utils/model_test.go create mode 100644 backend/utils/user_test.go diff --git a/backend/utils/file_test.go b/backend/utils/file_test.go new file mode 100644 index 00000000..484366f5 --- /dev/null +++ b/backend/utils/file_test.go @@ -0,0 +1,72 @@ +package utils + +import ( + . "github.com/smartystreets/goconvey/convey" + "os" + "testing" +) + +func TestExists(t *testing.T) { + var pathString = "../config" + var wrongPathString = "test" + + Convey("Test path or file is Exists or not", t, func() { + res := Exists(pathString) + Convey("The result should be true", func() { + So(res, ShouldEqual, true) + }) + wrongRes := Exists(wrongPathString) + Convey("The result should be false", func() { + So(wrongRes, ShouldEqual, false) + }) + }) +} + +func TestIsDir(t *testing.T) { + var pathString = "../config" + var fileString = "../config/config.go" + var wrongString = "test" + + Convey("Test path is folder or not", t, func() { + res := IsDir(pathString) + So(res, ShouldEqual, true) + fileRes := IsDir(fileString) + So(fileRes, ShouldEqual, false) + wrongRes := IsDir(wrongString) + So(wrongRes, ShouldEqual, false) + }) +} + +func TestCompress(t *testing.T) { + var pathString = "../utils" + var files []*os.File + var disPath = "../utils/test" + file, err := os.Open(pathString) + if err != nil { + t.Error("open source path failed") + } + files = append(files, file) + Convey("Verify dispath is valid path", t, func() { + er := Compress(files, disPath) + Convey("err should be nil", func() { + So(er, ShouldEqual, nil) + }) + }) + +} + +// 测试之前需存在有效的test(.zip)文件 +func TestDeCompress(t *testing.T) { + var tmpFilePath = "./test" + tmpFile, err := os.OpenFile(tmpFilePath, os.O_RDONLY, 0777) + if err != nil { + t.Fatal("open zip file failed") + } + var dstPath = "./testDeCompress" + Convey("Test DeCopmress func", t, func() { + + err := DeCompress(tmpFile, dstPath) + So(err, ShouldEqual, nil) + }) + +} diff --git a/backend/utils/model_test.go b/backend/utils/model_test.go new file mode 100644 index 00000000..d641865c --- /dev/null +++ b/backend/utils/model_test.go @@ -0,0 +1,49 @@ +package utils + +import ( + "github.com/globalsign/mgo/bson" + . "github.com/smartystreets/goconvey/convey" + "strconv" + "testing" + "time" +) + +func TestIsObjectIdNull(t *testing.T) { + var id bson.ObjectId = "123455" + Convey("Test Object ID is null or not", t, func() { + res := IsObjectIdNull(id) + So(res, ShouldEqual, false) + }) +} + +func TestInterfaceToString(t *testing.T) { + var valueBson bson.ObjectId = "12345" + var valueString = "12345" + var valueInt = 12345 + var valueTime = time.Now().Add(60 * time.Second) + var valueOther = []string{"a", "b"} + + Convey("Test InterfaceToString", t, func() { + resBson := InterfaceToString(valueBson) + Convey("resBson should be string value", func() { + So(resBson, ShouldEqual, valueBson.Hex()) + }) + resString := InterfaceToString(valueString) + Convey("resString should be string value", func() { + So(resString, ShouldEqual, valueString) + }) + resInt := InterfaceToString(valueInt) + Convey("resInt should be string value", func() { + So(resInt, ShouldEqual, strconv.Itoa(valueInt)) + }) + resTime := InterfaceToString(valueTime) + Convey("resTime should be string value", func() { + So(resTime, ShouldEqual, valueTime.String()) + }) + resOther := InterfaceToString(valueOther) + Convey("resOther should be empty string", func() { + So(resOther, ShouldEqual, "") + }) + }) + +} diff --git a/backend/utils/user_test.go b/backend/utils/user_test.go new file mode 100644 index 00000000..68cf4d65 --- /dev/null +++ b/backend/utils/user_test.go @@ -0,0 +1,14 @@ +package utils + +import ( + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestEncryptPassword(t *testing.T) { + var passwd = "test" + Convey("Test EncryptPassword", t, func() { + res := EncryptPassword(passwd) + t.Log(res) + }) +}