Merge pull request #147 from tikazyq/develop

Develop
2026-01-22 17:31:03 +01:00 · 2019-08-18 22:24:04 +08:00
parent f72d43a4b7 3e8a1dea54
commit 6d83a9fb99
10 changed files with 235 additions and 5 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -119,4 +119,6 @@ logs/
 tmp/
 _book/
 .idea
-*.lock
+*.lock
+
+backend/spiders
--- a/2
+++ b/2
@@ -46,7 +46,7 @@ pipeline {
            steps {
                echo 'Deploying....'
                sh """
-                if [ ${ENV:GIT_BRANCH} -eq master ] || [ ${ENV:GIT_BRANCH} -eq develop ]; then
+                if [[ ${ENV:GIT_BRANCH} = master ]] || [[ ${ENV:GIT_BRANCH} = develop ]]; then
                    # 重启docker compose
                    cd ./jenkins/${ENV:GIT_BRANCH}
                    docker-compose stop | true
--- a/README-zh.md
+++ b/README-zh.md
@@ -4,7 +4,7 @@
 ![](https://img.shields.io/github/release/tikazyq/crawlab.svg)
 ![](https://img.shields.io/github/last-commit/tikazyq/crawlab.svg)
 ![](https://img.shields.io/github/issues/tikazyq/crawlab.svg)
-![](https://img.shields.io/github/contributers/tikazyq/crawlab.svg)
+![](https://img.shields.io/github/contributors/tikazyq/crawlab.svg)
 ![](https://img.shields.io/docker/pulls/tikazyq/crawlab)
 ![](https://img.shields.io/github/license/tikazyq/crawlab.svg)

--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@
 ![](https://img.shields.io/github/release/tikazyq/crawlab.svg)
 ![](https://img.shields.io/github/last-commit/tikazyq/crawlab.svg)
 ![](https://img.shields.io/github/issues/tikazyq/crawlab.svg)
-![](https://img.shields.io/github/contributers/tikazyq/crawlab.svg)
+![](https://img.shields.io/github/contributors/tikazyq/crawlab.svg)
 ![](https://img.shields.io/docker/pulls/tikazyq/crawlab)
 ![](https://img.shields.io/github/license/tikazyq/crawlab.svg)

--- a/backend/go.sum
+++ b/backend/go.sum
@@ -53,6 +53,7 @@ github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -65,6 +66,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
 github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -117,8 +119,10 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -184,6 +188,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
--- a/backend/routes/spider.go
+++ b/backend/routes/spider.go
@@ -131,10 +131,22 @@ func PutSpider(c *gin.Context) {
 		return
 	}

+	// 以防tmp目录不存在
+	tmpPath := viper.GetString("other.tmppath")
+	if !utils.Exists(tmpPath) {
+		if err := os.Mkdir(tmpPath, os.ModePerm); err != nil {
+			log.Error("mkdir other.tmppath dir error:" + err.Error())
+			debug.PrintStack()
+			HandleError(http.StatusBadRequest, c, errors.New("Mkdir other.tmppath dir error"))
+			return
+		}
+	}
+
 	// 保存到本地临时文件
 	randomId := uuid.NewV4()
-	tmpFilePath := filepath.Join(viper.GetString("other.tmppath"), randomId.String()+".zip")
+	tmpFilePath := filepath.Join(tmpPath, randomId.String()+".zip")
 	if err := c.SaveUploadedFile(file, tmpFilePath); err != nil {
+		log.Error("save upload file error: " + err.Error())
 		debug.PrintStack()
 		HandleError(http.StatusInternalServerError, c, err)
 		return
--- a/backend/utils/chan_test.go
+++ b/backend/utils/chan_test.go
@@ -0,0 +1,76 @@
+package utils
+
+import (
+	. "github.com/smartystreets/goconvey/convey"
+	"testing"
+)
+
+func TestNewChanMap(t *testing.T) {
+	mapTest := make(map[string]chan string)
+	chanTest := make(chan string)
+	test := "test"
+
+	Convey("Call NewChanMap to generate ChanMap", t, func() {
+		mapTest[test] = chanTest
+		chanMapTest := ChanMap{mapTest}
+		chanMap := NewChanMap()
+		chanMap.m[test] = chanTest
+
+		Convey(test, func() {
+			So(chanMap, ShouldResemble, &chanMapTest)
+		})
+
+	})
+}
+
+func TestChan(t *testing.T) {
+	mapTest := make(map[string]chan string)
+	chanTest := make(chan string)
+	mapTest["test"] = chanTest
+	chanMapTest := ChanMap{mapTest}
+
+	Convey("Test Chan use exist key", t, func() {
+		ch1 := chanMapTest.Chan(
+			"test")
+		Convey("ch1 should equal chanTest", func() {
+			So(ch1, ShouldEqual, chanTest)
+		})
+
+	})
+	Convey("Test Chan use no-exist key", t, func() {
+		ch2 := chanMapTest.Chan("test2")
+		Convey("ch2 should equal chanMapTest.m[test2]", func() {
+
+			So(chanMapTest.m["test2"], ShouldEqual, ch2)
+		})
+		Convey("Cap of chanMapTest.m[test2] should equal 10", func() {
+			So(10, ShouldEqual, cap(chanMapTest.m["test2"]))
+		})
+	})
+}
+
+func TestChanBlocked(t *testing.T) {
+	mapTest := make(map[string]chan string)
+	chanTest := make(chan string)
+	mapTest["test"] = chanTest
+	chanMapTest := ChanMap{mapTest}
+
+	Convey("Test Chan use exist key", t, func() {
+		ch1 := chanMapTest.ChanBlocked(
+			"test")
+		Convey("ch1 should equal chanTest", func() {
+			So(ch1, ShouldEqual, chanTest)
+		})
+
+	})
+	Convey("Test Chan use no-exist key", t, func() {
+		ch2 := chanMapTest.ChanBlocked("test2")
+		Convey("ch2 should equal chanMapTest.m[test2]", func() {
+
+			So(chanMapTest.m["test2"], ShouldEqual, ch2)
+		})
+		Convey("Cap of chanMapTest.m[test2] should equal 10", func() {
+			So(0, ShouldEqual, cap(chanMapTest.m["test2"]))
+		})
+	})
+}
--- a/backend/utils/file_test.go
+++ b/backend/utils/file_test.go
@@ -0,0 +1,72 @@
+package utils
+
+import (
+	. "github.com/smartystreets/goconvey/convey"
+	"os"
+	"testing"
+)
+
+func TestExists(t *testing.T) {
+	var pathString = "../config"
+	var wrongPathString = "test"
+
+	Convey("Test path or file is Exists or not", t, func() {
+		res := Exists(pathString)
+		Convey("The result should be true", func() {
+			So(res, ShouldEqual, true)
+		})
+		wrongRes := Exists(wrongPathString)
+		Convey("The result should be false", func() {
+			So(wrongRes, ShouldEqual, false)
+		})
+	})
+}
+
+func TestIsDir(t *testing.T) {
+	var pathString = "../config"
+	var fileString = "../config/config.go"
+	var wrongString = "test"
+
+	Convey("Test path is folder or not", t, func() {
+		res := IsDir(pathString)
+		So(res, ShouldEqual, true)
+		fileRes := IsDir(fileString)
+		So(fileRes, ShouldEqual, false)
+		wrongRes := IsDir(wrongString)
+		So(wrongRes, ShouldEqual, false)
+	})
+}
+
+func TestCompress(t *testing.T) {
+	var pathString = "../utils"
+	var files []*os.File
+	var disPath = "../utils/test"
+	file, err := os.Open(pathString)
+	if err != nil {
+		t.Error("open source path failed")
+	}
+	files = append(files, file)
+	Convey("Verify dispath is valid path", t, func() {
+		er := Compress(files, disPath)
+		Convey("err should be nil", func() {
+			So(er, ShouldEqual, nil)
+		})
+	})
+
+}
+
+// 测试之前需存在有效的test(.zip)文件
+func TestDeCompress(t *testing.T) {
+	var tmpFilePath = "./test"
+	tmpFile, err := os.OpenFile(tmpFilePath, os.O_RDONLY, 0777)
+	if err != nil {
+		t.Fatal("open zip file failed")
+	}
+	var dstPath = "./testDeCompress"
+	Convey("Test DeCopmress func", t, func() {
+
+		err := DeCompress(tmpFile, dstPath)
+		So(err, ShouldEqual, nil)
+	})
+
+}
--- a/backend/utils/model_test.go
+++ b/backend/utils/model_test.go
@@ -0,0 +1,49 @@
+package utils
+
+import (
+	"github.com/globalsign/mgo/bson"
+	. "github.com/smartystreets/goconvey/convey"
+	"strconv"
+	"testing"
+	"time"
+)
+
+func TestIsObjectIdNull(t *testing.T) {
+	var id bson.ObjectId = "123455"
+	Convey("Test Object ID is null or not", t, func() {
+		res := IsObjectIdNull(id)
+		So(res, ShouldEqual, false)
+	})
+}
+
+func TestInterfaceToString(t *testing.T) {
+	var valueBson bson.ObjectId = "12345"
+	var valueString = "12345"
+	var valueInt = 12345
+	var valueTime = time.Now().Add(60 * time.Second)
+	var valueOther = []string{"a", "b"}
+
+	Convey("Test InterfaceToString", t, func() {
+		resBson := InterfaceToString(valueBson)
+		Convey("resBson should be string value", func() {
+			So(resBson, ShouldEqual, valueBson.Hex())
+		})
+		resString := InterfaceToString(valueString)
+		Convey("resString should be string value", func() {
+			So(resString, ShouldEqual, valueString)
+		})
+		resInt := InterfaceToString(valueInt)
+		Convey("resInt should be string value", func() {
+			So(resInt, ShouldEqual, strconv.Itoa(valueInt))
+		})
+		resTime := InterfaceToString(valueTime)
+		Convey("resTime should be string value", func() {
+			So(resTime, ShouldEqual, valueTime.String())
+		})
+		resOther := InterfaceToString(valueOther)
+		Convey("resOther should be empty string", func() {
+			So(resOther, ShouldEqual, "")
+		})
+	})
+
+}
--- a/backend/utils/user_test.go
+++ b/backend/utils/user_test.go
@@ -0,0 +1,14 @@
+package utils
+
+import (
+	. "github.com/smartystreets/goconvey/convey"
+	"testing"
+)
+
+func TestEncryptPassword(t *testing.T) {
+	var passwd = "test"
+	Convey("Test EncryptPassword", t, func() {
+		res := EncryptPassword(passwd)
+		t.Log(res)
+	})
+}