From b3a656c77700e475e269cc43b68def19acb5c613 Mon Sep 17 00:00:00 2001 From: yaziming Date: Tue, 1 Oct 2019 09:58:24 +0800 Subject: [PATCH 1/4] fix(backend): fix mongo connect error when password has special characters --- backend/database/mongo.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/backend/database/mongo.go b/backend/database/mongo.go index 1c2d6433..e72baeaa 100644 --- a/backend/database/mongo.go +++ b/backend/database/mongo.go @@ -3,6 +3,7 @@ package database import ( "github.com/globalsign/mgo" "github.com/spf13/viper" + "net" "time" ) @@ -39,13 +40,28 @@ func InitMongo() error { var mongoAuth = viper.GetString("mongo.authSource") if Session == nil { - var uri string - if mongoUsername == "" { - uri = "mongodb://" + mongoHost + ":" + mongoPort + "/" + mongoDb - } else { - uri = "mongodb://" + mongoUsername + ":" + mongoPassword + "@" + mongoHost + ":" + mongoPort + "/" + mongoDb + "?authSource=" + mongoAuth + var dialInfo mgo.DialInfo + addr := net.JoinHostPort(mongoHost, mongoPort) + timeout := time.Second * 10 + dialInfo = mgo.DialInfo{ + Addrs: []string{addr}, + Timeout: timeout, + Database: mongoDb, + PoolLimit: 100, + PoolTimeout: timeout, + ReadTimeout: timeout, + WriteTimeout: timeout, + AppName: "crawlab", + FailFast: true, + MinPoolSize: 10, + MaxIdleTimeMS: 1000 * 30, } - sess, err := mgo.DialWithTimeout(uri, time.Second*5) + if mongoUsername != "" { + dialInfo.Username = mongoUsername + dialInfo.Password = mongoPassword + dialInfo.Source = mongoAuth + } + sess, err := mgo.DialWithInfo(&dialInfo) if err != nil { return err } From d824a77c385b1d21d47c329a26c50bc905e5e7ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sun, 6 Oct 2019 18:46:51 +0800 Subject: [PATCH 2/4] add k8s deploy link --- README-zh.md | 1 + README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/README-zh.md b/README-zh.md index 6c2449f2..b4e2b469 100644 --- a/README-zh.md +++ b/README-zh.md @@ -21,6 +21,7 @@ 三种方式: 1. [Docker](https://tikazyq.github.io/crawlab-docs/Installation/Docker.html)（推荐） 2. [直接部署](https://tikazyq.github.io/crawlab-docs/Installation/Direct.html)（了解内核） +3. [Kubernetes](https://mp.weixin.qq.com/s/3Q1BQATUIEE_WXcHPqhYbA) ### 要求（Docker） - Docker 18.03+ diff --git a/README.md b/README.md index 9ae17a77..91a30b34 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Golang-based distributed web crawler management platform, supporting various lan Two methods: 1. [Docker](https://tikazyq.github.io/crawlab-docs/Installation/Docker.html) (Recommended) 2. [Direct Deploy](https://tikazyq.github.io/crawlab-docs/Installation/Direct.html) (Check Internal Kernel) +3. [Kubernetes](https://mp.weixin.qq.com/s/3Q1BQATUIEE_WXcHPqhYbA) ### Pre-requisite (Docker) - Docker 18.03+ From 0d10d6d045db392a0164693bda0ff115d4583333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 7 Oct 2019 12:21:32 +0800 Subject: [PATCH 3/4] =?UTF-8?q?fix=20=E5=88=9B=E5=BB=BA=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/routes/spider.go | 2 +- backend/services/spider.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/routes/spider.go b/backend/routes/spider.go index addddd99..4c26fcee 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -135,7 +135,7 @@ func PutSpider(c *gin.Context) { // 以防tmp目录不存在 tmpPath := viper.GetString("other.tmppath") if !utils.Exists(tmpPath) { - if err := os.Mkdir(tmpPath, os.ModePerm); err != nil { + if err := os.MkdirAll(tmpPath, os.ModePerm); err != nil { log.Error("mkdir other.tmppath dir error:" + err.Error()) debug.PrintStack() HandleError(http.StatusBadRequest, c, errors.New("Mkdir other.tmppath dir error")) diff --git a/backend/services/spider.go b/backend/services/spider.go index a2e9a60f..c03ebe38 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -145,6 +145,7 @@ func PublishSpider(spider model.Spider) { // md5值不一样，则下载 md5Str := utils.ReadFileOneLine(md5) if gfFile.Md5 != md5Str { + log.Infof("md5 is different, gf-md5:%s, file-md5:%s", gfFile.Md5, md5Str) spiderSync.RemoveSpiderFile() spiderSync.Download() spiderSync.CreateMd5File(gfFile.Md5) From 7c75c24f65add1fc3987af3b5c509b068ed9240d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 7 Oct 2019 12:49:37 +0800 Subject: [PATCH 4/4] =?UTF-8?q?fix=20md5=E5=80=BC=E4=B8=8D=E4=B8=80?= =?UTF-8?q?=E8=87=B4=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/spider.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/services/spider.go b/backend/services/spider.go index c03ebe38..7aea456f 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -16,6 +16,7 @@ import ( "os" "path/filepath" "runtime/debug" + "strings" ) type SpiderFileData struct { @@ -144,6 +145,9 @@ func PublishSpider(spider model.Spider) { } // md5值不一样，则下载 md5Str := utils.ReadFileOneLine(md5) + // 去掉空格以及换行符 + md5Str = strings.Replace(md5Str, " ", "", -1) + md5Str = strings.Replace(md5Str, "\n", "", -1) if gfFile.Md5 != md5Str { log.Infof("md5 is different, gf-md5:%s, file-md5:%s", gfFile.Md5, md5Str) spiderSync.RemoveSpiderFile()