From 44c9bde96f3cf5e5b4243bf086a97e046d5c7698 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 19 Dec 2019 10:42:20 +0800 Subject: [PATCH 1/2] fixed https://github.com/crawlab-team/crawlab/issues/373 --- backend/database/mongo.go | 32 +++++++++++++++++++++++++++++--- backend/model/node.go | 2 +- docker_init.sh | 7 +------ 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/backend/database/mongo.go b/backend/database/mongo.go index e72baeaa..d646285d 100644 --- a/backend/database/mongo.go +++ b/backend/database/mongo.go @@ -61,10 +61,36 @@ func InitMongo() error { dialInfo.Password = mongoPassword dialInfo.Source = mongoAuth } - sess, err := mgo.DialWithInfo(&dialInfo) - if err != nil { - return err + + // mongo session + var sess *mgo.Session + + // 错误次数 + errNum := 0 + + // 重复尝试连接mongo + for { + var err error + + // 连接mongo + sess, err = mgo.DialWithInfo(&dialInfo) + + if err != nil { + // 如果连接错误,休息1秒,错误次数+1 + time.Sleep(1 * time.Second) + errNum++ + + // 如果错误次数超过30,返回错误 + if errNum >= 30 { + return err + } + } else { + // 如果没有错误,退出循环 + break + } } + + // 赋值给全局mongo session Session = sess } return nil diff --git a/backend/model/node.go b/backend/model/node.go index d662ab6d..2fe810f8 100644 --- a/backend/model/node.go +++ b/backend/model/node.go @@ -55,7 +55,7 @@ func GetCurrentNode() (Node, error) { for { // 如果错误次数超过10次 if errNum >= 10 { - panic("cannot get current node") + return node, errors.New("cannot get current node") } // 尝试获取节点 diff --git a/docker_init.sh b/docker_init.sh index 8c256839..97c505dc 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -22,10 +22,5 @@ fi # start nginx service nginx start -# wait for mongo service to be ready -#/app/wait-for-it.sh $CRAWLAB_MONGO_HOST:$CRAWLAB_MONGO_PORT - -# wait for redis service to be ready -#/app/wait-for-it.sh $CRAWLAB_REDIS_ADDRESS:$CRAWLAB_REDIS_PORT - +# start backend crawlab \ No newline at end of file From 72692133eedfee5b5bea21287c2b6b8597c55fee Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 19 Dec 2019 10:45:11 +0800 Subject: [PATCH 2/2] fixed unable to check is page error for configurable spider --- frontend/src/components/Config/ConfigList.vue | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue index 9b274b3b..1e0f7dce 100644 --- a/frontend/src/components/Config/ConfigList.vue +++ b/frontend/src/components/Config/ConfigList.vue @@ -819,12 +819,12 @@ ${f.css || f.xpath} ${f.attr ? ('(' + f.attr + ')') : ''} ${f.next_stage ? (' -- }, onCheckIsList (value, stage) { if (value) { - this.$st('爬虫详情', '配置', '勾选列表页') + this.$st.sendEv('爬虫详情', '配置', '勾选列表页') if (!stage.list_css && !stage.list_xpath) { stage.list_xpath = '//body' } } else { - this.$st('爬虫详情', '配置', '取消勾选列表页') + this.$st.sendEv('爬虫详情', '配置', '取消勾选列表页') stage.list_css = '' stage.list_xpath = '' } @@ -846,12 +846,12 @@ ${f.css || f.xpath} ${f.attr ? ('(' + f.attr + ')') : ''} ${f.next_stage ? (' -- }, onCheckIsPage (value, stage) { if (value) { - this.$st('爬虫详情', '配置', '勾选分页') + this.$st.sendEv('爬虫详情', '配置', '勾选分页') if (!stage.page_css && !stage.page_xpath) { stage.page_xpath = '//body' } } else { - this.$st('爬虫详情', '配置', '取消勾选分页') + this.$st.sendEv('爬虫详情', '配置', '取消勾选分页') stage.page_css = '' stage.page_xpath = '' }