From d701e235580ece3bc0f55fbc218f52bab6ca81ab Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 31 Jul 2019 13:29:56 +0800 Subject: [PATCH 01/10] updated Jenkinsfile --- Dockerfile.local | 1 - Jenkinsfile | 10 ++-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/Dockerfile.local b/Dockerfile.local index c08719aa..fc10b586 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,4 +1,3 @@ - FROM golang:1.12 AS backend-build WORKDIR /go/src/app diff --git a/Jenkinsfile b/Jenkinsfile index b22488cc..af9d6a13 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,7 +24,7 @@ pipeline { steps { echo "Building..." sh """ - docker build -t tikazyq/crawlab:latest . + docker build -t tikazyq/crawlab:latest -f Dockerfile.local . """ } } @@ -37,13 +37,7 @@ pipeline { steps { echo 'Deploying....' sh """ - docker rm -f crawlab | true - docker run -d --rm --name crawlab \ - -p 8080:8080 \ - -p 8000:8000 \ - -v /home/yeqing/.env.production:/opt/crawlab/frontend/.env.production \ - -v /home/yeqing/config.py:/opt/crawlab/crawlab/config/config.py \ - tikazyq/crawlab master + docker-compose up """ } } From f6028542b7f6931bfcba5252e402a03674523fdd Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 31 Jul 2019 13:34:22 +0800 Subject: [PATCH 02/10] updated Jenkinsfile --- Dockerfile.local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.local b/Dockerfile.local index fc10b586..ed4e7e96 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -8,7 +8,7 @@ ENV GOPROXY https://mirrors.aliyun.com/goproxy/ RUN go install -v ./... -FROM node:8.16.0-alpine AS frontend-build +FROM node:8.16.0 AS frontend-build ADD ./frontend /app WORKDIR /app From 34ca8cbd8f3185f171fb1b0866cbb11266106093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Wed, 31 Jul 2019 14:35:36 +0800 Subject: [PATCH 03/10] =?UTF-8?q?fix=20redis=E8=BF=9E=E6=8E=A5=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/conf/config.yml | 6 ++++-- backend/database/redis.go | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 3e7aa234..e9f29b3c 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -7,8 +7,10 @@ mongo: username: "" password: "" redis: - network: tcp - address: "localhost:6379" + address: 127.0.0.1 + password: redis-1.0 + database: 1 + port: 6379 log: level: info path: "/var/logs/crawlab" diff --git a/backend/database/redis.go b/backend/database/redis.go index 17382122..ffebf776 100644 --- a/backend/database/redis.go +++ b/backend/database/redis.go @@ -103,10 +103,18 @@ func (r *Redis) HKeys(collection string) ([]string, error) { } func GetRedisConn() (redis.Conn, error) { - c, err := redis.Dial( - viper.GetString("redis.network"), - viper.GetString("redis.address"), - ) + var address = viper.GetString("redis.address") + var port = viper.GetString("redis.port") + var database = viper.GetString("redis.database") + var password = viper.GetString("redis.password") + + var url string + if password == "" { + url = "redis://" + address + ":" + port + "/" + database + } else { + url = "redis://x:" + password + "@" + address + ":" + port + "/" + database + } + c, err := redis.DialURL(url) if err != nil { debug.PrintStack() return c, err From c2d1debdad9027b8977afb0c3969117f13c27bb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Wed, 31 Jul 2019 16:03:33 +0800 Subject: [PATCH 04/10] =?UTF-8?q?update=20=E9=85=8D=E7=BD=AE=E6=96=87?= =?UTF-8?q?=E4=BB=B6Redis=E5=88=9D=E5=A7=8B=E5=AF=86=E7=A0=81=E4=B8=BA?= =?UTF-8?q?=E7=A9=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/conf/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/conf/config.yml b/backend/conf/config.yml index e9f29b3c..820ff077 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -8,7 +8,7 @@ mongo: password: "" redis: address: 127.0.0.1 - password: redis-1.0 + password: "" database: 1 port: 6379 log: From d855fb5114149c69ae935ad49dcc4892d51b1710 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 31 Jul 2019 17:01:17 +0800 Subject: [PATCH 05/10] updated Jenkinsfile --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index af9d6a13..af8f4768 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,6 +37,7 @@ pipeline { steps { echo 'Deploying....' sh """ + docker-compose stop | true docker-compose up """ } From 3087e925a7d7950569d35829bd7eaf015fa5c29a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 31 Jul 2019 17:13:48 +0800 Subject: [PATCH 06/10] updated Jenkinsfile --- Jenkinsfile | 2 +- docker-compose.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index af8f4768..4e3f86cb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,7 +38,7 @@ pipeline { echo 'Deploying....' sh """ docker-compose stop | true - docker-compose up + docker-compose up -d """ } } diff --git a/docker-compose.yml b/docker-compose.yml index 8b7019f9..536b7d6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,8 @@ services: CRAWLAB_SERVER_MASTER: "Y" CRAWLAB_MONGO_HOST: "mongo" CRAWLAB_REDIS_ADDRESS: "redis:6379" + networks: + - host ports: - "8080:8080" # frontend - "8000:8000" # backend From 682a64d7517bfbbfbb47b75dc3b3b387937e50a4 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 31 Jul 2019 19:38:03 +0800 Subject: [PATCH 07/10] updated Jenkinsfile --- Jenkinsfile | 3 ++- README-zh.md | 18 ++++++++++++++---- docker-compose.yml | 12 +++++------- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4e3f86cb..64d123f8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,7 @@ pipeline { echo "Running Setup..." script { if (env.GIT_BRANCH == 'develop') { - env.MODE = 'test' + env.MODE = 'develop' } else if (env.GIT_BRANCH == 'master') { env.MODE = 'production' } else { @@ -37,6 +37,7 @@ pipeline { steps { echo 'Deploying....' sh """ + cd ./jenkins docker-compose stop | true docker-compose up -d """ diff --git a/README-zh.md b/README-zh.md index 0b4d9849..49877554 100644 --- a/README-zh.md +++ b/README-zh.md @@ -50,25 +50,35 @@ docker run -d --rm --name crawlab \ tikazyq/crawlab:0.3.0 ``` -当然也可以用`docker-compose`来一键启动，甚至不用配置MongoDB和Redis数据库。在当前目录中创建`docker-compose.yml`文件，输入以下内容。 +当然也可以用`docker-compose`来一键启动，甚至不用配置MongoDB和Redis数据库，**当然我们推荐这样做**。在当前目录中创建`docker-compose.yml`文件，输入以下内容。 ```bash version: '3.3' services: master: image: tikazyq/crawlab:latest - container_name: crawlab-master + container_name: master environment: - CRAWLAB_API_ADDRESS: "192.168.99.100:8000" + CRAWLAB_API_ADDRESS: "localhost:8000" CRAWLAB_SERVER_MASTER: "Y" CRAWLAB_MONGO_HOST: "mongo" - CRAWLAB_REDIS_ADDRESS: "redis:6379" + CRAWLAB_REDIS_ADDRESS: "redis" ports: - "8080:8080" # frontend - "8000:8000" # backend depends_on: - mongo - redis + worker: + image: tikazyq/crawlab:latest + container_name: worker + environment: + CRAWLAB_SERVER_MASTER: "N" + CRAWLAB_MONGO_HOST: "mongo" + CRAWLAB_REDIS_ADDRESS: "redis" + depends_on: + - mongo + - redis mongo: image: mongo:latest restart: always diff --git a/docker-compose.yml b/docker-compose.yml index 536b7d6b..d44d5024 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,14 +2,12 @@ version: '3.3' services: master: image: tikazyq/crawlab:latest - container_name: crawlab-master + container_name: master environment: - CRAWLAB_API_ADDRESS: "192.168.99.100:8000" + CRAWLAB_API_ADDRESS: "localhost:8000" CRAWLAB_SERVER_MASTER: "Y" CRAWLAB_MONGO_HOST: "mongo" - CRAWLAB_REDIS_ADDRESS: "redis:6379" - networks: - - host + CRAWLAB_REDIS_ADDRESS: "redis" ports: - "8080:8080" # frontend - "8000:8000" # backend @@ -18,11 +16,11 @@ services: - redis worker: image: tikazyq/crawlab:latest - container_name: crawlab-worker + container_name: worker environment: CRAWLAB_SERVER_MASTER: "N" CRAWLAB_MONGO_HOST: "mongo" - CRAWLAB_REDIS_ADDRESS: "redis:6379" + CRAWLAB_REDIS_ADDRESS: "redis" depends_on: - mongo - redis From 218d8c5db3afb36e2fe3577bb08c10caf4a67149 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 31 Jul 2019 19:45:43 +0800 Subject: [PATCH 08/10] updated config.yml --- README-zh.md | 10 ---------- README.md | 6 +++--- backend/conf/config.yml | 2 +- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/README-zh.md b/README-zh.md index 49877554..bcb0c75e 100644 --- a/README-zh.md +++ b/README-zh.md @@ -69,16 +69,6 @@ services: depends_on: - mongo - redis - worker: - image: tikazyq/crawlab:latest - container_name: worker - environment: - CRAWLAB_SERVER_MASTER: "N" - CRAWLAB_MONGO_HOST: "mongo" - CRAWLAB_REDIS_ADDRESS: "redis" - depends_on: - - mongo - - redis mongo: image: mongo:latest restart: always diff --git a/README.md b/README.md index 8da20e5a..50c289b1 100644 --- a/README.md +++ b/README.md @@ -58,12 +58,12 @@ version: '3.3' services: master: image: tikazyq/crawlab:latest - container_name: crawlab-master + container_name: master environment: - CRAWLAB_API_ADDRESS: "192.168.99.100:8000" + CRAWLAB_API_ADDRESS: "localhost:8000" CRAWLAB_SERVER_MASTER: "Y" CRAWLAB_MONGO_HOST: "mongo" - CRAWLAB_REDIS_ADDRESS: "redis:6379" + CRAWLAB_REDIS_ADDRESS: "redis" ports: - "8080:8080" # frontend - "8000:8000" # backend diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 820ff077..30b7e786 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -7,7 +7,7 @@ mongo: username: "" password: "" redis: - address: 127.0.0.1 + address: localhost password: "" database: 1 port: 6379 From b963512b2c212cfd54a3cf5aee3e76939688f2f7 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 31 Jul 2019 19:46:55 +0800 Subject: [PATCH 09/10] added jenkins --- jenkins/docker-compose.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 jenkins/docker-compose.yml diff --git a/jenkins/docker-compose.yml b/jenkins/docker-compose.yml new file mode 100644 index 00000000..57c85e1e --- /dev/null +++ b/jenkins/docker-compose.yml @@ -0,0 +1,36 @@ +version: '3.3' +services: + master: + image: tikazyq/crawlab:latest + container_name: master + environment: + CRAWLAB_API_ADDRESS: "114.67.75.98:8000" + CRAWLAB_SERVER_MASTER: "Y" + CRAWLAB_MONGO_HOST: "mongo" + CRAWLAB_REDIS_ADDRESS: "redis" + ports: + - "8080:8080" # frontend + - "8000:8000" # backend + depends_on: + - mongo + - redis + worker: + image: tikazyq/crawlab:latest + container_name: worker + environment: + CRAWLAB_SERVER_MASTER: "N" + CRAWLAB_MONGO_HOST: "mongo" + CRAWLAB_REDIS_ADDRESS: "redis" + depends_on: + - mongo + - redis + mongo: + image: mongo:latest + restart: always + ports: + - "27017:27017" + redis: + image: redis:latest + restart: always + ports: + - "6379:6379" \ No newline at end of file From 6e1a70124e5e2a8de5c176a2603c06d905e71a76 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 31 Jul 2019 20:06:37 +0800 Subject: [PATCH 10/10] added jenkins badge --- README-zh.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README-zh.md b/README-zh.md index bcb0c75e..5203cd66 100644 --- a/README-zh.md +++ b/README-zh.md @@ -1,6 +1,6 @@ # Crawlab -![](http://114.67.75.98:8081/buildStatus/icon?job=crawlab%2Fdevelop) +![](http://114.67.75.98:8082/buildStatus/icon?job=crawlab%2Fdevelop) ![](https://img.shields.io/badge/版本-v0.3.0-blue.svg) diff --git a/README.md b/README.md index 50c289b1..fa3c5625 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Crawlab -![](http://114.67.75.98:8081/buildStatus/icon?job=crawlab%2Fdevelop) +![](http://114.67.75.98:8082/buildStatus/icon?job=crawlab%2Fdevelop) ![](https://img.shields.io/badge/version-v0.3.0-blue.svg)