From 11961c2999c00bd74a52c97cf9d88b762230d28e Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 9 Jun 2022 21:24:34 +0800 Subject: [PATCH 01/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 8 ++++++-- bin/docker-init.sh | 12 +++++------ bin/docker-start-master.sh | 41 +++++++++++++++++++------------------- 4 files changed, 34 insertions(+), 29 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index 48f0d69e..9ec7fdcb 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.0-7 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220609124710-a6fcaa2e7ea6 diff --git a/backend/go.sum b/backend/go.sum index 57afa5db..e884d1a2 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -122,8 +122,12 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k= -github.com/crawlab-team/crawlab-core v0.6.0-7 h1:23kTlKOZ3OqfSmD+SYJiwlqP1Cu1NJtPmbR8KeIGvgw= -github.com/crawlab-team/crawlab-core v0.6.0-7/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.0-8 h1:P1l8dBdecZ6vVxwWwOwjLtCUriSNyBAe01Ghwfndoqw= +github.com/crawlab-team/crawlab-core v0.6.0-8/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.0-9 h1:oBwtAzafOVl2r2vLbLYClgpXDViwVVzqdXnbwC2aGj4= +github.com/crawlab-team/crawlab-core v0.6.0-9/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220609124710-a6fcaa2e7ea6 h1:DGXTxMCJhLZkbBZcwHE6wisUO96nsy5rTlV7e9pQ+Kg= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220609124710-a6fcaa2e7ea6/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= diff --git a/bin/docker-init.sh b/bin/docker-init.sh index f15755db..170dabbf 100755 --- a/bin/docker-init.sh +++ b/bin/docker-init.sh @@ -1,12 +1,12 @@ #!/bin/bash if [ "${CRAWLAB_NODE_MASTER}" = "Y" ]; then - # start master - /bin/bash /app/bin/docker-start-master.sh + # start master + /bin/bash /app/bin/docker-start-master.sh - # start crawlab - crawlab-server master + # start crawlab + crawlab-server master else - # start crawlab - crawlab-server worker + # start crawlab + crawlab-server worker fi diff --git a/bin/docker-start-master.sh b/bin/docker-start-master.sh index fd271eec..b6bd84ff 100755 --- a/bin/docker-start-master.sh +++ b/bin/docker-start-master.sh @@ -1,23 +1,25 @@ +#!/bin/bash + # replace env indexpath=/app/dist/index.html if test -z "$CRAWLAB_BASE_URL"; then - : + : else - sed -i "s?/js/?${CRAWLAB_BASE_URL}/js/?g" ${indexpath} - sed -i "s?/css/?${CRAWLAB_BASE_URL}/css/?g" ${indexpath} - sed -i "s/ > /var/log/weed.log 2>&1 & - + -dir /data \ + -master.dir ${seaweedfsDataPath} \ + -volume.dir.idx ${seaweedfsDataPath} \ + -ip localhost \ + -volume.port 9999 \ + -filer \ + >>/var/log/weed.log 2>&1 & From a15997f96a634aacd9e3229ec51438a1ac127744 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 9 Jun 2022 21:40:40 +0800 Subject: [PATCH 02/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index 9ec7fdcb..663d9c09 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220609124710-a6fcaa2e7ea6 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220609133021-540f1f228a89 diff --git a/backend/go.sum b/backend/go.sum index e884d1a2..f5ee9775 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -128,6 +128,8 @@ github.com/crawlab-team/crawlab-core v0.6.0-9 h1:oBwtAzafOVl2r2vLbLYClgpXDViwVVz github.com/crawlab-team/crawlab-core v0.6.0-9/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220609124710-a6fcaa2e7ea6 h1:DGXTxMCJhLZkbBZcwHE6wisUO96nsy5rTlV7e9pQ+Kg= github.com/crawlab-team/crawlab-core v0.6.1-0.20220609124710-a6fcaa2e7ea6/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220609133021-540f1f228a89 h1:8/9AsqjaWN19379M6Pr/KXHxvRKWOE/S7SbbpzrYpIc= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220609133021-540f1f228a89/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From 8a5f30ade7683a890da7f4d373ea562968360c07 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 9 Jun 2022 21:53:30 +0800 Subject: [PATCH 03/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index 663d9c09..a4d3e602 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220609133021-540f1f228a89 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220609135233-fed5b2972049 diff --git a/backend/go.sum b/backend/go.sum index f5ee9775..d991d774 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -130,6 +130,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220609124710-a6fcaa2e7ea6 h1:DGX github.com/crawlab-team/crawlab-core v0.6.1-0.20220609124710-a6fcaa2e7ea6/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220609133021-540f1f228a89 h1:8/9AsqjaWN19379M6Pr/KXHxvRKWOE/S7SbbpzrYpIc= github.com/crawlab-team/crawlab-core v0.6.1-0.20220609133021-540f1f228a89/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220609135233-fed5b2972049 h1:UDkyU63T8E/vvD2cc58d7YPKnH64w6qdlyi1MnwIWO8= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220609135233-fed5b2972049/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From dbfc83a28a69613e05d3fee531447fe263cecab5 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 20:49:57 +0800 Subject: [PATCH 04/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index fd336c54..4a78184d 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -133,11 +133,14 @@ jobs: sed -i "s/${IMAGE_NAME}:latest/${IMAGE_NAME}:${IMAGE_VERSION}/" Dockerfile done fi - - name: Build image run: docker build . --file Dockerfile --tag image - name: Log into registry run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + - name: Test + run: | + sleep 15 + curl http://localhost:8080/api/system-info - name: Push image run: | IMAGE_VERSION=${{needs.setup.outputs.version}} From 1c7106d6db831cb7d2b5c8fd7e8e0064096a4603 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 20:50:08 +0800 Subject: [PATCH 05/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 4a78184d..0bd7d961 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -137,7 +137,7 @@ jobs: run: docker build . --file Dockerfile --tag image - name: Log into registry run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - - name: Test + - name: Test image run: | sleep 15 curl http://localhost:8080/api/system-info From 94056d2c2ebe3fd7aecba2e67c5c948cbb7481b7 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 20:54:04 +0800 Subject: [PATCH 06/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 4 +- docker-compose.local.yml | 85 ---------------------------- docker-compose.yml | 32 ++--------- 3 files changed, 8 insertions(+), 113 deletions(-) delete mode 100644 docker-compose.local.yml diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 0bd7d961..e1c7bee6 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -139,8 +139,10 @@ jobs: run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - name: Test image run: | - sleep 15 + docker run -d --name crawlab_master --rm image + sleep 10 curl http://localhost:8080/api/system-info + docker stop crawlab_master - name: Push image run: | IMAGE_VERSION=${{needs.setup.outputs.version}} diff --git a/docker-compose.local.yml b/docker-compose.local.yml deleted file mode 100644 index 96def475..00000000 --- a/docker-compose.local.yml +++ /dev/null @@ -1,85 +0,0 @@ -version: '3.3' -services: - master: - build: - context: ./ - dockerfile: Dockerfile.local - container_name: local_master - environment: - # CRAWLAB_API_ADDRESS: "https://:" # backend API address 后端 API 地址. 适用于 https 或者源码部署 - CRAWLAB_SERVER_MASTER: "Y" # whether to be master node 是否为主节点，主节点为 Y，工作节点为 N - CRAWLAB_MONGO_HOST: "mongo" # MongoDB host address MongoDB 的地址，在 docker compose 网络中，直接引用服务名称 - # CRAWLAB_MONGO_PORT: "27017" # MongoDB port MongoDB 的端口 - # CRAWLAB_MONGO_DB: "crawlab_test" # MongoDB database MongoDB 的数据库 - # CRAWLAB_MONGO_USERNAME: "username" # MongoDB username MongoDB 的用户名 - # CRAWLAB_MONGO_PASSWORD: "password" # MongoDB password MongoDB 的密码 - # CRAWLAB_MONGO_AUTHSOURCE: "admin" # MongoDB auth source MongoDB 的验证源 - CRAWLAB_REDIS_ADDRESS: "redis" # Redis host address Redis 的地址，在 docker compose 网络中，直接引用服务名称 - # CRAWLAB_REDIS_PORT: "6379" # Redis port Redis 的端口 - # CRAWLAB_REDIS_DATABASE: "1" # Redis database Redis 的数据库 - # CRAWLAB_REDIS_PASSWORD: "password" # Redis password Redis 的密码 - # CRAWLAB_LOG_LEVEL: "info" # log level 日志级别. 默认为 info - # CRAWLAB_LOG_ISDELETEPERIODICALLY: "N" # whether to periodically delete log files 是否周期性删除日志文件. 默认不删除 - # CRAWLAB_LOG_DELETEFREQUENCY: "@hourly" # frequency of deleting log files 删除日志文件的频率. 默认为每小时 - # CRAWLAB_TASK_WORKERS: 8 # number of task executors 任务执行器个数（并行执行任务数） - # CRAWLAB_SERVER_REGISTER_TYPE: "mac" # node register type 节点注册方式. 默认为 mac 地址，也可设置为 ip（防止 mac 地址冲突） - # CRAWLAB_SERVER_REGISTER_IP: "127.0.0.1" # node register ip 节点注册IP. 节点唯一识别号，只有当 CRAWLAB_SERVER_REGISTER_TYPE 为 "ip" 时才生效 - # CRAWLAB_SERVER_LANG_NODE: "Y" # whether to pre-install Node.js 预安装 Node.js 语言环境 - # CRAWLAB_SERVER_LANG_JAVA: "Y" # whether to pre-install Java 预安装 Java 语言环境 - # CRAWLAB_SERVER_LANG_DOTNET: "Y" # whether to pre-install .Net core 预安装 .Net Core 语言环境 - # CRAWLAB_SERVER_LANG_PHP: "Y" # whether to pre-install PHP 预安装 PHP 语言环境 - # CRAWLAB_SETTING_ALLOWREGISTER: "N" # whether to allow user registration 是否允许用户注册 - # CRAWLAB_SETTING_ENABLETUTORIAL: "N" # whether to enable tutorial 是否启用教程 - # CRAWLAB_SETTING_RUNONMASTER: "N" # whether to run on master node 是否在主节点上运行任务 - # CRAWLAB_SETTING_DEMOSPIDERS: "Y" # whether to init demo spiders 是否使用Demo爬虫 - # CRAWLAB_SETTING_CHECKSCRAPY: "Y" # whether to automatically check if the spider is scrapy 是否自动检测爬虫为scrapy - # CRAWLAB_NOTIFICATION_MAIL_SERVER: smtp.exmaple.com # STMP server address STMP 服务器地址 - # CRAWLAB_NOTIFICATION_MAIL_PORT: 465 # STMP server port STMP 服务器端口 - # CRAWLAB_NOTIFICATION_MAIL_SENDEREMAIL: admin@exmaple.com # sender email 发送者邮箱 - # CRAWLAB_NOTIFICATION_MAIL_SENDERIDENTITY: admin@exmaple.com # sender ID 发送者 ID - # CRAWLAB_NOTIFICATION_MAIL_SMTP_USER: username # SMTP username SMTP 用户名 - # CRAWLAB_NOTIFICATION_MAIL_SMTP_PASSWORD: password # SMTP password SMTP 密码 - ports: - - "8080:8080" # frontend port mapping 前端端口映射 - depends_on: - - mongo - - redis - # volumes: - # - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 - worker: - build: - context: ./ - dockerfile: Dockerfile.local - container_name: local_master - environment: - CRAWLAB_SERVER_MASTER: "N" - CRAWLAB_MONGO_HOST: "mongo" - CRAWLAB_REDIS_ADDRESS: "redis" - depends_on: - - mongo - - redis - # environment: - # MONGO_INITDB_ROOT_USERNAME: username - # MONGO_INITDB_ROOT_PASSWORD: password - # volumes: - # - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 - mongo: - image: mongo:latest - restart: always - # volumes: - # - "/opt/crawlab/mongo/data/db:/data/db" # make data persistent 持久化 - # ports: - # - "27017:27017" # expose port to host machine 暴露接口到宿主机 - redis: - image: redis:latest - restart: always - # command: redis-server --requirepass "password" # set redis password 设置 Redis 密码 - # volumes: - # - "/opt/crawlab/redis/data:/data" # make data persistent 持久化 - # ports: - # - "6379:6379" # expose port to host machine 暴露接口到宿主机 - # splash: # use Splash to run spiders on dynamic pages - # image: scrapinghub/splash - # container_name: splash - # ports: - # - "8050:8050" diff --git a/docker-compose.yml b/docker-compose.yml index cad1a694..1c5d779e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,36 +1,14 @@ version: '3.3' services: master: - image: crawlabteam/crawlab:latest + image: crawlabteam/crawlab container_name: crawlab_master environment: - CRAWLAB_BASE_URL: crawlab - CRAWLAB_SERVER_MASTER: Y - CRAWLAB_MONGO_HOST: mongo - ports: - - "8080:8080" # frontend port mapping 前端端口映射 - depends_on: - - mongo - # volumes: - # - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 - worker: - image: crawlabteam/crawlab:latest - container_name: worker - environment: - CRAWLAB_SERVER_MASTER: "N" + CRAWLAB_NODE_MASTER: "Y" CRAWLAB_MONGO_HOST: "mongo" - # CRAWLAB_REDIS_ADDRESS: "redis" + ports: + - "8080:8080" depends_on: - mongo - # volumes: - # - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 mongo: - image: mongo:latest - #restart: always - # environment: - # MONGO_INITDB_ROOT_USERNAME: username - # MONGO_INITDB_ROOT_PASSWORD: password - # volumes: - # - "/opt/crawlab/mongo/data/db:/data/db" # make data persistent 持久化 - # ports: - # - "27017:27017" # expose port to host machine 暴露接口到宿主机 + image: mongo:4.2 \ No newline at end of file From 5f28d82515779e9b138cd50aa6a21e38fb1dd13c Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 21:06:59 +0800 Subject: [PATCH 07/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index e1c7bee6..a5ffc3c3 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -141,6 +141,7 @@ jobs: run: | docker run -d --name crawlab_master --rm image sleep 10 + docker ps curl http://localhost:8080/api/system-info docker stop crawlab_master - name: Push image From 07eba269a23e0b67e4d4b463a8e86747fa8b3f5f Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 21:07:15 +0800 Subject: [PATCH 08/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index a5ffc3c3..c7a20763 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -143,6 +143,7 @@ jobs: sleep 10 docker ps curl http://localhost:8080/api/system-info + docker logs crawlab_master docker stop crawlab_master - name: Push image run: | From 8813cd9260b12a5e86e61038a180a8fbae427d16 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 21:07:19 +0800 Subject: [PATCH 09/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index c7a20763..9e813902 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -141,8 +141,8 @@ jobs: run: | docker run -d --name crawlab_master --rm image sleep 10 - docker ps curl http://localhost:8080/api/system-info + docker ps docker logs crawlab_master docker stop crawlab_master - name: Push image From 2c863fee8388854721d2e2fa3b585693353c93ec Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 21:20:28 +0800 Subject: [PATCH 10/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index a4d3e602..c8cee481 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220609135233-fed5b2972049 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220610131302-39654de64d31 diff --git a/backend/go.sum b/backend/go.sum index d991d774..ca361169 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -132,6 +132,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220609133021-540f1f228a89 h1:8/9 github.com/crawlab-team/crawlab-core v0.6.1-0.20220609133021-540f1f228a89/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220609135233-fed5b2972049 h1:UDkyU63T8E/vvD2cc58d7YPKnH64w6qdlyi1MnwIWO8= github.com/crawlab-team/crawlab-core v0.6.1-0.20220609135233-fed5b2972049/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220610131302-39654de64d31 h1:Fws59e6GrWl3J4VcRJt4dBv0MIvXnzLoRt4ZA1xQ08Y= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220610131302-39654de64d31/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From f67fe03f2c56bdc97faff04496811225279e7cf1 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 22:05:34 +0800 Subject: [PATCH 11/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 9e813902..dbd40ade 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -141,9 +141,9 @@ jobs: run: | docker run -d --name crawlab_master --rm image sleep 10 - curl http://localhost:8080/api/system-info docker ps docker logs crawlab_master + curl http://localhost:8080/api/system-info docker stop crawlab_master - name: Push image run: | From 9da3e65c525fac8ad7e46b67b41601d9d29d103f Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 22:10:36 +0800 Subject: [PATCH 12/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index dbd40ade..9feccc34 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -122,6 +122,11 @@ jobs: if: ${{ always() }} needs: [ setup, build-backend, build-frontend ] runs-on: ubuntu-latest + services: + mongo: + image: mongo:4.2 + ports: + - 27017:27017 steps: - uses: actions/checkout@v2 - name: Update Dockerfile @@ -139,8 +144,8 @@ jobs: run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - name: Test image run: | - docker run -d --name crawlab_master --rm image - sleep 10 + docker run -d --name crawlab_master --rm image -e CRAWLAB_NODE_MASTER=Y + sleep 30 docker ps docker logs crawlab_master curl http://localhost:8080/api/system-info From 9ba4ccaf802df054d858ab15fa2b61da87435006 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 22:10:48 +0800 Subject: [PATCH 13/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 9feccc34..f48cad98 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -145,7 +145,7 @@ jobs: - name: Test image run: | docker run -d --name crawlab_master --rm image -e CRAWLAB_NODE_MASTER=Y - sleep 30 + sleep 10 docker ps docker logs crawlab_master curl http://localhost:8080/api/system-info From b80271982e2324559b38375eb0aa10edeef8c290 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 22:13:47 +0800 Subject: [PATCH 14/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index f48cad98..975901ad 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -144,7 +144,7 @@ jobs: run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - name: Test image run: | - docker run -d --name crawlab_master --rm image -e CRAWLAB_NODE_MASTER=Y + docker run -d --name crawlab_master --rm -e CRAWLAB_NODE_MASTER=Y -p 8080:8080 image sleep 10 docker ps docker logs crawlab_master From 154e6fb5122dbafe69fbce25fc283314e0800cc5 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 22:17:44 +0800 Subject: [PATCH 15/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 975901ad..c1ea8f35 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -138,18 +138,25 @@ jobs: sed -i "s/${IMAGE_NAME}:latest/${IMAGE_NAME}:${IMAGE_VERSION}/" Dockerfile done fi + - name: Build image run: docker build . --file Dockerfile --tag image - - name: Log into registry - run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + - name: Test image run: | - docker run -d --name crawlab_master --rm -e CRAWLAB_NODE_MASTER=Y -p 8080:8080 image - sleep 10 + docker run -d --rm --name crawlab_master \ + -e CRAWLAB_NODE_MASTER=Y \ + -p 8080:8080 \ + image + sleep 30 docker ps docker logs crawlab_master curl http://localhost:8080/api/system-info docker stop crawlab_master + + - name: Log into registry + run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + - name: Push image run: | IMAGE_VERSION=${{needs.setup.outputs.version}} From e6a75bc5a4bf8c3d5cb5b47a84792143358d3b4f Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 22:21:55 +0800 Subject: [PATCH 16/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index c1ea8f35..0c4a1888 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -151,7 +151,12 @@ jobs: sleep 30 docker ps docker logs crawlab_master - curl http://localhost:8080/api/system-info + res=`curl http://localhost:8080/api/system-info -s` + if [[ $res =~ "success" ]]; then + : + else + exit 1 + fi docker stop crawlab_master - name: Log into registry From 1f889e380cb85791a1ac395eaba64654181959ee Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 22:25:37 +0800 Subject: [PATCH 17/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 0c4a1888..9a0dff0a 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -146,12 +146,12 @@ jobs: run: | docker run -d --rm --name crawlab_master \ -e CRAWLAB_NODE_MASTER=Y \ - -p 8080:8080 \ image sleep 30 docker ps docker logs crawlab_master - res=`curl http://localhost:8080/api/system-info -s` + res=`docker exec crawlab_master curl http://localhost:8000/system-info -s` + echo $res if [[ $res =~ "success" ]]; then : else From 8a0b7bf8e8aadde1ea473d800bcfbfa4e510d9fa Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 22:29:25 +0800 Subject: [PATCH 18/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 9a0dff0a..24963b5e 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -146,12 +146,13 @@ jobs: run: | docker run -d --rm --name crawlab_master \ -e CRAWLAB_NODE_MASTER=Y \ + -p 8080:8080 \ image sleep 30 docker ps docker logs crawlab_master - res=`docker exec crawlab_master curl http://localhost:8000/system-info -s` - echo $res + res=`curl http://localhost:8080/api/system-info -s` + echo "res: ${res}" if [[ $res =~ "success" ]]; then : else From 9ce09f7744e21a776cc254230ebb205cedee99d5 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 10 Jun 2022 22:32:36 +0800 Subject: [PATCH 19/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 24963b5e..f243acd6 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -146,6 +146,7 @@ jobs: run: | docker run -d --rm --name crawlab_master \ -e CRAWLAB_NODE_MASTER=Y \ + -e CRAWLAB_MONGO_HOST=localhost \ -p 8080:8080 \ image sleep 30 From 6deecf1774864200443f8bbc96b04ffe732e37f5 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 10:43:24 +0800 Subject: [PATCH 20/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index f243acd6..5dfd6e70 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -147,9 +147,10 @@ jobs: docker run -d --rm --name crawlab_master \ -e CRAWLAB_NODE_MASTER=Y \ -e CRAWLAB_MONGO_HOST=localhost \ + -e CRAWLAB_MONGO_PORT=27017 \ -p 8080:8080 \ image - sleep 30 + sleep 10 docker ps docker logs crawlab_master res=`curl http://localhost:8080/api/system-info -s` From 0a8aef3bf5b55b7fe51960044ddb63b24e60221c Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 10:47:48 +0800 Subject: [PATCH 21/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 5dfd6e70..29c25d46 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -150,6 +150,7 @@ jobs: -e CRAWLAB_MONGO_PORT=27017 \ -p 8080:8080 \ image + docker exec crawlab_master env sleep 10 docker ps docker logs crawlab_master From 95e3c37522690d3d8272e1154af9b4111edb6f9e Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 10:50:03 +0800 Subject: [PATCH 22/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 29c25d46..fd6053c8 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -145,7 +145,7 @@ jobs: - name: Test image run: | docker run -d --rm --name crawlab_master \ - -e CRAWLAB_NODE_MASTER=Y \ + -e CRAWLAB_NODE_MASTER=true \ -e CRAWLAB_MONGO_HOST=localhost \ -e CRAWLAB_MONGO_PORT=27017 \ -p 8080:8080 \ From 39807715d181cfc0faf6b85ee354ede7cbbb34ce Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 10:53:58 +0800 Subject: [PATCH 23/66] trigger workflow --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index c8cee481..78f6265e 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220610131302-39654de64d31 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611025234-152e21d0e57f diff --git a/backend/go.sum b/backend/go.sum index ca361169..9ca8312c 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -134,6 +134,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220609135233-fed5b2972049 h1:UDk github.com/crawlab-team/crawlab-core v0.6.1-0.20220609135233-fed5b2972049/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220610131302-39654de64d31 h1:Fws59e6GrWl3J4VcRJt4dBv0MIvXnzLoRt4ZA1xQ08Y= github.com/crawlab-team/crawlab-core v0.6.1-0.20220610131302-39654de64d31/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611025234-152e21d0e57f h1:M0OFYwG68cWydh5guI7aXJy/i8+vXLleUd5wOAZgTq0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611025234-152e21d0e57f/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From 31d68da949f7f0ceb56fab37cb8382bb17f64678 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 11:02:50 +0800 Subject: [PATCH 24/66] trigger workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index fd6053c8..11ec19ff 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -151,7 +151,7 @@ jobs: -p 8080:8080 \ image docker exec crawlab_master env - sleep 10 + sleep 20 docker ps docker logs crawlab_master res=`curl http://localhost:8080/api/system-info -s` From c0303f5fdb9119a7329340c32af1dc93cdf923aa Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 11:14:12 +0800 Subject: [PATCH 25/66] trigger workflow --- bin/docker-init.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bin/docker-init.sh b/bin/docker-init.sh index 170dabbf..7c72713c 100755 --- a/bin/docker-init.sh +++ b/bin/docker-init.sh @@ -4,9 +4,15 @@ if [ "${CRAWLAB_NODE_MASTER}" = "Y" ]; then # start master /bin/bash /app/bin/docker-start-master.sh + # env + export IS_MASTER=1 + # start crawlab crawlab-server master else + # env + export IS_MASTER=0 + # start crawlab crawlab-server worker fi From 87704c2cd8d07e8f7dbde6de3222fa8017b25790 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 11:17:18 +0800 Subject: [PATCH 26/66] trigger workflow --- bin/docker-init.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/docker-init.sh b/bin/docker-init.sh index 7c72713c..88327ded 100755 --- a/bin/docker-init.sh +++ b/bin/docker-init.sh @@ -4,14 +4,14 @@ if [ "${CRAWLAB_NODE_MASTER}" = "Y" ]; then # start master /bin/bash /app/bin/docker-start-master.sh - # env - export IS_MASTER=1 + # node type + echo "node type: master" # start crawlab crawlab-server master else - # env - export IS_MASTER=0 + # node type + echo "node type: worker" # start crawlab crawlab-server worker From 78fe8d2960a587ab3696e0ef5dd9b5a44778521f Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 11:28:41 +0800 Subject: [PATCH 27/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index 78f6265e..e8b94cbb 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611025234-152e21d0e57f +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611032702-c31cbc7d599b diff --git a/backend/go.sum b/backend/go.sum index 9ca8312c..c83cab70 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -136,6 +136,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220610131302-39654de64d31 h1:Fws github.com/crawlab-team/crawlab-core v0.6.1-0.20220610131302-39654de64d31/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611025234-152e21d0e57f h1:M0OFYwG68cWydh5guI7aXJy/i8+vXLleUd5wOAZgTq0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611025234-152e21d0e57f/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611032702-c31cbc7d599b h1:oUwe+ap4+e0WhQ6JVZ6YPALfuVKhcYzeRTgyOeKPk28= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611032702-c31cbc7d599b/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From c6ee25491e6dd6120b8fc60364f28e61bf5a22b9 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 13:02:40 +0800 Subject: [PATCH 28/66] updated docker scripts --- backend/go.mod | 2 +- backend/go.sum | 4 ++++ bin/docker-init.sh | 17 +---------------- bin/docker-start-master.sh | 16 ++++++++-------- 4 files changed, 14 insertions(+), 25 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index e8b94cbb..07f0b78d 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611032702-c31cbc7d599b +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611045456-7deb18a3b809 diff --git a/backend/go.sum b/backend/go.sum index c83cab70..dfa3a760 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -126,6 +126,8 @@ github.com/crawlab-team/crawlab-core v0.6.0-8 h1:P1l8dBdecZ6vVxwWwOwjLtCUriSNyBA github.com/crawlab-team/crawlab-core v0.6.0-8/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.0-9 h1:oBwtAzafOVl2r2vLbLYClgpXDViwVVzqdXnbwC2aGj4= github.com/crawlab-team/crawlab-core v0.6.0-9/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.0 h1:CGxdztjcIkozZp2EHXFN1brvKxIAdD0Xe1l4c+SFNE8= +github.com/crawlab-team/crawlab-core v0.6.0/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220609124710-a6fcaa2e7ea6 h1:DGXTxMCJhLZkbBZcwHE6wisUO96nsy5rTlV7e9pQ+Kg= github.com/crawlab-team/crawlab-core v0.6.1-0.20220609124710-a6fcaa2e7ea6/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220609133021-540f1f228a89 h1:8/9AsqjaWN19379M6Pr/KXHxvRKWOE/S7SbbpzrYpIc= @@ -138,6 +140,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220611025234-152e21d0e57f h1:M0O github.com/crawlab-team/crawlab-core v0.6.1-0.20220611025234-152e21d0e57f/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611032702-c31cbc7d599b h1:oUwe+ap4+e0WhQ6JVZ6YPALfuVKhcYzeRTgyOeKPk28= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611032702-c31cbc7d599b/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611045456-7deb18a3b809 h1:0S7Mo1wqVrCF64yiTsuINBZWVzCZozvqihTu7XP/jkU= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611045456-7deb18a3b809/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= diff --git a/bin/docker-init.sh b/bin/docker-init.sh index 88327ded..452348b5 100755 --- a/bin/docker-init.sh +++ b/bin/docker-init.sh @@ -1,18 +1,3 @@ #!/bin/bash -if [ "${CRAWLAB_NODE_MASTER}" = "Y" ]; then - # start master - /bin/bash /app/bin/docker-start-master.sh - - # node type - echo "node type: master" - - # start crawlab - crawlab-server master -else - # node type - echo "node type: worker" - - # start crawlab - crawlab-server worker -fi +crawlab-server server diff --git a/bin/docker-start-master.sh b/bin/docker-start-master.sh index b6bd84ff..48118b74 100755 --- a/bin/docker-start-master.sh +++ b/bin/docker-start-master.sh @@ -1,25 +1,25 @@ #!/bin/bash # replace env -indexpath=/app/dist/index.html +indexPath=/app/dist/index.html if test -z "$CRAWLAB_BASE_URL"; then : else - sed -i "s?/js/?${CRAWLAB_BASE_URL}/js/?g" ${indexpath} - sed -i "s?/css/?${CRAWLAB_BASE_URL}/css/?g" ${indexpath} - sed -i "s/ Date: Sat, 11 Jun 2022 13:46:30 +0800 Subject: [PATCH 29/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index 07f0b78d..9bda162d 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611045456-7deb18a3b809 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611051011-90cbe93eda1a diff --git a/backend/go.sum b/backend/go.sum index dfa3a760..3db2b0ae 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -142,6 +142,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220611032702-c31cbc7d599b h1:oUw github.com/crawlab-team/crawlab-core v0.6.1-0.20220611032702-c31cbc7d599b/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611045456-7deb18a3b809 h1:0S7Mo1wqVrCF64yiTsuINBZWVzCZozvqihTu7XP/jkU= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611045456-7deb18a3b809/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611051011-90cbe93eda1a h1:4qC+7Q9rsebOz5zkTmSMmpjs7rN74HvDkiZ9FACi1cg= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611051011-90cbe93eda1a/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From 5aed1edb9ddab5028362a3e488ee589578ae6d74 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 13:55:55 +0800 Subject: [PATCH 30/66] updated workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 11ec19ff..8913ebe9 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -154,7 +154,7 @@ jobs: sleep 20 docker ps docker logs crawlab_master - res=`curl http://localhost:8080/api/system-info -s` + res=`docker exec crawlab_master curl http://localhost:8080/api/system-info -s` echo "res: ${res}" if [[ $res =~ "success" ]]; then : From c84f519915eefbc1710c670f4bc9970eaeafd1e0 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 16:06:45 +0800 Subject: [PATCH 31/66] updated workflow --- .github/workflows/docker-crawlab.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 8913ebe9..f9258104 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -156,11 +156,11 @@ jobs: docker logs crawlab_master res=`docker exec crawlab_master curl http://localhost:8080/api/system-info -s` echo "res: ${res}" - if [[ $res =~ "success" ]]; then - : - else - exit 1 - fi + #if [[ $res =~ "success" ]]; then + # : + #else + # exit 1 + #fi docker stop crawlab_master - name: Log into registry From 2491526340cc49444a247faeb80b0b3b38632b31 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 16:13:32 +0800 Subject: [PATCH 32/66] updated workflow --- .github/workflows/docker-crawlab.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index f9258104..099ebbe4 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -154,8 +154,8 @@ jobs: sleep 20 docker ps docker logs crawlab_master - res=`docker exec crawlab_master curl http://localhost:8080/api/system-info -s` - echo "res: ${res}" + docker exec crawlab_master curl http://localhost:8080/api/system-info -s + #echo "res: ${res}" #if [[ $res =~ "success" ]]; then # : #else From ea6bf670336a6d02ca3a22570c84d79bc0c31c09 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 16:16:57 +0800 Subject: [PATCH 33/66] updated workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 099ebbe4..99cd9767 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -154,7 +154,7 @@ jobs: sleep 20 docker ps docker logs crawlab_master - docker exec crawlab_master curl http://localhost:8080/api/system-info -s + docker exec crawlab_master curl http://localhost:8000/system-info -v #echo "res: ${res}" #if [[ $res =~ "success" ]]; then # : From 70ab1069e1c809805c07aeaf82c2043702bc453c Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 16:17:17 +0800 Subject: [PATCH 34/66] updated workflow --- .github/workflows/docker-crawlab.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 99cd9767..92ad7b03 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -146,6 +146,7 @@ jobs: run: | docker run -d --rm --name crawlab_master \ -e CRAWLAB_NODE_MASTER=true \ + -e CRAWLAB_DEMO=false \ -e CRAWLAB_MONGO_HOST=localhost \ -e CRAWLAB_MONGO_PORT=27017 \ -p 8080:8080 \ From 6eb7a7cde3d7f915c82b2f672590fef757141347 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 16:22:50 +0800 Subject: [PATCH 35/66] updated workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 92ad7b03..49fa9816 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -155,7 +155,7 @@ jobs: sleep 20 docker ps docker logs crawlab_master - docker exec crawlab_master curl http://localhost:8000/system-info -v + #docker exec crawlab_master curl http://localhost:8000/system-info -v #echo "res: ${res}" #if [[ $res =~ "success" ]]; then # : From 115ebe08f7839c833770e5c7d9f6fe871669da14 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 16:32:21 +0800 Subject: [PATCH 36/66] updated workflow --- .github/workflows/docker-crawlab.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 49fa9816..c7af6caa 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -155,7 +155,8 @@ jobs: sleep 20 docker ps docker logs crawlab_master - #docker exec crawlab_master curl http://localhost:8000/system-info -v + docker exec crawlab_master curl http://localhost:8888 -v + docker exec crawlab_master curl http://localhost:8000/system-info -v #echo "res: ${res}" #if [[ $res =~ "success" ]]; then # : From dcbe1f13288be8af2341b14dc3ae7c2326e3ff97 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 16:33:10 +0800 Subject: [PATCH 37/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index 9bda162d..deecf062 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611051011-90cbe93eda1a +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611083139-4b7bdcc24974 diff --git a/backend/go.sum b/backend/go.sum index 3db2b0ae..ec5b54e6 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -144,6 +144,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220611045456-7deb18a3b809 h1:0S7 github.com/crawlab-team/crawlab-core v0.6.1-0.20220611045456-7deb18a3b809/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611051011-90cbe93eda1a h1:4qC+7Q9rsebOz5zkTmSMmpjs7rN74HvDkiZ9FACi1cg= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611051011-90cbe93eda1a/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611083139-4b7bdcc24974 h1:/HNg9aJWhIxy9XZL4VYs8RFMGP2KE8GM1edfhoRd9pc= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611083139-4b7bdcc24974/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From cbdd66e80e6e9a0e1dcc4b71325ca3936588733d Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 16:49:15 +0800 Subject: [PATCH 38/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index deecf062..dedaebd2 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611083139-4b7bdcc24974 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611084717-69dfd3bf6776 diff --git a/backend/go.sum b/backend/go.sum index ec5b54e6..472287ef 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -146,6 +146,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220611051011-90cbe93eda1a h1:4qC github.com/crawlab-team/crawlab-core v0.6.1-0.20220611051011-90cbe93eda1a/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611083139-4b7bdcc24974 h1:/HNg9aJWhIxy9XZL4VYs8RFMGP2KE8GM1edfhoRd9pc= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611083139-4b7bdcc24974/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611084717-69dfd3bf6776 h1:M8stvkcx1fBdG8FfzmoRtsfnKjF46mXNFK/lDTTu+ZA= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611084717-69dfd3bf6776/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From 9b8f0a3669e3cd2568ccedce4520d6dc752f5348 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 17:12:55 +0800 Subject: [PATCH 39/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index dedaebd2..ab53cdbc 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611084717-69dfd3bf6776 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611090603-dfb61d4ad3a8 diff --git a/backend/go.sum b/backend/go.sum index 472287ef..d0ab2688 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -148,6 +148,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220611083139-4b7bdcc24974 h1:/HN github.com/crawlab-team/crawlab-core v0.6.1-0.20220611083139-4b7bdcc24974/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611084717-69dfd3bf6776 h1:M8stvkcx1fBdG8FfzmoRtsfnKjF46mXNFK/lDTTu+ZA= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611084717-69dfd3bf6776/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611090603-dfb61d4ad3a8 h1:/CgDdb3FITtXerFTzY84XyzN4nt4uDSJf7nDD6e7s/8= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611090603-dfb61d4ad3a8/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From bb06ea643c35a3ebddc4a11e5e065c0de5ebcc51 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 11 Jun 2022 17:22:39 +0800 Subject: [PATCH 40/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index ab53cdbc..bab41228 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611090603-dfb61d4ad3a8 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611092109-15a2a615d560 diff --git a/backend/go.sum b/backend/go.sum index d0ab2688..e228a737 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -150,6 +150,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220611084717-69dfd3bf6776 h1:M8s github.com/crawlab-team/crawlab-core v0.6.1-0.20220611084717-69dfd3bf6776/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611090603-dfb61d4ad3a8 h1:/CgDdb3FITtXerFTzY84XyzN4nt4uDSJf7nDD6e7s/8= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611090603-dfb61d4ad3a8/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611092109-15a2a615d560 h1:HxU7s4AjYEIT0jvvePAfHM5yFGmVuRQunKqV/hP70hQ= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611092109-15a2a615d560/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From 63bbe0e97e8929abf0b5cc370493e695cb6400eb Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 10:51:03 +0800 Subject: [PATCH 41/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index bab41228..844e83dd 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220611092109-15a2a615d560 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612024940-b4df67e569a6 diff --git a/backend/go.sum b/backend/go.sum index e228a737..ae0ced68 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -152,6 +152,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220611090603-dfb61d4ad3a8 h1:/Cg github.com/crawlab-team/crawlab-core v0.6.1-0.20220611090603-dfb61d4ad3a8/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611092109-15a2a615d560 h1:HxU7s4AjYEIT0jvvePAfHM5yFGmVuRQunKqV/hP70hQ= github.com/crawlab-team/crawlab-core v0.6.1-0.20220611092109-15a2a615d560/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612024940-b4df67e569a6 h1:RpJsizVjCB4zx6AYf5cul3EgDInL3iax8qRO2CZSGJY= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612024940-b4df67e569a6/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From 2a6b61ebf0f6d9f192491ad258c4403243bfb7a6 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 10:56:08 +0800 Subject: [PATCH 42/66] updated workflow --- .github/workflows/docker-crawlab.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index c7af6caa..1b69d703 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -155,8 +155,9 @@ jobs: sleep 20 docker ps docker logs crawlab_master - docker exec crawlab_master curl http://localhost:8888 -v - docker exec crawlab_master curl http://localhost:8000/system-info -v + docker exec crawlab_master ls -l /var/log + #docker exec crawlab_master curl http://localhost:8888 -v + #docker exec crawlab_master curl http://localhost:8000/system-info -v #echo "res: ${res}" #if [[ $res =~ "success" ]]; then # : From 7aacfac23d628534ae8f9e6074c4ae5062fad2ef Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 11:06:02 +0800 Subject: [PATCH 43/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index 844e83dd..cb06dec8 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612024940-b4df67e569a6 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612030521-bbc184169243 diff --git a/backend/go.sum b/backend/go.sum index ae0ced68..041d1772 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -154,6 +154,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220611092109-15a2a615d560 h1:HxU github.com/crawlab-team/crawlab-core v0.6.1-0.20220611092109-15a2a615d560/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220612024940-b4df67e569a6 h1:RpJsizVjCB4zx6AYf5cul3EgDInL3iax8qRO2CZSGJY= github.com/crawlab-team/crawlab-core v0.6.1-0.20220612024940-b4df67e569a6/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612030521-bbc184169243 h1:04Ay5eUDB/PzjB+qUltdGp48uNBn1z5WXQGHS/uQ8Ug= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612030521-bbc184169243/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From 021c3f16866131eef84e4435eef7548b6b7e6c66 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 11:10:14 +0800 Subject: [PATCH 44/66] updated workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 1b69d703..66ac7c19 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -152,7 +152,7 @@ jobs: -p 8080:8080 \ image docker exec crawlab_master env - sleep 20 + sleep 10 docker ps docker logs crawlab_master docker exec crawlab_master ls -l /var/log From b7d86e341526c0ee291733076affc271ef8914c4 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 11:18:53 +0800 Subject: [PATCH 45/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index cb06dec8..e4f0c232 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612030521-bbc184169243 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612031417-78e1024b1838 diff --git a/backend/go.sum b/backend/go.sum index 041d1772..b8785627 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -156,6 +156,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220612024940-b4df67e569a6 h1:RpJ github.com/crawlab-team/crawlab-core v0.6.1-0.20220612024940-b4df67e569a6/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220612030521-bbc184169243 h1:04Ay5eUDB/PzjB+qUltdGp48uNBn1z5WXQGHS/uQ8Ug= github.com/crawlab-team/crawlab-core v0.6.1-0.20220612030521-bbc184169243/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612031417-78e1024b1838 h1:akTxObujaS2XobGs7NbYBYtmZNzEBEasCvE/ru3KjOg= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612031417-78e1024b1838/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From 3171af5906e29356fffe3849ea4ddcde6b34457c Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 11:40:51 +0800 Subject: [PATCH 46/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index e4f0c232..9c9c946e 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612031417-78e1024b1838 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612033735-e730686bb127 diff --git a/backend/go.sum b/backend/go.sum index b8785627..7cd7e6fe 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -158,6 +158,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220612030521-bbc184169243 h1:04A github.com/crawlab-team/crawlab-core v0.6.1-0.20220612030521-bbc184169243/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220612031417-78e1024b1838 h1:akTxObujaS2XobGs7NbYBYtmZNzEBEasCvE/ru3KjOg= github.com/crawlab-team/crawlab-core v0.6.1-0.20220612031417-78e1024b1838/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612033735-e730686bb127 h1:YjL1pDpBMhSMCfhhJqTUdYxHU5LGumZI0xfZfwVs1oI= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612033735-e730686bb127/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From 37e60b41797fef369866ccec7ddd15d267348cbd Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 11:41:32 +0800 Subject: [PATCH 47/66] updated workflow --- .github/workflows/docker-crawlab.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 66ac7c19..3242b0d5 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -156,14 +156,13 @@ jobs: docker ps docker logs crawlab_master docker exec crawlab_master ls -l /var/log - #docker exec crawlab_master curl http://localhost:8888 -v - #docker exec crawlab_master curl http://localhost:8000/system-info -v - #echo "res: ${res}" - #if [[ $res =~ "success" ]]; then - # : - #else - # exit 1 - #fi + docker exec crawlab_master curl http://localhost:8000/system-info -v + echo "res: ${res}" + if [[ $res =~ "success" ]]; then + : + else + exit 1 + fi docker stop crawlab_master - name: Log into registry From ca6c380fc830b2d2c3d9576db38829708c6660a7 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 11:50:56 +0800 Subject: [PATCH 48/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index 9c9c946e..b3c8c691 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612033735-e730686bb127 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612034952-1cc047c47719 diff --git a/backend/go.sum b/backend/go.sum index 7cd7e6fe..511d9497 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -160,6 +160,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220612031417-78e1024b1838 h1:akT github.com/crawlab-team/crawlab-core v0.6.1-0.20220612031417-78e1024b1838/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220612033735-e730686bb127 h1:YjL1pDpBMhSMCfhhJqTUdYxHU5LGumZI0xfZfwVs1oI= github.com/crawlab-team/crawlab-core v0.6.1-0.20220612033735-e730686bb127/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612034952-1cc047c47719 h1:TDeEOYyFkGuaSItG709k+qDwIi/08ZxgHXhz6ti0Pwk= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612034952-1cc047c47719/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From 36b7d5abd948a59d2e5d729c71593d4601dec205 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 12:02:51 +0800 Subject: [PATCH 49/66] updated workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 3242b0d5..fe294b3c 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -152,7 +152,7 @@ jobs: -p 8080:8080 \ image docker exec crawlab_master env - sleep 10 + sleep 30 docker ps docker logs crawlab_master docker exec crawlab_master ls -l /var/log From 7c9bd853700c68c5cfd211021138d7ce5587da65 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 12:05:58 +0800 Subject: [PATCH 50/66] updated workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index fe294b3c..b3fc6815 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -156,7 +156,7 @@ jobs: docker ps docker logs crawlab_master docker exec crawlab_master ls -l /var/log - docker exec crawlab_master curl http://localhost:8000/system-info -v + res=`docker exec crawlab_master curl http://localhost:8000/system-info -s` echo "res: ${res}" if [[ $res =~ "success" ]]; then : From 2d728bd029748d19425ffb0a6560b6d370d9d278 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 12:06:02 +0800 Subject: [PATCH 51/66] updated workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index b3fc6815..76e3c0d8 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -152,7 +152,7 @@ jobs: -p 8080:8080 \ image docker exec crawlab_master env - sleep 30 + sleep 10 docker ps docker logs crawlab_master docker exec crawlab_master ls -l /var/log From 90e31c2c09933b0764b2e9a37a5faa0d5e156d65 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 12:09:26 +0800 Subject: [PATCH 52/66] updated workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 76e3c0d8..b3fc6815 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -152,7 +152,7 @@ jobs: -p 8080:8080 \ image docker exec crawlab_master env - sleep 10 + sleep 30 docker ps docker logs crawlab_master docker exec crawlab_master ls -l /var/log From a01528c851d295b1f07f8088d92052e0ba0e25c6 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 12:17:39 +0800 Subject: [PATCH 53/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index b3c8c691..b9c96327 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612034952-1cc047c47719 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612041650-37795c0ced68 diff --git a/backend/go.sum b/backend/go.sum index 511d9497..42f47e4a 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -162,6 +162,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220612033735-e730686bb127 h1:YjL github.com/crawlab-team/crawlab-core v0.6.1-0.20220612033735-e730686bb127/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220612034952-1cc047c47719 h1:TDeEOYyFkGuaSItG709k+qDwIi/08ZxgHXhz6ti0Pwk= github.com/crawlab-team/crawlab-core v0.6.1-0.20220612034952-1cc047c47719/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612041650-37795c0ced68 h1:aDA3j5F6PoXy5hl1xD2vO9qtLLa+1JAIvpYf55vPq3k= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612041650-37795c0ced68/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From b1ec141c00b4e207f8a7a80378cee119fd2fa1dc Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 12:31:34 +0800 Subject: [PATCH 54/66] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go.mod b/backend/go.mod index b9c96327..907828a8 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612041650-37795c0ced68 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612043053-72fb6748a131 diff --git a/backend/go.sum b/backend/go.sum index 42f47e4a..34e0c478 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -164,6 +164,8 @@ github.com/crawlab-team/crawlab-core v0.6.1-0.20220612034952-1cc047c47719 h1:TDe github.com/crawlab-team/crawlab-core v0.6.1-0.20220612034952-1cc047c47719/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-core v0.6.1-0.20220612041650-37795c0ced68 h1:aDA3j5F6PoXy5hl1xD2vO9qtLLa+1JAIvpYf55vPq3k= github.com/crawlab-team/crawlab-core v0.6.1-0.20220612041650-37795c0ced68/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612043053-72fb6748a131 h1:ZjXs65vYQcA26EC7qmDBA3ny3UHEdx/GNEXjVJWxmek= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612043053-72fb6748a131/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= From 4d2924467adca78274ce7017bf94b4c01b44db78 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 13:37:11 +0800 Subject: [PATCH 55/66] updated workflow --- .github/workflows/docker-crawlab.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index b3fc6815..fd9a7b11 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -144,13 +144,13 @@ jobs: - name: Test image run: | - docker run -d --rm --name crawlab_master \ + docker run --rm --name crawlab_master \ -e CRAWLAB_NODE_MASTER=true \ -e CRAWLAB_DEMO=false \ -e CRAWLAB_MONGO_HOST=localhost \ -e CRAWLAB_MONGO_PORT=27017 \ -p 8080:8080 \ - image + image & docker exec crawlab_master env sleep 30 docker ps From 4b20bb0dfa9154b84c8c2b88ab5ceb45f1739f87 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 14:03:10 +0800 Subject: [PATCH 56/66] updated workflow --- .github/workflows/docker-crawlab.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index fd9a7b11..74d35c01 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -144,13 +144,14 @@ jobs: - name: Test image run: | - docker run --rm --name crawlab_master \ + docker run --rm -d --name crawlab_master \ -e CRAWLAB_NODE_MASTER=true \ -e CRAWLAB_DEMO=false \ -e CRAWLAB_MONGO_HOST=localhost \ -e CRAWLAB_MONGO_PORT=27017 \ -p 8080:8080 \ - image & + image + docker logs -f crawlab_master & docker exec crawlab_master env sleep 30 docker ps From 773d0378bfffb684231a01a1ade4131fe3fe87f2 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 14:08:53 +0800 Subject: [PATCH 57/66] updated workflow --- .github/workflows/docker-crawlab.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 74d35c01..1fd6f58f 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -147,13 +147,13 @@ jobs: docker run --rm -d --name crawlab_master \ -e CRAWLAB_NODE_MASTER=true \ -e CRAWLAB_DEMO=false \ - -e CRAWLAB_MONGO_HOST=localhost \ + -e CRAWLAB_MONGO_HOST=mongo \ -e CRAWLAB_MONGO_PORT=27017 \ -p 8080:8080 \ image docker logs -f crawlab_master & docker exec crawlab_master env - sleep 30 + sleep 10 docker ps docker logs crawlab_master docker exec crawlab_master ls -l /var/log From 18373d9b325d46d034c53daf5857169785790761 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 14:13:42 +0800 Subject: [PATCH 58/66] updated workflow --- .github/workflows/docker-crawlab.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 1fd6f58f..ee29fc7c 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -153,9 +153,8 @@ jobs: image docker logs -f crawlab_master & docker exec crawlab_master env - sleep 10 + sleep 30 docker ps - docker logs crawlab_master docker exec crawlab_master ls -l /var/log res=`docker exec crawlab_master curl http://localhost:8000/system-info -s` echo "res: ${res}" From 65631fe4067b7a14e7395dac33a58f11d1c77aab Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 14:14:52 +0800 Subject: [PATCH 59/66] updated workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index ee29fc7c..19a3b5b7 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -147,7 +147,7 @@ jobs: docker run --rm -d --name crawlab_master \ -e CRAWLAB_NODE_MASTER=true \ -e CRAWLAB_DEMO=false \ - -e CRAWLAB_MONGO_HOST=mongo \ + -e CRAWLAB_MONGO_HOST=localhost \ -e CRAWLAB_MONGO_PORT=27017 \ -p 8080:8080 \ image From efa21d7645a690d0b6192db634d7e9f94a09bb5d Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 15:04:46 +0800 Subject: [PATCH 60/66] updated workflow --- .github/workflows/docker-crawlab.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 19a3b5b7..f4ffb9f7 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -144,6 +144,7 @@ jobs: - name: Test image run: | + curl http://localhost:27017 docker run --rm -d --name crawlab_master \ -e CRAWLAB_NODE_MASTER=true \ -e CRAWLAB_DEMO=false \ From f896bd8ba5075b37f8c373d4862e20e1e75e486d Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 15:13:21 +0800 Subject: [PATCH 61/66] updated workflow --- .github/workflows/docker-crawlab.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index f4ffb9f7..88ffa3a0 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -145,6 +145,7 @@ jobs: - name: Test image run: | curl http://localhost:27017 + docker network ls docker run --rm -d --name crawlab_master \ -e CRAWLAB_NODE_MASTER=true \ -e CRAWLAB_DEMO=false \ From c73c27589df913100d57a398e40e3b13dfa7428a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 15:18:36 +0800 Subject: [PATCH 62/66] updated workflow --- .github/workflows/docker-crawlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 88ffa3a0..e85b166b 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -145,13 +145,13 @@ jobs: - name: Test image run: | curl http://localhost:27017 - docker network ls docker run --rm -d --name crawlab_master \ -e CRAWLAB_NODE_MASTER=true \ -e CRAWLAB_DEMO=false \ -e CRAWLAB_MONGO_HOST=localhost \ -e CRAWLAB_MONGO_PORT=27017 \ -p 8080:8080 \ + --network host \ image docker logs -f crawlab_master & docker exec crawlab_master env From 00f31ece9e9825568cda6170ab6bcb101b136007 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 15:19:56 +0800 Subject: [PATCH 63/66] updated workflow --- .github/workflows/docker-crawlab.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index e85b166b..fdb62c35 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -144,7 +144,6 @@ jobs: - name: Test image run: | - curl http://localhost:27017 docker run --rm -d --name crawlab_master \ -e CRAWLAB_NODE_MASTER=true \ -e CRAWLAB_DEMO=false \ @@ -153,9 +152,9 @@ jobs: -p 8080:8080 \ --network host \ image - docker logs -f crawlab_master & docker exec crawlab_master env - sleep 30 + docker logs -f crawlab_master & + sleep 10 docker ps docker exec crawlab_master ls -l /var/log res=`docker exec crawlab_master curl http://localhost:8000/system-info -s` From 6e24b243766132dda0b5d7250052b65a7360242a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 15:22:07 +0800 Subject: [PATCH 64/66] updated workflow --- .github/workflows/docker-crawlab.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index fdb62c35..dabe945c 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -157,7 +157,9 @@ jobs: sleep 10 docker ps docker exec crawlab_master ls -l /var/log - res=`docker exec crawlab_master curl http://localhost:8000/system-info -s` + cmd='curl http://localhost:8080/api/system-info -s' + echo "cmd: ${cmd}" + res=`${cmd}` echo "res: ${res}" if [[ $res =~ "success" ]]; then : From 64a57b8ef1064cd8b3b3eaa4b537edae5a14e98d Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 15:24:16 +0800 Subject: [PATCH 65/66] updated scripts --- bin/docker-start-master.sh | 1 + 1 file changed, 1 insertion(+) mode change 100755 => 100644 bin/docker-start-master.sh diff --git a/bin/docker-start-master.sh b/bin/docker-start-master.sh old mode 100755 new mode 100644 index 48118b74..e3720814 --- a/bin/docker-start-master.sh +++ b/bin/docker-start-master.sh @@ -1,4 +1,5 @@ #!/bin/bash +# NOTE: deprecated # replace env indexPath=/app/dist/index.html From 93a1e8bbd3ec89a97fc4d9b398c6a9c29716d774 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 12 Jun 2022 15:24:49 +0800 Subject: [PATCH 66/66] updated scripts --- .github/workflows/docker-crawlab.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index dabe945c..570324af 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -156,7 +156,6 @@ jobs: docker logs -f crawlab_master & sleep 10 docker ps - docker exec crawlab_master ls -l /var/log cmd='curl http://localhost:8080/api/system-info -s' echo "cmd: ${cmd}" res=`${cmd}`