Merge pull request #1122 from crawlab-team/develop

Develop
2026-01-22 17:31:03 +01:00 · 2022-06-12 15:56:56 +08:00
parent be4194e08d a3f17a8187
commit f612464b90
6 changed files with 106 additions and 59 deletions
--- a/.github/workflows/docker-crawlab.yml
+++ b/.github/workflows/docker-crawlab.yml
@@ -122,6 +122,11 @@ jobs:
    if: ${{ always() }}
    needs: [ setup, build-backend, build-frontend ]
    runs-on: ubuntu-latest
+    services:
+      mongo:
+        image: mongo:4.2
+        ports:
+          - 27017:27017
    steps:
      - uses: actions/checkout@v2
      - name: Update Dockerfile
@@ -136,8 +141,35 @@ jobs:

      - name: Build image
        run: docker build . --file Dockerfile --tag image
+
+      - name: Test image
+        run: |
+          docker run --rm -d --name crawlab_master \
+            -e CRAWLAB_NODE_MASTER=true \
+            -e CRAWLAB_DEMO=false \
+            -e CRAWLAB_MONGO_HOST=localhost \
+            -e CRAWLAB_MONGO_PORT=27017 \
+            -p 8080:8080 \
+            --network host \
+            image
+          docker exec crawlab_master env
+          docker logs -f crawlab_master &
+          sleep 10
+          docker ps
+          cmd='curl http://localhost:8080/api/system-info -s'
+          echo "cmd: ${cmd}"
+          res=`${cmd}`
+          echo "res: ${res}"
+          if [[ $res =~ "success" ]]; then
+            :
+          else
+            exit 1
+          fi
+          docker stop crawlab_master
+
      - name: Log into registry
-        run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+        run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+
      - name: Push image
        run: |
          IMAGE_VERSION=${{needs.setup.outputs.version}}
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -2,4 +2,4 @@ module crawlab

 go 1.16

-require github.com/crawlab-team/crawlab-core v0.6.0-7
+require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612043053-72fb6748a131
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -122,8 +122,50 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k=
-github.com/crawlab-team/crawlab-core v0.6.0-7 h1:23kTlKOZ3OqfSmD+SYJiwlqP1Cu1NJtPmbR8KeIGvgw=
-github.com/crawlab-team/crawlab-core v0.6.0-7/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.0-8 h1:P1l8dBdecZ6vVxwWwOwjLtCUriSNyBAe01Ghwfndoqw=
+github.com/crawlab-team/crawlab-core v0.6.0-8/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.0-9 h1:oBwtAzafOVl2r2vLbLYClgpXDViwVVzqdXnbwC2aGj4=
+github.com/crawlab-team/crawlab-core v0.6.0-9/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.0 h1:CGxdztjcIkozZp2EHXFN1brvKxIAdD0Xe1l4c+SFNE8=
+github.com/crawlab-team/crawlab-core v0.6.0/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220609124710-a6fcaa2e7ea6 h1:DGXTxMCJhLZkbBZcwHE6wisUO96nsy5rTlV7e9pQ+Kg=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220609124710-a6fcaa2e7ea6/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220609133021-540f1f228a89 h1:8/9AsqjaWN19379M6Pr/KXHxvRKWOE/S7SbbpzrYpIc=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220609133021-540f1f228a89/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220609135233-fed5b2972049 h1:UDkyU63T8E/vvD2cc58d7YPKnH64w6qdlyi1MnwIWO8=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220609135233-fed5b2972049/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220610131302-39654de64d31 h1:Fws59e6GrWl3J4VcRJt4dBv0MIvXnzLoRt4ZA1xQ08Y=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220610131302-39654de64d31/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611025234-152e21d0e57f h1:M0OFYwG68cWydh5guI7aXJy/i8+vXLleUd5wOAZgTq0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611025234-152e21d0e57f/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611032702-c31cbc7d599b h1:oUwe+ap4+e0WhQ6JVZ6YPALfuVKhcYzeRTgyOeKPk28=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611032702-c31cbc7d599b/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611045456-7deb18a3b809 h1:0S7Mo1wqVrCF64yiTsuINBZWVzCZozvqihTu7XP/jkU=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611045456-7deb18a3b809/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611051011-90cbe93eda1a h1:4qC+7Q9rsebOz5zkTmSMmpjs7rN74HvDkiZ9FACi1cg=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611051011-90cbe93eda1a/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611083139-4b7bdcc24974 h1:/HNg9aJWhIxy9XZL4VYs8RFMGP2KE8GM1edfhoRd9pc=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611083139-4b7bdcc24974/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611084717-69dfd3bf6776 h1:M8stvkcx1fBdG8FfzmoRtsfnKjF46mXNFK/lDTTu+ZA=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611084717-69dfd3bf6776/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611090603-dfb61d4ad3a8 h1:/CgDdb3FITtXerFTzY84XyzN4nt4uDSJf7nDD6e7s/8=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611090603-dfb61d4ad3a8/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611092109-15a2a615d560 h1:HxU7s4AjYEIT0jvvePAfHM5yFGmVuRQunKqV/hP70hQ=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220611092109-15a2a615d560/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220612024940-b4df67e569a6 h1:RpJsizVjCB4zx6AYf5cul3EgDInL3iax8qRO2CZSGJY=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220612024940-b4df67e569a6/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220612030521-bbc184169243 h1:04Ay5eUDB/PzjB+qUltdGp48uNBn1z5WXQGHS/uQ8Ug=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220612030521-bbc184169243/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220612031417-78e1024b1838 h1:akTxObujaS2XobGs7NbYBYtmZNzEBEasCvE/ru3KjOg=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220612031417-78e1024b1838/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220612033735-e730686bb127 h1:YjL1pDpBMhSMCfhhJqTUdYxHU5LGumZI0xfZfwVs1oI=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220612033735-e730686bb127/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220612034952-1cc047c47719 h1:TDeEOYyFkGuaSItG709k+qDwIi/08ZxgHXhz6ti0Pwk=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220612034952-1cc047c47719/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220612041650-37795c0ced68 h1:aDA3j5F6PoXy5hl1xD2vO9qtLLa+1JAIvpYf55vPq3k=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220612041650-37795c0ced68/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220612043053-72fb6748a131 h1:ZjXs65vYQcA26EC7qmDBA3ny3UHEdx/GNEXjVJWxmek=
+github.com/crawlab-team/crawlab-core v0.6.1-0.20220612043053-72fb6748a131/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
 github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78=
 github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k=
 github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU=
--- a/bin/docker-init.sh
+++ b/bin/docker-init.sh
@@ -1,12 +1,3 @@
 #!/bin/bash

-if [ "${CRAWLAB_NODE_MASTER}" = "Y" ]; then
-	# start master
-	/bin/bash /app/bin/docker-start-master.sh
-
-	# start crawlab
-	crawlab-server master
-else
-	# start crawlab
-	crawlab-server worker
-fi
+crawlab-server server
--- a/bin/docker-start-master.sh
+++ b/bin/docker-start-master.sh
@@ -1,23 +1,26 @@
+#!/bin/bash
+# NOTE: deprecated
+
 # replace env
-indexpath=/app/dist/index.html
+indexPath=/app/dist/index.html
 if test -z "$CRAWLAB_BASE_URL"; then
-	:
+  :
 else
-	sed -i "s?/js/?${CRAWLAB_BASE_URL}/js/?g" ${indexpath}
-	sed -i "s?/css/?${CRAWLAB_BASE_URL}/css/?g" ${indexpath}
-	sed -i "s/  <link rel=\"icon\" type=\"image\/x-icon\" href=\"/  <link rel=\"icon\" type=\"image\/x-icon\" href=\"\/${CRAWLAB_BASE_URL}/g"  ${indexpath}
-	sed -i "s/  <link rel=\"stylesheet\" href=\"/  <link rel=\"stylesheet\" href=\"${CRAWLAB_BASE_URL}\//g"  ${indexpath}
-	sed -i "s/  window.VUE_APP_API_BASE_URL = '/  window.VUE_APP_API_BASE_URL = '\/${CRAWLAB_BASE_URL}/g" ${indexpath}
+  sed -i "s?/js/?${CRAWLAB_BASE_URL}/js/?g" ${indexPath}
+  sed -i "s?/css/?${CRAWLAB_BASE_URL}/css/?g" ${indexPath}
+  sed -i "s/  <link rel=\"icon\" type=\"image\/x-icon\" href=\"/  <link rel=\"icon\" type=\"image\/x-icon\" href=\"\/${CRAWLAB_BASE_URL}/g" ${indexPath}
+  sed -i "s/  <link rel=\"stylesheet\" href=\"/  <link rel=\"stylesheet\" href=\"${CRAWLAB_BASE_URL}\//g" ${indexPath}
+  sed -i "s/  window.VUE_APP_API_BASE_URL = '/  window.VUE_APP_API_BASE_URL = '\/${CRAWLAB_BASE_URL}/g" ${indexPath}
 fi
 if test -z "$CRAWLAB_INIT_BAIDU_TONGJI"; then
-	:
+  :
 else
-	sed -i "s/  window.VUE_APP_INIT_BAIDU_TONGJI = ''/  window.VUE_APP_INIT_BAIDU_TONGJI = '${CRAWLAB_INIT_BAIDU_TONGJI}'/g" ${indexpath}
+  sed -i "s/  window.VUE_APP_INIT_BAIDU_TONGJI = ''/  window.VUE_APP_INIT_BAIDU_TONGJI = '${CRAWLAB_INIT_BAIDU_TONGJI}'/g" ${indexPath}
 fi
 if test -z "$CRAWLAB_INIT_UMENG"; then
-	:
+  :
 else
-	sed -i "s/  window.VUE_APP_INIT_UMENG = ''/  window.VUE_APP_INIT_UMENG = '${CRAWLAB_INIT_UMENG}'/g" ${indexpath}
+  sed -i "s/  window.VUE_APP_INIT_UMENG = ''/  window.VUE_APP_INIT_UMENG = '${CRAWLAB_INIT_UMENG}'/g" ${indexPath}
 fi

 # start nginx
@@ -26,16 +29,15 @@ service nginx start
 # start seaweedfs server
 seaweedfsDataPath=/data/seaweedfs
 if [ -e ${seaweedfsDataPath} ]; then
-	:
+  :
 else
-	mkdir -p ${seaweedfsDataPath}
+  mkdir -p ${seaweedfsDataPath}
 fi
 weed server \
-	-dir /data \
-	-master.dir ${seaweedfsDataPath} \
-	-volume.dir.idx ${seaweedfsDataPath} \
-	-ip localhost \
-	-volume.port 9999 \
-	-filer \
-	>> /var/log/weed.log 2>&1 &
-
+  -dir /data \
+  -master.dir ${seaweedfsDataPath} \
+  -volume.dir.idx ${seaweedfsDataPath} \
+  -ip localhost \
+  -volume.port 9999 \
+  -filer \
+  >>/var/log/weed.log 2>&1 &
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,34 +1,14 @@
 version: '3.3'
 services:
  master:
-    image: crawlabteam/crawlab:latest
+    image: crawlabteam/crawlab
    container_name: crawlab_master
    environment:
-      CRAWLAB_NODE_MASTER: Y
-      CRAWLAB_MONGO_HOST: mongo
-    ports:
-      - "8080:8080" # frontend port mapping 前端端口映射
-    depends_on:
-      - mongo
-    # volumes:
-    #   - "/opt/crawlab/master:/data"  # data persistent 持久化数据
-  worker:
-    image: crawlabteam/crawlab:latest
-    container_name: crawlab_worker
-    environment:
-      CRAWLAB_NODE_MASTER: "N"
+      CRAWLAB_NODE_MASTER: "Y"
      CRAWLAB_MONGO_HOST: "mongo"
+    ports:
+      - "8080:8080"
    depends_on:
      - mongo
-    # volumes:
-    #   - "/opt/crawlab/worker:/data"  # data persistent 持久化数据
  mongo:
    image: mongo:4.2
-    #restart: always
-    # environment:
-    #   MONGO_INITDB_ROOT_USERNAME: username
-    #   MONGO_INITDB_ROOT_PASSWORD: password
-    # volumes:
-    #   - "/opt/crawlab/mongo/data/db:/data/db"  # data persistent 持久化数据
-    # ports:
-    #   - "27017:27017"  # expose port to host machine 暴露接口到宿主机