diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index fd336c54..570324af 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -122,6 +122,11 @@ jobs: if: ${{ always() }} needs: [ setup, build-backend, build-frontend ] runs-on: ubuntu-latest + services: + mongo: + image: mongo:4.2 + ports: + - 27017:27017 steps: - uses: actions/checkout@v2 - name: Update Dockerfile @@ -136,8 +141,35 @@ jobs: - name: Build image run: docker build . --file Dockerfile --tag image + + - name: Test image + run: | + docker run --rm -d --name crawlab_master \ + -e CRAWLAB_NODE_MASTER=true \ + -e CRAWLAB_DEMO=false \ + -e CRAWLAB_MONGO_HOST=localhost \ + -e CRAWLAB_MONGO_PORT=27017 \ + -p 8080:8080 \ + --network host \ + image + docker exec crawlab_master env + docker logs -f crawlab_master & + sleep 10 + docker ps + cmd='curl http://localhost:8080/api/system-info -s' + echo "cmd: ${cmd}" + res=`${cmd}` + echo "res: ${res}" + if [[ $res =~ "success" ]]; then + : + else + exit 1 + fi + docker stop crawlab_master + - name: Log into registry - run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + - name: Push image run: | IMAGE_VERSION=${{needs.setup.outputs.version}} diff --git a/backend/go.mod b/backend/go.mod index 48f0d69e..907828a8 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.0-7 +require github.com/crawlab-team/crawlab-core v0.6.1-0.20220612043053-72fb6748a131 diff --git a/backend/go.sum b/backend/go.sum index 57afa5db..34e0c478 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -122,8 +122,50 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k= -github.com/crawlab-team/crawlab-core v0.6.0-7 h1:23kTlKOZ3OqfSmD+SYJiwlqP1Cu1NJtPmbR8KeIGvgw= -github.com/crawlab-team/crawlab-core v0.6.0-7/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.0-8 h1:P1l8dBdecZ6vVxwWwOwjLtCUriSNyBAe01Ghwfndoqw= +github.com/crawlab-team/crawlab-core v0.6.0-8/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.0-9 h1:oBwtAzafOVl2r2vLbLYClgpXDViwVVzqdXnbwC2aGj4= +github.com/crawlab-team/crawlab-core v0.6.0-9/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.0 h1:CGxdztjcIkozZp2EHXFN1brvKxIAdD0Xe1l4c+SFNE8= +github.com/crawlab-team/crawlab-core v0.6.0/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220609124710-a6fcaa2e7ea6 h1:DGXTxMCJhLZkbBZcwHE6wisUO96nsy5rTlV7e9pQ+Kg= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220609124710-a6fcaa2e7ea6/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220609133021-540f1f228a89 h1:8/9AsqjaWN19379M6Pr/KXHxvRKWOE/S7SbbpzrYpIc= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220609133021-540f1f228a89/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220609135233-fed5b2972049 h1:UDkyU63T8E/vvD2cc58d7YPKnH64w6qdlyi1MnwIWO8= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220609135233-fed5b2972049/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220610131302-39654de64d31 h1:Fws59e6GrWl3J4VcRJt4dBv0MIvXnzLoRt4ZA1xQ08Y= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220610131302-39654de64d31/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611025234-152e21d0e57f h1:M0OFYwG68cWydh5guI7aXJy/i8+vXLleUd5wOAZgTq0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611025234-152e21d0e57f/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611032702-c31cbc7d599b h1:oUwe+ap4+e0WhQ6JVZ6YPALfuVKhcYzeRTgyOeKPk28= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611032702-c31cbc7d599b/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611045456-7deb18a3b809 h1:0S7Mo1wqVrCF64yiTsuINBZWVzCZozvqihTu7XP/jkU= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611045456-7deb18a3b809/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611051011-90cbe93eda1a h1:4qC+7Q9rsebOz5zkTmSMmpjs7rN74HvDkiZ9FACi1cg= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611051011-90cbe93eda1a/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611083139-4b7bdcc24974 h1:/HNg9aJWhIxy9XZL4VYs8RFMGP2KE8GM1edfhoRd9pc= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611083139-4b7bdcc24974/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611084717-69dfd3bf6776 h1:M8stvkcx1fBdG8FfzmoRtsfnKjF46mXNFK/lDTTu+ZA= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611084717-69dfd3bf6776/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611090603-dfb61d4ad3a8 h1:/CgDdb3FITtXerFTzY84XyzN4nt4uDSJf7nDD6e7s/8= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611090603-dfb61d4ad3a8/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611092109-15a2a615d560 h1:HxU7s4AjYEIT0jvvePAfHM5yFGmVuRQunKqV/hP70hQ= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220611092109-15a2a615d560/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612024940-b4df67e569a6 h1:RpJsizVjCB4zx6AYf5cul3EgDInL3iax8qRO2CZSGJY= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612024940-b4df67e569a6/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612030521-bbc184169243 h1:04Ay5eUDB/PzjB+qUltdGp48uNBn1z5WXQGHS/uQ8Ug= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612030521-bbc184169243/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612031417-78e1024b1838 h1:akTxObujaS2XobGs7NbYBYtmZNzEBEasCvE/ru3KjOg= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612031417-78e1024b1838/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612033735-e730686bb127 h1:YjL1pDpBMhSMCfhhJqTUdYxHU5LGumZI0xfZfwVs1oI= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612033735-e730686bb127/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612034952-1cc047c47719 h1:TDeEOYyFkGuaSItG709k+qDwIi/08ZxgHXhz6ti0Pwk= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612034952-1cc047c47719/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612041650-37795c0ced68 h1:aDA3j5F6PoXy5hl1xD2vO9qtLLa+1JAIvpYf55vPq3k= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612041650-37795c0ced68/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612043053-72fb6748a131 h1:ZjXs65vYQcA26EC7qmDBA3ny3UHEdx/GNEXjVJWxmek= +github.com/crawlab-team/crawlab-core v0.6.1-0.20220612043053-72fb6748a131/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= diff --git a/bin/docker-init.sh b/bin/docker-init.sh index f15755db..452348b5 100755 --- a/bin/docker-init.sh +++ b/bin/docker-init.sh @@ -1,12 +1,3 @@ #!/bin/bash -if [ "${CRAWLAB_NODE_MASTER}" = "Y" ]; then - # start master - /bin/bash /app/bin/docker-start-master.sh - - # start crawlab - crawlab-server master -else - # start crawlab - crawlab-server worker -fi +crawlab-server server diff --git a/bin/docker-start-master.sh b/bin/docker-start-master.sh old mode 100755 new mode 100644 index fd271eec..e3720814 --- a/bin/docker-start-master.sh +++ b/bin/docker-start-master.sh @@ -1,23 +1,26 @@ +#!/bin/bash +# NOTE: deprecated + # replace env -indexpath=/app/dist/index.html +indexPath=/app/dist/index.html if test -z "$CRAWLAB_BASE_URL"; then - : + : else - sed -i "s?/js/?${CRAWLAB_BASE_URL}/js/?g" ${indexpath} - sed -i "s?/css/?${CRAWLAB_BASE_URL}/css/?g" ${indexpath} - sed -i "s/ > /var/log/weed.log 2>&1 & - + -dir /data \ + -master.dir ${seaweedfsDataPath} \ + -volume.dir.idx ${seaweedfsDataPath} \ + -ip localhost \ + -volume.port 9999 \ + -filer \ + >>/var/log/weed.log 2>&1 & diff --git a/docker-compose.yml b/docker-compose.yml index 4c517901..bc018590 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,34 +1,14 @@ version: '3.3' services: master: - image: crawlabteam/crawlab:latest + image: crawlabteam/crawlab container_name: crawlab_master environment: - CRAWLAB_NODE_MASTER: Y - CRAWLAB_MONGO_HOST: mongo - ports: - - "8080:8080" # frontend port mapping 前端端口映射 - depends_on: - - mongo - # volumes: - # - "/opt/crawlab/master:/data" # data persistent 持久化数据 - worker: - image: crawlabteam/crawlab:latest - container_name: crawlab_worker - environment: - CRAWLAB_NODE_MASTER: "N" + CRAWLAB_NODE_MASTER: "Y" CRAWLAB_MONGO_HOST: "mongo" + ports: + - "8080:8080" depends_on: - mongo - # volumes: - # - "/opt/crawlab/worker:/data" # data persistent 持久化数据 mongo: image: mongo:4.2 - #restart: always - # environment: - # MONGO_INITDB_ROOT_USERNAME: username - # MONGO_INITDB_ROOT_PASSWORD: password - # volumes: - # - "/opt/crawlab/mongo/data/db:/data/db" # data persistent 持久化数据 - # ports: - # - "27017:27017" # expose port to host machine 暴露接口到宿主机