ci: optimized workflows

.github/workflows/docker-crawlab-tencent.yml

@@ -1,197 +0,0 @@
-name: "Docker Image CI: crawlab (tencent)"
-
-on:
-  push:
-    branches: [ develop, main ]
-  #pull_request:
-  #  branches: [ main ]
-  release:
-    types: [ published ]
-  workflow_dispatch:
-  repository_dispatch:
-    types: [ docker-crawlab ]
-
-env:
-  IMAGE_PATH_CRAWLAB_BACKEND: backend
-  IMAGE_PATH_CRAWLAB_FRONTEND: frontend
-  IMAGE_NAME_CRAWLAB: ccr.ccs.tencentyun.com/crawlab/crawlab
-  IMAGE_NAME_CRAWLAB_BACKEND: crawlabteam/crawlab-backend
-  IMAGE_NAME_CRAWLAB_FRONTEND: crawlabteam/crawlab-frontend
-
-jobs:
-  setup:
-    runs-on: ubuntu-latest
-    outputs:
-      is_matched_backend: ${{ steps.check_changed_files.outputs.is_matched_backend }}
-      is_matched_frontend: ${{ steps.check_changed_files.outputs.is_matched_frontend }}
-      is_matched_dockerfile: ${{ steps.check_changed_files.outputs.is_matched_dockerfile }}
-      version: ${{ steps.version.outputs.version }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get changed files
-        id: changed_files
-        uses: tj-actions/changed-files@v18.7
-      - id: check_changed_files
-        name: Check changed files
-        run: |
-          # check changed files
-          is_matched_backend=0
-          is_matched_frontend=0
-          is_matched_dockerfile=0
-          for file in ${{ steps.changed_files.outputs.all_changed_files }}; do
-            if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_BACKEND}/.* ]]; then
-              file_backend=$file
-              is_matched_backend=1
-            fi
-            if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_FRONTEND}/.* ]]; then
-              file_frontend=$file
-              is_matched_frontend=1
-            fi
-            if [[ $file == Dockerfile ]]; then
-              file_dockerfile=$file
-              is_matched_dockerfile=1
-            fi
-          done
-          
-          # set outputs
-          if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then
-            is_matched_backend=1
-            is_matched_frontend=1
-            is_matched_dockerfile=1
-          fi
-          
-          echo "::set-output name=is_matched_backend::$is_matched_backend"
-          echo "::set-output name=is_matched_frontend::$is_matched_frontend"
-          echo "::set-output name=is_matched_dockerfile::$is_matched_dockerfile"
-          
-          # echo outputs
-          echo "is_matched_backend=$is_matched_backend, file_backend=$file_backend"
-          echo "is_matched_frontend=$is_matched_frontend, file_frontend=$file_frontend"
-          echo "is_matched_dockerfile=$is_matched_dockerfile, file_dockerfile=$file_dockerfile"
-          
-      - id: version
-        name: Get version
-        run: |
-          # Strip git ref prefix from version
-          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-          
-          # Strip "v" prefix from tag name
-          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
-          
-          # Use Docker `latest` tag convention
-          [ "$VERSION" == "main" ] && VERSION=latest
-          
-          echo "::set-output name=version::$VERSION"
-      
-  build-backend:
-    needs: [ setup ]
-    if: needs.setup.outputs.is_matched_backend == '1'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get changed files
-        id: changed-files
-        uses: tj-actions/changed-files@v18.7
-      - name: Build image
-        run: |
-          cd $IMAGE_PATH_CRAWLAB_BACKEND
-          docker build . --file Dockerfile --tag image
-      - name: Log into registry
-        run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
-      - name: Push image
-        run: |
-          IMAGE_VERSION=${{needs.setup.outputs.version}}
-          IMAGE_NAME=$IMAGE_NAME_CRAWLAB_BACKEND
-          docker tag image $IMAGE_NAME:$IMAGE_VERSION
-          docker push $IMAGE_NAME:$IMAGE_VERSION
-
-  build-frontend:
-    needs: [ setup ]
-    if: needs.setup.outputs.is_matched_frontend == '1'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get changed files
-        id: changed-files
-        uses: tj-actions/changed-files@v18.7
-      - name: Build image
-        run: |
-          cd $IMAGE_PATH_CRAWLAB_FRONTEND
-          docker build . --file Dockerfile --tag image
-      - name: Log into registry
-        run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
-      - name: Push image
-        run: |
-          IMAGE_VERSION=${{needs.setup.outputs.version}}
-          IMAGE_NAME=$IMAGE_NAME_CRAWLAB_FRONTEND
-          docker tag image $IMAGE_NAME:$IMAGE_VERSION
-          docker push $IMAGE_NAME:$IMAGE_VERSION
-
-  build-crawlab:
-    if: ${{ always() }}
-    needs: [ setup, build-backend, build-frontend ]
-    runs-on: ubuntu-latest
-    services:
-      mongo:
-        image: mongo:4.2
-        ports:
-          - 27017:27017
-    steps:
-      - uses: actions/checkout@v2
-      - name: Update Dockerfile
-        run: |
-          IMAGE_VERSION=${{needs.setup.outputs.version}}
-          if [[ $IMAGE_VERSION != "latest" ]]; then
-            for n in crawlab-backend crawlab-frontend; do
-              IMAGE_NAME=$n
-              sed -i "s/${IMAGE_NAME}:latest/${IMAGE_NAME}:${IMAGE_VERSION}/" Dockerfile
-            done
-          fi
-
-      - name: Build image
-        run: docker build . --file Dockerfile --tag image
-
-      - name: Test image
-        run: |
-          docker run --rm -d --name crawlab_master \
-            -e CRAWLAB_NODE_MASTER=true \
-            -e CRAWLAB_DEMO=true \
-            -e CRAWLAB_MONGO_HOST=localhost \
-            -e CRAWLAB_MONGO_PORT=27017 \
-            -p 8080:8080 \
-            --network host \
-            image
-          docker exec crawlab_master env
-          docker logs -f crawlab_master &
-          sleep 10
-          docker ps
-          cmd='curl http://localhost:8080/api/system-info -s'
-          echo "cmd: ${cmd}"
-          res=`${cmd}`
-          echo "res: ${res}"
-          if [[ $res =~ "success" ]]; then
-            :
-          else
-            exit 1
-          fi
-          docker stop crawlab_master
-
-      - name: Set up Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: '3.8'
-
-      - name: Test demo
-        run: |
-          pip install crawlab-demo
-          crawlab-demo validate
-
-      - name: Log into registry
-        run: echo ${{ secrets.DOCKER_TENCENT_PASSWORD }} | docker login -u ${{ secrets.DOCKER_TENCENT_USERNAME }} --password-stdin ccr.ccs.tencentyun.com
-
-      - name: Push image
-        run: |
-          IMAGE_VERSION=${{needs.setup.outputs.version}}
-          IMAGE_ID=$IMAGE_NAME_CRAWLAB
-          docker tag image $IMAGE_ID:$IMAGE_VERSION
-          docker push $IMAGE_ID:$IMAGE_VERSION

.github/workflows/docker-crawlab.yml

@@ -17,6 +17,7 @@ env:
   IMAGE_NAME_CRAWLAB: crawlabteam/crawlab
   IMAGE_NAME_CRAWLAB_BACKEND: crawlabteam/crawlab-backend
   IMAGE_NAME_CRAWLAB_FRONTEND: crawlabteam/crawlab-frontend
+  IMAGE_NAME_CRAWLAB_TENCENT: ccr.ccs.tencentyun.com/crawlabteam/crawlab
 
 jobs:
   setup:
@@ -131,6 +132,8 @@ jobs:
     if: ${{ always() }}
     needs: [ setup, build-backend, build-frontend ]
     runs-on: ubuntu-latest
+    outputs:
+      test_passed: ${{ steps.build_and_test.outputs.test_passed }}
     services:
       mongo:
         image: mongo:4.2
@@ -179,19 +182,76 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
-          python-version: '3.8'
+          python-version: '3.12'
 
       - name: Test demo
         run: |
           pip install crawlab-demo
           crawlab-demo validate
 
-      - name: Log into registry
-        run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
-
-      - name: Push image
+      - name: Build and test image
+        id: build_and_test
         run: |
+          # Update Dockerfile
           IMAGE_VERSION=${{needs.setup.outputs.version}}
-          IMAGE_ID=$IMAGE_NAME_CRAWLAB
-          docker tag image $IMAGE_ID:$IMAGE_VERSION
-          docker push $IMAGE_ID:$IMAGE_VERSION
+          if [[ $IMAGE_VERSION != "latest" ]]; then
+            for n in crawlab-backend crawlab-frontend; do
+              IMAGE_NAME=$n
+              sed -i "s/${IMAGE_NAME}:latest/${IMAGE_NAME}:${IMAGE_VERSION}/" Dockerfile
+            done
+          fi
+
+          # Build image
+          docker build . --file Dockerfile --tag image
+
+          # Test image
+          docker run --rm -d --name crawlab_master \
+            -e CRAWLAB_NODE_MASTER=true \
+            -e CRAWLAB_DEMO=true \
+            -e CRAWLAB_MONGO_HOST=localhost \
+            -e CRAWLAB_MONGO_PORT=27017 \
+            -p 8080:8080 \
+            --network host \
+            image
+          docker exec crawlab_master env
+          docker logs -f crawlab_master &
+          sleep 10
+          docker ps
+          cmd='curl http://localhost:8080/api/system-info -s'
+          echo "cmd: ${cmd}"
+          res=`${cmd}`
+          echo "res: ${res}"
+          if [[ $res =~ "success" ]]; then
+            echo "test_passed=true" >> $GITHUB_OUTPUT
+          else
+            echo "test_passed=false" >> $GITHUB_OUTPUT
+            exit 1
+          fi
+          docker stop crawlab_master
+
+  push-images:
+    needs: [setup, build-crawlab]
+    if: needs.build-crawlab.outputs.test_passed == 'true'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        registry: [dockerhub, tencent]
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ (matrix.registry == 'dockerhub' && secrets.DOCKER_USERNAME) || (matrix.registry == 'tencent' && secrets.DOCKER_TENCENT_USERNAME) }}
+          password: ${{ (matrix.registry == 'dockerhub' && secrets.DOCKER_PASSWORD) || (matrix.registry == 'tencent' && secrets.DOCKER_TENCENT_PASSWORD) }}
+
+      - name: Push to DockerHub
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: ${{ (matrix.registry == 'dockerhub' && env.IMAGE_NAME_CRAWLAB) || (matrix.registry == 'tencent' && env.IMAGE_NAME_CRAWLAB_TENCENT) }}:${{ needs.setup.outputs.version }}
+