From 247ce3c4e21899524926825ed80f47ce45745c72 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 16 Oct 2024 20:46:35 +0800 Subject: [PATCH] ci: optimized workflows --- .github/workflows/docker-crawlab-tencent.yml | 197 ------------------- .github/workflows/docker-crawlab.yml | 76 ++++++- 2 files changed, 68 insertions(+), 205 deletions(-) delete mode 100644 .github/workflows/docker-crawlab-tencent.yml diff --git a/.github/workflows/docker-crawlab-tencent.yml b/.github/workflows/docker-crawlab-tencent.yml deleted file mode 100644 index c1e0a8e9..00000000 --- a/.github/workflows/docker-crawlab-tencent.yml +++ /dev/null @@ -1,197 +0,0 @@ -name: "Docker Image CI: crawlab (tencent)" - -on: - push: - branches: [ develop, main ] - #pull_request: - # branches: [ main ] - release: - types: [ published ] - workflow_dispatch: - repository_dispatch: - types: [ docker-crawlab ] - -env: - IMAGE_PATH_CRAWLAB_BACKEND: backend - IMAGE_PATH_CRAWLAB_FRONTEND: frontend - IMAGE_NAME_CRAWLAB: ccr.ccs.tencentyun.com/crawlab/crawlab - IMAGE_NAME_CRAWLAB_BACKEND: crawlabteam/crawlab-backend - IMAGE_NAME_CRAWLAB_FRONTEND: crawlabteam/crawlab-frontend - -jobs: - setup: - runs-on: ubuntu-latest - outputs: - is_matched_backend: ${{ steps.check_changed_files.outputs.is_matched_backend }} - is_matched_frontend: ${{ steps.check_changed_files.outputs.is_matched_frontend }} - is_matched_dockerfile: ${{ steps.check_changed_files.outputs.is_matched_dockerfile }} - version: ${{ steps.version.outputs.version }} - steps: - - uses: actions/checkout@v2 - - name: Get changed files - id: changed_files - uses: tj-actions/changed-files@v18.7 - - id: check_changed_files - name: Check changed files - run: | - # check changed files - is_matched_backend=0 - is_matched_frontend=0 - is_matched_dockerfile=0 - for file in ${{ steps.changed_files.outputs.all_changed_files }}; do - if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_BACKEND}/.* ]]; then - file_backend=$file - is_matched_backend=1 - fi - if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_FRONTEND}/.* ]]; then - file_frontend=$file - is_matched_frontend=1 - fi - if [[ $file == Dockerfile ]]; then - file_dockerfile=$file - is_matched_dockerfile=1 - fi - done - - # set outputs - if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then - is_matched_backend=1 - is_matched_frontend=1 - is_matched_dockerfile=1 - fi - - echo "::set-output name=is_matched_backend::$is_matched_backend" - echo "::set-output name=is_matched_frontend::$is_matched_frontend" - echo "::set-output name=is_matched_dockerfile::$is_matched_dockerfile" - - # echo outputs - echo "is_matched_backend=$is_matched_backend, file_backend=$file_backend" - echo "is_matched_frontend=$is_matched_frontend, file_frontend=$file_frontend" - echo "is_matched_dockerfile=$is_matched_dockerfile, file_dockerfile=$file_dockerfile" - - - id: version - name: Get version - run: | - # Strip git ref prefix from version - VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - - # Strip "v" prefix from tag name - [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') - - # Use Docker `latest` tag convention - [ "$VERSION" == "main" ] && VERSION=latest - - echo "::set-output name=version::$VERSION" - - build-backend: - needs: [ setup ] - if: needs.setup.outputs.is_matched_backend == '1' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v18.7 - - name: Build image - run: | - cd $IMAGE_PATH_CRAWLAB_BACKEND - docker build . --file Dockerfile --tag image - - name: Log into registry - run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - - name: Push image - run: | - IMAGE_VERSION=${{needs.setup.outputs.version}} - IMAGE_NAME=$IMAGE_NAME_CRAWLAB_BACKEND - docker tag image $IMAGE_NAME:$IMAGE_VERSION - docker push $IMAGE_NAME:$IMAGE_VERSION - - build-frontend: - needs: [ setup ] - if: needs.setup.outputs.is_matched_frontend == '1' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v18.7 - - name: Build image - run: | - cd $IMAGE_PATH_CRAWLAB_FRONTEND - docker build . --file Dockerfile --tag image - - name: Log into registry - run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - - name: Push image - run: | - IMAGE_VERSION=${{needs.setup.outputs.version}} - IMAGE_NAME=$IMAGE_NAME_CRAWLAB_FRONTEND - docker tag image $IMAGE_NAME:$IMAGE_VERSION - docker push $IMAGE_NAME:$IMAGE_VERSION - - build-crawlab: - if: ${{ always() }} - needs: [ setup, build-backend, build-frontend ] - runs-on: ubuntu-latest - services: - mongo: - image: mongo:4.2 - ports: - - 27017:27017 - steps: - - uses: actions/checkout@v2 - - name: Update Dockerfile - run: | - IMAGE_VERSION=${{needs.setup.outputs.version}} - if [[ $IMAGE_VERSION != "latest" ]]; then - for n in crawlab-backend crawlab-frontend; do - IMAGE_NAME=$n - sed -i "s/${IMAGE_NAME}:latest/${IMAGE_NAME}:${IMAGE_VERSION}/" Dockerfile - done - fi - - - name: Build image - run: docker build . --file Dockerfile --tag image - - - name: Test image - run: | - docker run --rm -d --name crawlab_master \ - -e CRAWLAB_NODE_MASTER=true \ - -e CRAWLAB_DEMO=true \ - -e CRAWLAB_MONGO_HOST=localhost \ - -e CRAWLAB_MONGO_PORT=27017 \ - -p 8080:8080 \ - --network host \ - image - docker exec crawlab_master env - docker logs -f crawlab_master & - sleep 10 - docker ps - cmd='curl http://localhost:8080/api/system-info -s' - echo "cmd: ${cmd}" - res=`${cmd}` - echo "res: ${res}" - if [[ $res =~ "success" ]]; then - : - else - exit 1 - fi - docker stop crawlab_master - - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: '3.8' - - - name: Test demo - run: | - pip install crawlab-demo - crawlab-demo validate - - - name: Log into registry - run: echo ${{ secrets.DOCKER_TENCENT_PASSWORD }} | docker login -u ${{ secrets.DOCKER_TENCENT_USERNAME }} --password-stdin ccr.ccs.tencentyun.com - - - name: Push image - run: | - IMAGE_VERSION=${{needs.setup.outputs.version}} - IMAGE_ID=$IMAGE_NAME_CRAWLAB - docker tag image $IMAGE_ID:$IMAGE_VERSION - docker push $IMAGE_ID:$IMAGE_VERSION diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index f447a764..517ff609 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -17,6 +17,7 @@ env: IMAGE_NAME_CRAWLAB: crawlabteam/crawlab IMAGE_NAME_CRAWLAB_BACKEND: crawlabteam/crawlab-backend IMAGE_NAME_CRAWLAB_FRONTEND: crawlabteam/crawlab-frontend + IMAGE_NAME_CRAWLAB_TENCENT: ccr.ccs.tencentyun.com/crawlabteam/crawlab jobs: setup: @@ -131,6 +132,8 @@ jobs: if: ${{ always() }} needs: [ setup, build-backend, build-frontend ] runs-on: ubuntu-latest + outputs: + test_passed: ${{ steps.build_and_test.outputs.test_passed }} services: mongo: image: mongo:4.2 @@ -179,19 +182,76 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: '3.8' + python-version: '3.12' - name: Test demo run: | pip install crawlab-demo crawlab-demo validate - - name: Log into registry - run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - - - name: Push image + - name: Build and test image + id: build_and_test run: | + # Update Dockerfile IMAGE_VERSION=${{needs.setup.outputs.version}} - IMAGE_ID=$IMAGE_NAME_CRAWLAB - docker tag image $IMAGE_ID:$IMAGE_VERSION - docker push $IMAGE_ID:$IMAGE_VERSION + if [[ $IMAGE_VERSION != "latest" ]]; then + for n in crawlab-backend crawlab-frontend; do + IMAGE_NAME=$n + sed -i "s/${IMAGE_NAME}:latest/${IMAGE_NAME}:${IMAGE_VERSION}/" Dockerfile + done + fi + + # Build image + docker build . --file Dockerfile --tag image + + # Test image + docker run --rm -d --name crawlab_master \ + -e CRAWLAB_NODE_MASTER=true \ + -e CRAWLAB_DEMO=true \ + -e CRAWLAB_MONGO_HOST=localhost \ + -e CRAWLAB_MONGO_PORT=27017 \ + -p 8080:8080 \ + --network host \ + image + docker exec crawlab_master env + docker logs -f crawlab_master & + sleep 10 + docker ps + cmd='curl http://localhost:8080/api/system-info -s' + echo "cmd: ${cmd}" + res=`${cmd}` + echo "res: ${res}" + if [[ $res =~ "success" ]]; then + echo "test_passed=true" >> $GITHUB_OUTPUT + else + echo "test_passed=false" >> $GITHUB_OUTPUT + exit 1 + fi + docker stop crawlab_master + + push-images: + needs: [setup, build-crawlab] + if: needs.build-crawlab.outputs.test_passed == 'true' + runs-on: ubuntu-latest + strategy: + matrix: + registry: [dockerhub, tencent] + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ (matrix.registry == 'dockerhub' && secrets.DOCKER_USERNAME) || (matrix.registry == 'tencent' && secrets.DOCKER_TENCENT_USERNAME) }} + password: ${{ (matrix.registry == 'dockerhub' && secrets.DOCKER_PASSWORD) || (matrix.registry == 'tencent' && secrets.DOCKER_TENCENT_PASSWORD) }} + + - name: Push to DockerHub + uses: docker/build-push-action@v2 + with: + push: true + tags: ${{ (matrix.registry == 'dockerhub' && env.IMAGE_NAME_CRAWLAB) || (matrix.registry == 'tencent' && env.IMAGE_NAME_CRAWLAB_TENCENT) }}:${{ needs.setup.outputs.version }} +