crawlab/.github/workflows/docker-crawlab.yml

name: "Docker Image CI: crawlab"

on:
  push:
    branches: [ develop, main ]
  #pull_request:
  #  branches: [ main ]
  release:
    types: [ published ]
  workflow_dispatch:
  repository_dispatch:
    types: [ docker-crawlab ]

env:
  IMAGE_PATH_CRAWLAB_BACKEND: backend
  IMAGE_PATH_CRAWLAB_FRONTEND: frontend
  IMAGE_NAME_CRAWLAB: crawlabteam/crawlab
  IMAGE_NAME_CRAWLAB_BACKEND: crawlabteam/crawlab-backend
  IMAGE_NAME_CRAWLAB_FRONTEND: crawlabteam/crawlab-frontend
  IMAGE_NAME_CRAWLAB_TENCENT: ccr.ccs.tencentyun.com/crawlabteam/crawlab

jobs:
  setup:
    runs-on: ubuntu-latest
    outputs:
      is_matched_backend: ${{ steps.check_changed_files.outputs.is_matched_backend }}
      is_matched_frontend: ${{ steps.check_changed_files.outputs.is_matched_frontend }}
      is_matched_dockerfile: ${{ steps.check_changed_files.outputs.is_matched_dockerfile }}
      version: ${{ steps.version.outputs.version }}
    steps:
      - uses: actions/checkout@v2
      - name: Get changed files
        id: changed_files
        uses: tj-actions/changed-files@v18.7
      - id: check_changed_files
        name: Check changed files
        run: |
          # check changed files
          is_matched_backend=0
          is_matched_frontend=0
          is_matched_dockerfile=0
          for file in ${{ steps.changed_files.outputs.all_changed_files }}; do
            if [[ $file =~ ^(backend|core|db|fs|grpc|vcs|trace)/.* ]]; then
              is_matched_backend=1
            fi
            if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_FRONTEND}/.* ]]; then
              is_matched_frontend=1
            fi
            if [[ $file == Dockerfile ]]; then
              is_matched_dockerfile=1
            fi
          done
          
          # set outputs
          if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then
            is_matched_backend=1
            is_matched_frontend=1
            is_matched_dockerfile=1
          fi
          
          echo "::set-output name=is_matched_backend::$is_matched_backend"
          echo "::set-output name=is_matched_frontend::$is_matched_frontend"
          echo "::set-output name=is_matched_dockerfile::$is_matched_dockerfile"
          # echo outputs
          echo "is_matched_backend=$is_matched_backend"
          echo "is_matched_frontend=$is_matched_frontend"
          echo "is_matched_dockerfile=$is_matched_dockerfile"
          
      - id: version
        name: Get version
        run: |
          # Strip git ref prefix from version
          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
          
          # Strip "v" prefix from tag name
          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
          
          # Use Docker `latest` tag convention
          [ "$VERSION" == "main" ] && VERSION=latest
          
          echo "::set-output name=version::$VERSION"
     
  test_backend:
    needs: [ setup ]
    if: needs.setup.outputs.is_matched_backend == '1'
    runs-on: ubuntu-latest
    services:
      mongo:
        image: mongo:5
        ports:
          - 27017:27017
    steps:
      - uses: actions/checkout@v2
      - name: Set up Go
        uses: actions/setup-go@v2
        with:
          go-version: '1.22'
      - name: Run tests
        run: |
          cd core
          # Find all directories containing *_test.go files
          test_dirs=$(find . -name "*_test.go" -exec dirname {} \; | sort -u)

          # Run go test on each directory
          for dir in $test_dirs
          do
            echo "Running tests in $dir"
            go test ./$dir
          done

  build_backend:
    needs: [ setup, test_backend ]
    if: needs.setup.outputs.is_matched_backend == '1'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Build image
        run: |
          cd $IMAGE_PATH_CRAWLAB_BACKEND
          docker build . --file Dockerfile --tag image
      - name: Log into registry
        run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
      - name: Push image
        run: |
          IMAGE_VERSION=${{needs.setup.outputs.version}}
          IMAGE_NAME=$IMAGE_NAME_CRAWLAB_BACKEND
          docker tag image $IMAGE_NAME:$IMAGE_VERSION
          docker push $IMAGE_NAME:$IMAGE_VERSION

  build_frontend:
    needs: [ setup ]
    if: needs.setup.outputs.is_matched_frontend == '1'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Build image
        run: |
          cd $IMAGE_PATH_CRAWLAB_FRONTEND
          docker build . --file Dockerfile --tag image
      - name: Log into registry
        run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
      - name: Push image
        run: |
          IMAGE_VERSION=${{needs.setup.outputs.version}}
          IMAGE_NAME=$IMAGE_NAME_CRAWLAB_FRONTEND
          docker tag image $IMAGE_NAME:$IMAGE_VERSION
          docker push $IMAGE_NAME:$IMAGE_VERSION

  build_crawlab:
    needs: [setup, build_backend, build_frontend]
    if: |
      always() &&
      (needs.setup.outputs.is_matched_backend != '1' || needs.build_backend.result == 'success') &&
      (needs.setup.outputs.is_matched_frontend != '1' || needs.build_frontend.result == 'success')
    runs-on: ubuntu-latest
    outputs:
      test_passed: ${{ steps.build_and_test.outputs.test_passed }}
    services:
      mongo:
        image: mongo:5
        ports:
          - 27017:27017
    steps:
      - uses: actions/checkout@v2
      - name: Update Dockerfile
        run: |
          IMAGE_VERSION=${{needs.setup.outputs.version}}
          if [[ $IMAGE_VERSION != "latest" ]]; then
            for n in crawlab-backend crawlab-frontend; do
              IMAGE_NAME=$n
              sed -i "s/${IMAGE_NAME}:latest/${IMAGE_NAME}:${IMAGE_VERSION}/" Dockerfile
            done
          fi

      - name: Build image
        run: docker build . --file Dockerfile --tag image

      - name: Test image
        run: |
          docker run --rm -d --name crawlab_master \
            -e CRAWLAB_NODE_MASTER=true \
            -e CRAWLAB_DEMO=true \
            -e CRAWLAB_MONGO_HOST=localhost \
            -e CRAWLAB_MONGO_PORT=27017 \
            -p 8080:8080 \
            --network host \
            image
          docker exec crawlab_master env
          docker logs -f crawlab_master &
          sleep 10
          docker ps
          cmd='curl http://localhost:8080/api/system-info -s'
          echo "cmd: ${cmd}"
          res=`${cmd}`
          echo "res: ${res}"
          if [[ $res =~ "success" ]]; then
            :
          else
            exit 1
          fi
          docker stop crawlab_master

      - name: Set up Python
        uses: actions/setup-python@v1
        with:
          python-version: '3.12'

      # - name: Test demo
      #   run: |
      #     pip install crawlab-demo
      #     crawlab-demo validate

      - name: Build and test image
        id: build_and_test
        run: |
          # Update Dockerfile
          IMAGE_VERSION=${{needs.setup.outputs.version}}
          if [[ $IMAGE_VERSION != "latest" ]]; then
            for n in crawlab-backend crawlab-frontend; do
              IMAGE_NAME=$n
              sed -i "s/${IMAGE_NAME}:latest/${IMAGE_NAME}:${IMAGE_VERSION}/" Dockerfile
            done
          fi

          # Build image
          docker build . --file Dockerfile --tag image

          # Test image
          docker run --rm -d --name crawlab_master \
            -e CRAWLAB_NODE_MASTER=true \
            -e CRAWLAB_DEMO=true \
            -e CRAWLAB_MONGO_HOST=localhost \
            -e CRAWLAB_MONGO_PORT=27017 \
            -p 8080:8080 \
            --network host \
            image
          docker exec crawlab_master env
          docker logs -f crawlab_master &
          sleep 10
          docker ps
          cmd='curl http://localhost:8080/api/system-info -s'
          echo "cmd: ${cmd}"
          res=`${cmd}`
          echo "res: ${res}"
          docker stop crawlab_master
          if [[ $res =~ "success" ]]; then
            :
          else
            exit 1
          fi

  push_images:
    if: ${{ success() && needs.build_crawlab.result == 'success' }}
    needs: [setup, build_crawlab]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        registry: [dockerhub, tencent]
    steps:
      - name: Check out code
        uses: actions/checkout@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1

      - name: Login to DockerHub
        uses: docker/login-action@v1
        with:
          username: ${{ (matrix.registry == 'dockerhub' && secrets.DOCKER_USERNAME) || (matrix.registry == 'tencent' && secrets.DOCKER_TENCENT_USERNAME) }}
          password: ${{ (matrix.registry == 'dockerhub' && secrets.DOCKER_PASSWORD) || (matrix.registry == 'tencent' && secrets.DOCKER_TENCENT_PASSWORD) }}

      - name: Push to DockerHub
        uses: docker/build-push-action@v2
        with:
          push: true
          tags: ${{ (matrix.registry == 'dockerhub' && env.IMAGE_NAME_CRAWLAB) || (matrix.registry == 'tencent' && env.IMAGE_NAME_CRAWLAB_TENCENT) }}:${{ needs.setup.outputs.version }}