name: "Docker Image CI: crawlab" on: push: branches: [ develop, main ] release: types: [ published ] workflow_dispatch: repository_dispatch: types: [ docker-crawlab ] env: IMAGE_PATH_CRAWLAB_BACKEND: backend IMAGE_PATH_CRAWLAB_FRONTEND: frontend GH_PKG_NAME_CRAWLAB_BACKEND: ${{ github.repository_owner }}/crawlab-backend GH_PKG_NAME_CRAWLAB_FRONTEND: ${{ github.repository_owner }}/crawlab-frontend GH_PKG_NAME_CRAWLAB: ${{ github.repository_owner }}/crawlab IMAGE_NAME_CRAWLAB_BACKEND: ghcr.io/${{ github.repository_owner }}/crawlab-backend IMAGE_NAME_CRAWLAB_FRONTEND: ghcr.io/${{ github.repository_owner }}/crawlab-frontend IMAGE_NAME_CRAWLAB: ghcr.io/${{ github.repository_owner }}/crawlab IMAGE_NAME_CRAWLAB_DOCKERHUB: crawlabteam/crawlab IMAGE_NAME_CRAWLAB_TENCENT: ccr.ccs.tencentyun.com/crawlab/crawlab jobs: setup: runs-on: ubuntu-latest outputs: workflow_changed: ${{ steps.check_changed_files.outputs.workflow_changed }} backend_changed: ${{ steps.check_changed_files.outputs.backend_changed }} frontend_changed: ${{ steps.check_changed_files.outputs.frontend_changed }} dockerfile_changed: ${{ steps.check_changed_files.outputs.dockerfile_changed }} version: ${{ steps.version.outputs.version }} image_hash: ${{ steps.hash.outputs.hash }} steps: - uses: actions/checkout@v4 - name: Get changed files id: changed_files uses: tj-actions/changed-files@v45 with: files_yaml: | backend: - 'backend/**' - 'core/**' - 'db/**' - 'fs/**' - 'grpc/**' - 'vcs/**' - 'trace/**' frontend: - '${{ env.IMAGE_PATH_CRAWLAB_FRONTEND }}/**' dockerfile: - 'Dockerfile' workflow: - '.github/workflows/docker-crawlab.yml' - id: check_changed_files name: Check changed files run: | # set outputs if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then echo "backend_changed=true" >> $GITHUB_OUTPUT echo "frontend_changed=true" >> $GITHUB_OUTPUT echo "dockerfile_changed=true" >> $GITHUB_OUTPUT else echo "backend_changed=${{ steps.changed_files.outputs.backend_any_changed }}" >> $GITHUB_OUTPUT echo "frontend_changed=${{ steps.changed_files.outputs.frontend_any_changed }}" >> $GITHUB_OUTPUT echo "dockerfile_changed=${{ steps.changed_files.outputs.dockerfile_any_changed }}" >> $GITHUB_OUTPUT echo "workflow_changed=${{ steps.changed_files.outputs.workflow_any_changed }}" >> $GITHUB_OUTPUT fi # Display change status for each component echo "Backend changed: ${{ steps.changed_files.outputs.backend_any_changed }}" echo "Frontend changed: ${{ steps.changed_files.outputs.frontend_any_changed }}" echo "Dockerfile changed: ${{ steps.changed_files.outputs.dockerfile_any_changed }}" echo "Workflow changed: ${{ steps.changed_files.outputs.workflow_any_changed }}" - id: version name: Get version run: | # Strip git ref prefix from version VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') # Strip "v" prefix from tag name [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') # Use Docker `latest` tag convention [ "$VERSION" == "main" ] && VERSION=latest echo "version=$VERSION" >> $GITHUB_OUTPUT - id: hash name: Generate unique hash run: echo "hash=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT test_backend: needs: [ setup ] if: needs.setup.outputs.backend_changed == 'true' || needs.setup.outputs.workflow_changed == 'true' runs-on: ubuntu-latest services: mongo: image: mongo:5 ports: - 27017:27017 steps: - uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v5 with: go-version-file: 'core/go.mod' cache-dependency-path: 'core/go.sum' - name: Run tests working-directory: core run: | # Find all directories containing *_test.go files test_dirs=$(find . -name "*_test.go" -exec dirname {} \; | sort -u) # Run go test on each directory for dir in $test_dirs do echo "Running tests in $dir" go test ./$dir done build_backend: needs: [ setup, test_backend ] if: needs.test_backend.result == 'success' || needs.setup.outputs.workflow_changed == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Log in to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push image uses: docker/build-push-action@v5 with: context: ${{ env.IMAGE_PATH_CRAWLAB_BACKEND }} push: true tags: ${{ env.IMAGE_NAME_CRAWLAB_BACKEND }}:${{ needs.setup.outputs.image_hash }} build_frontend: needs: [ setup ] if: needs.setup.outputs.frontend_changed == 'true' || needs.setup.outputs.workflow_changed == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Log in to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push image uses: docker/build-push-action@v5 with: context: ${{ env.IMAGE_PATH_CRAWLAB_FRONTEND }} push: true tags: ${{ env.IMAGE_NAME_CRAWLAB_FRONTEND }}:${{ needs.setup.outputs.image_hash }} build_crawlab: needs: [setup, build_backend, build_frontend] if: | always() && ((needs.setup.outputs.backend_changed == 'true' && needs.build_backend.result == 'success') || (needs.setup.outputs.frontend_changed == 'true' && needs.build_frontend.result == 'success') || (needs.setup.outputs.dockerfile_changed == 'true') || (needs.setup.outputs.workflow_changed == 'true')) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Update Dockerfile run: | for name in crawlab-backend crawlab-frontend; do IMAGE_NAME=ghcr.io/crawlab-team/crawlab/$name sed -i "s|crawlabteam/${name}:latest|${IMAGE_NAME}:${{ needs.setup.outputs.image_hash }}|" Dockerfile done - name: Log in to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push image uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile push: true tags: | ${{ env.IMAGE_NAME_CRAWLAB }}:${{ needs.setup.outputs.image_hash }} test_crawlab: needs: [setup, build_crawlab] if: ${{ always() && needs.build_crawlab.result == 'success' }} runs-on: ubuntu-latest services: mongo: image: mongo:5 options: >- --health-cmd "mongosh --eval 'db.adminCommand(\"ping\")' || exit 1" --health-interval 10s --health-timeout 5s --health-retries 5 crawlab: image: ghcr.io/${{ github.repository }}:${{ needs.setup.outputs.image_hash }} env: CRAWLAB_NODE_MASTER: Y CRAWLAB_MONGO_HOST: mongo CRAWLAB_MONGO_PORT: 27017 ports: - 8080:8080 options: >- --health-cmd "curl -f http://localhost:8080/api/system-info || exit 1" --health-interval 30s --health-timeout 10s --health-retries 5 steps: - uses: actions/checkout@v4 with: submodules: 'recursive' - name: Setup Node.js uses: actions/setup-node@v3 with: node-version: '20' - name: Install pnpm run: npm install -g pnpm@9 - name: Install dependencies working-directory: tests/e2e run: pnpm install - name: Install Playwright browser id: install_playwright working-directory: tests/e2e run: pnpm playwright install --with-deps - name: Run Playwright tests id: run_playwright working-directory: tests/e2e run: pnpm run test - name: Deploy to GitHub Pages if: ${{ always() && steps.install_playwright.outcome == 'success' }} uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: tests/e2e/playwright-report destination_dir: playwright-report/${{ needs.setup.outputs.version }} push_images: if: ${{ always() && needs.test_crawlab.result == 'success' }} needs: [setup, test_crawlab] runs-on: ubuntu-latest strategy: matrix: registry: [dockerhub, tencent] steps: - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Pull Docker image from GitHub Container Registry run: docker pull ${{ env.IMAGE_NAME_CRAWLAB }}:${{ needs.setup.outputs.image_hash }} - name: Login to DockerHub or Tencent Registry uses: docker/login-action@v3 with: registry: ${{ (matrix.registry == 'dockerhub' && 'https://index.docker.io/v1/') || (matrix.registry == 'tencent' && 'ccr.ccs.tencentyun.com') }} username: ${{ (matrix.registry == 'dockerhub' && secrets.DOCKER_USERNAME) || (matrix.registry == 'tencent' && secrets.DOCKER_TENCENT_USERNAME) }} password: ${{ (matrix.registry == 'dockerhub' && secrets.DOCKER_PASSWORD) || (matrix.registry == 'tencent' && secrets.DOCKER_TENCENT_PASSWORD) }} - name: Tag and push image run: | docker tag ${{ env.IMAGE_NAME_CRAWLAB }}:${{ needs.setup.outputs.image_hash }} ${{ (matrix.registry == 'dockerhub' && env.IMAGE_NAME_CRAWLAB_DOCKERHUB) || (matrix.registry == 'tencent' && env.IMAGE_NAME_CRAWLAB_TENCENT) }}:${{ needs.setup.outputs.version }} docker push ${{ (matrix.registry == 'dockerhub' && env.IMAGE_NAME_CRAWLAB_DOCKERHUB) || (matrix.registry == 'tencent' && env.IMAGE_NAME_CRAWLAB_TENCENT) }}:${{ needs.setup.outputs.version }} cleanup: needs: [setup, build_backend, build_frontend, build_crawlab, push_images] if: always() runs-on: ubuntu-latest strategy: matrix: image: [ { name: 'crawlab' }, { name: 'crawlab/crawlab-backend' }, { name: 'crawlab/crawlab-frontend' } ] steps: - name: List Docker image versions if: | ${{ (matrix.image.name == 'crawlab' && needs.build_crawlab.result == 'success') || (matrix.image.name == 'crawlab-backend' && needs.build_backend.result == 'success') || (matrix.image.name == 'crawlab-frontend' && needs.build_frontend.result == 'success') }} id: list_versions run: | curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ https://api.github.com/orgs/${{ github.repository_owner }}/packages/container/${{ matrix.image.name }}/versions > versions.json cat versions.json jq --arg tag "${{ needs.setup.outputs.image_hash }}" '.[] | select(.metadata.container.tags[] | contains($tag)) | .id' versions.json > version_ids.txt - name: Delete Docker image versions uses: actions/delete-package-versions@v5 with: package-name: "${{ matrix.image.name }}:${{ needs.setup.outputs.image_hash }}" package-type: 'container' token: ${{ secrets.GITHUB_TOKEN }} package-version-ids: $(cat version_ids.txt)