updated workflows

2026-01-21 17:21:09 +01:00 · 2022-06-03 15:04:54 +08:00
parent 9998cd52da
commit 22ad92dc00
7 changed files with 131 additions and 118 deletions
--- a/.github/workflows/docker-crawlab.yml
+++ b/.github/workflows/docker-crawlab.yml
@@ -0,0 +1,107 @@
+name: "Docker Image CI: crawlab"
+
+on:
+  push:
+    branches: [ develop, main ]
+  #pull_request:
+  #  branches: [ main ]
+  release:
+    types: [ published ]
+
+env:
+  IMAGE_PATH_CRAWLAB_BACKEND: backend
+  IMAGE_PATH_CRAWLAB_FRONTEND: frontend
+  IMAGE_NAME_CRAWLAB: crawlabteam/crawlab
+  IMAGE_NAME_CRAWLAB_BACKEND: crawlabteam/crawlab-backend
+  IMAGE_NAME_CRAWLAB_FRONTEND: crawlabteam/crawlab-frontend
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v18.7
+      - name: Check changed files
+        run: |
+          for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
+            if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_BACKEND}/.* ]]; then
+              echo "is_matched_backend=1" >> $GITHUB_ENV
+              break
+            fi
+            if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_FRONTEND}/.* ]]; then
+              echo "is_matched_frontend=1" >> $GITHUB_ENV
+              break
+            fi
+          done
+      - name: Get version
+        run: |
+          # Strip git ref prefix from version
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+          
+          # Strip "v" prefix from tag name
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+          
+          # Use Docker `latest` tag convention
+          [ "$VERSION" == "main" ] && VERSION=latest
+          
+          echo "IMAGE_VERSION=$VERSION" >> $GITHUB_ENV
+
+  build-backend:
+    if: ${{ env.is_matched_backend == '1' }}
+    needs: [ setup ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v18.7
+      - name: Build image
+        run: |
+          cd $IMAGE_PATH_CRAWLAB_BACKEND
+          docker build . --file Dockerfile --tag image
+      - name: Log into registry
+        run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+      - name: Push image
+        run: |
+          IMAGE_NAME=$IMAGE_NAME_CRAWLAB_BACKEND
+          docker tag image $IMAGE_NAME:$IMAGE_VERSION
+          docker push $IMAGE_NAME:$IMAGE_VERSION
+
+  build-frontend:
+    if: ${{ env.is_matched_frontend == '1' }}
+    needs: [ setup ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v18.7
+      - name: Build image
+        run: |
+          cd $IMAGE_PATH_CRAWLAB_FRONTEND
+          docker build . --file Dockerfile --tag image
+      - name: Log into registry
+        run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+      - name: Push image
+        run: |
+          IMAGE_NAME=$IMAGE_NAME_CRAWLAB_FRONTEND
+          docker tag image $IMAGE_NAME:$IMAGE_VERSION
+          docker push $IMAGE_NAME:$IMAGE_VERSION
+
+  build:
+    if: ${{ env.is_matched_backend == '1' || env.is_matched_frontend == '1' }}
+    needs: [ build-backend, build-frontend ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build image
+        run: docker build . --file Dockerfile --tag image
+      - name: Log into registry
+        run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+      - name: Push image
+        run: |
+          IMAGE_ID=$IMAGE_NAME_CRAWLAB
+          docker tag image $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION
--- a/.github/workflows/dockerpush.yml
+++ b/.github/workflows/dockerpush.yml
@@ -1,50 +0,0 @@
-name: Docker Image CI
-
-on:
-  push:
-    branches: [ develop, main ]
-  #pull_request:
-  #  branches: [ main ]
-  release:
-    types: [ published ]
-    
-env:
-  IMAGE_NAME: crawlabteam/crawlab
-  
-jobs:
-
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Build image
-      run: docker build . --file Dockerfile --tag image
-        
-    - name: Log into registry
-      run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
-
-    - name: Push image
-      run: |
-        IMAGE_ID=$IMAGE_NAME
-
-        # Strip git ref prefix from version
-        VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-
-        # Strip "v" prefix from tag name
-        [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
-
-        # Use Docker `latest` tag convention
-        [ "$VERSION" == "main" ] && VERSION=latest
-
-        echo IMAGE_ID=$IMAGE_ID
-        echo VERSION=$VERSION
-
-        docker tag image $IMAGE_ID:$VERSION
-        docker push $IMAGE_ID:$VERSION
-
-        if [[ $VERSION == "latest" ]]; then
-          docker tag image $IMAGE_ID:main
-          docker push $IMAGE_ID:main
-        fi
--- a/68
+++ b/68
@@ -1,52 +1,9 @@
-FROM golang:1.16 AS backend-build
+FROM crawlabteam/crawlab-backend:latest AS backend-build

-WORKDIR /go/src/app
-COPY ./backend .
-
-ENV GO111MODULE on
-#ENV GOPROXY https://goproxy.io
-
-RUN go mod tidy \
-  && go install -v ./...
-
-FROM node:12 AS frontend-build
-
-ADD ./frontend /app
-WORKDIR /app
-RUN rm /app/.npmrc
-
-# install frontend
-RUN yarn install && yarn run build:docker
+FROM crawlabteam/crawlab-frontend:latest AS frontend-build

 # images
-FROM ubuntu:20.04
-
-# set as non-interactive
-ENV DEBIAN_FRONTEND noninteractive
-
-# install packages
-RUN chmod 777 /tmp \
-	&& apt-get update \
-	&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate nginx wget dumb-init cloc
-
-# install python
-RUN apt-get install -y python3 python3-pip \
-	&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
-	&& ln -s /usr/bin/python3 /usr/local/bin/python
-
-# install golang
-RUN curl -OL https://golang.org/dl/go1.16.7.linux-amd64.tar.gz \
-	&& tar -C /usr/local -xvf go1.16.7.linux-amd64.tar.gz \
-	&& ln -s /usr/local/go/bin/go /usr/local/bin/go
-
-# install seaweedfs
-RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux_amd64.tar.gz \
-  && tar -zxf linux_amd64.tar.gz \
-  && cp weed /usr/local/bin
-
-# install backend
-RUN pip install scrapy pymongo bs4 requests
-RUN pip install crawlab-sdk==0.6.b20211224-1500
+FROM crawlabteam/crawlab-base:latest

 # add files
 COPY ./backend/conf /app/backend/conf
@@ -67,24 +24,5 @@ COPY ./nginx/crawlab.conf /etc/nginx/conf.d
 # install plugins
 RUN /bin/bash /app/bin/docker-install-plugins.sh

-# working directory
-WORKDIR /app/backend
-
-# timezone environment
-ENV TZ Asia/Shanghai
-
-# language environment
-ENV LC_ALL C.UTF-8
-ENV LANG C.UTF-8
-
-# docker
-ENV CRAWLAB_DOCKER Y
-
-# frontend port
-EXPOSE 8080
-
-# backend port
-EXPOSE 8000
-
 # start backend
 CMD ["/bin/bash", "/app/bin/docker-init.sh"]
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -0,0 +1,10 @@
+FROM golang:1.16
+
+WORKDIR /go/src/app
+COPY ./backend .
+
+ENV GO111MODULE on
+#ENV GOPROXY https://goproxy.io
+
+RUN go mod tidy \
+  && go install -v ./...
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -2,4 +2,4 @@ module crawlab

 go 1.16

-require github.com/crawlab-team/crawlab-core v0.6.0-3
+require github.com/crawlab-team/crawlab-core v0.6.0-4
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -122,8 +122,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k=
-github.com/crawlab-team/crawlab-core v0.6.0-3 h1:K4QtGSl3p+naaYf8hqupKagEpRDz17qD/zwugbFPhOI=
-github.com/crawlab-team/crawlab-core v0.6.0-3/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.0-4 h1:91Wsu+AS+0SEGgYlzhIjsRVrIFE5E0nU2kP0BJdBaRc=
+github.com/crawlab-team/crawlab-core v0.6.0-4/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
 github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78=
 github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k=
 github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU=
--- a/frontend/Dockerfile
+++ b/frontend/Dockerfile
@@ -0,0 +1,8 @@
+FROM node:12
+
+ADD ./frontend /app
+WORKDIR /app
+RUN rm /app/.npmrc
+
+# install frontend
+RUN yarn install && yarn run build:docker