Merge pull request #1117 from crawlab-team/develop

Develop

.github/workflows/docker-crawlab.yml

@@ -0,0 +1,145 @@
+name: "Docker Image CI: crawlab"
+
+on:
+  push:
+    branches: [ develop, main ]
+  #pull_request:
+  #  branches: [ main ]
+  release:
+    types: [ published ]
+
+env:
+  IMAGE_PATH_CRAWLAB_BACKEND: backend
+  IMAGE_PATH_CRAWLAB_FRONTEND: frontend
+  IMAGE_NAME_CRAWLAB: crawlabteam/crawlab
+  IMAGE_NAME_CRAWLAB_BACKEND: crawlabteam/crawlab-backend
+  IMAGE_NAME_CRAWLAB_FRONTEND: crawlabteam/crawlab-frontend
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      is_matched_backend: ${{ steps.check_changed_files.outputs.is_matched_backend }}
+      is_matched_frontend: ${{ steps.check_changed_files.outputs.is_matched_frontend }}
+      is_matched_dockerfile: ${{ steps.check_changed_files.outputs.is_matched_dockerfile }}
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get changed files
+        id: changed_files
+        uses: tj-actions/changed-files@v18.7
+      - id: check_changed_files
+        name: Check changed files
+        run: |
+          # check changed files
+          is_matched_backend=0
+          is_matched_frontend=0
+          is_matched_dockerfile=0
+          for file in ${{ steps.changed_files.outputs.all_changed_files }}; do
+            if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_BACKEND}/.* ]]; then
+              file_backend=$file
+              is_matched_backend=1
+            fi
+            if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_FRONTEND}/.* ]]; then
+              file_frontend=$file
+              is_matched_frontend=1
+            fi
+            if [[ $file == Dockerfile ]]; then
+              file_dockerfile=$file
+              is_matched_dockerfile=1
+            fi
+          done
+          
+          # set outputs
+          echo "::set-output name=is_matched_backend::$is_matched_backend"
+          echo "::set-output name=is_matched_frontend::$is_matched_frontend"
+          echo "::set-output name=is_matched_dockerfile::$is_matched_dockerfile"
+          
+          # echo outputs
+          echo "is_matched_backend=$is_matched_backend, file_backend=$file_backend"
+          echo "is_matched_frontend=$is_matched_frontend, file_frontend=$file_frontend"
+          echo "is_matched_dockerfile=$is_matched_dockerfile, file_dockerfile=$file_dockerfile"
+      - id: version
+        name: Get version
+        run: |
+          # Strip git ref prefix from version
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+          
+          # Strip "v" prefix from tag name
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+          
+          # Use Docker `latest` tag convention
+          [ "$VERSION" == "main" ] && VERSION=latest
+          
+          echo "::set-output name=version::$VERSION"
+
+  build-backend:
+    needs: [ setup ]
+    if: needs.setup.outputs.is_matched_backend == '1'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v18.7
+      - name: Build image
+        run: |
+          cd $IMAGE_PATH_CRAWLAB_BACKEND
+          docker build . --file Dockerfile --tag image
+      - name: Log into registry
+        run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+      - name: Push image
+        run: |
+          IMAGE_VERSION=${{needs.setup.outputs.version}}
+          IMAGE_NAME=$IMAGE_NAME_CRAWLAB_BACKEND
+          docker tag image $IMAGE_NAME:$IMAGE_VERSION
+          docker push $IMAGE_NAME:$IMAGE_VERSION
+
+  build-frontend:
+    needs: [ setup ]
+    if: needs.setup.outputs.is_matched_frontend == '1'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v18.7
+      - name: Build image
+        run: |
+          cd $IMAGE_PATH_CRAWLAB_FRONTEND
+          docker build . --file Dockerfile --tag image
+      - name: Log into registry
+        run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+      - name: Push image
+        run: |
+          IMAGE_VERSION=${{needs.setup.outputs.version}}
+          IMAGE_NAME=$IMAGE_NAME_CRAWLAB_FRONTEND
+          docker tag image $IMAGE_NAME:$IMAGE_VERSION
+          docker push $IMAGE_NAME:$IMAGE_VERSION
+
+  build-crawlab:
+    if: ${{ always() }}
+    needs: [ setup, build-backend, build-frontend ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Update Dockerfile
+        run: |
+          IMAGE_VERSION=${{needs.setup.outputs.version}}
+          if [[ $IMAGE_VERSION != "latest" ]]; then
+            for n in crawlab-backend crawlab-frontend; do
+              IMAGE_NAME=$n
+              sed -i "s/${IMAGE_NAME}:latest/${IMAGE_NAME}:${IMAGE_VERSION}/" Dockerfile
+            done
+          fi
+
+      - name: Build image
+        run: docker build . --file Dockerfile --tag image
+      - name: Log into registry
+        run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+      - name: Push image
+        run: |
+          IMAGE_VERSION=${{needs.setup.outputs.version}}
+          IMAGE_ID=$IMAGE_NAME_CRAWLAB
+          docker tag image $IMAGE_ID:$IMAGE_VERSION
+          docker push $IMAGE_ID:$IMAGE_VERSION

.github/workflows/dockerpush.yml

@@ -1,50 +0,0 @@
-name: Docker Image CI
-
-on:
-  push:
-    branches: [ develop, main ]
-  #pull_request:
-  #  branches: [ main ]
-  release:
-    types: [ published ]
-    
-env:
-  IMAGE_NAME: crawlabteam/crawlab
-  
-jobs:
-
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Build image
-      run: docker build . --file Dockerfile --tag image
-        
-    - name: Log into registry
-      run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
-
-    - name: Push image
-      run: |
-        IMAGE_ID=$IMAGE_NAME
-
-        # Strip git ref prefix from version
-        VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-
-        # Strip "v" prefix from tag name
-        [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
-
-        # Use Docker `latest` tag convention
-        [ "$VERSION" == "main" ] && VERSION=latest
-
-        echo IMAGE_ID=$IMAGE_ID
-        echo VERSION=$VERSION
-
-        docker tag image $IMAGE_ID:$VERSION
-        docker push $IMAGE_ID:$VERSION
-
-        if [[ $VERSION == "latest" ]]; then
-          docker tag image $IMAGE_ID:main
-          docker push $IMAGE_ID:main
-        fi

Dockerfile

@@ -1,55 +1,11 @@
-FROM golang:1.16 AS backend-build
+FROM crawlabteam/crawlab-backend:latest AS backend-build
 
-WORKDIR /go/src/app
+FROM crawlabteam/crawlab-frontend:latest AS frontend-build
-COPY ./backend .
 
-ENV GO111MODULE on
+FROM crawlabteam/crawlab-public-plugins:latest AS public-plugins-build
-#ENV GOPROXY https://goproxy.io
-
-RUN go mod tidy \
-  && go install -v ./...
-
-FROM node:12 AS frontend-build
-
-ADD ./frontend /app
-WORKDIR /app
-RUN rm /app/.npmrc
-
-# install frontend
-RUN yarn install && yarn run build:docker
 
 # images
-FROM ubuntu:20.04
+FROM crawlabteam/crawlab-base:latest
-
-# set as non-interactive
-ENV DEBIAN_FRONTEND noninteractive
-
-# set CRAWLAB_IS_DOCKER
-ENV CRAWLAB_IS_DOCKER Y
-
-# install packages
-RUN chmod 777 /tmp \
-	&& apt-get update \
-	&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate nginx wget dumb-init cloc
-
-# install python
-RUN apt-get install -y python3 python3-pip \
-	&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
-	&& ln -s /usr/bin/python3 /usr/local/bin/python
-
-# install golang
-RUN curl -OL https://golang.org/dl/go1.16.7.linux-amd64.tar.gz \
-	&& tar -C /usr/local -xvf go1.16.7.linux-amd64.tar.gz \
-	&& ln -s /usr/local/go/bin/go /usr/local/bin/go
-
-# install seaweedfs
-RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux_amd64.tar.gz \
-  && tar -zxf linux_amd64.tar.gz \
-  && cp weed /usr/local/bin
-
-# install backend
-RUN pip install scrapy pymongo bs4 requests
-RUN pip install crawlab-sdk==0.6.b20211224-1500
 
 # add files
 COPY ./backend/conf /app/backend/conf
@@ -64,24 +20,11 @@ RUN cp /opt/bin/crawlab /usr/local/bin/crawlab-server
 # copy frontend files
 COPY --from=frontend-build /app/dist /app/dist
 
+# copy public-plugins files
+COPY --from=public-plugins-build /app/plugins /app/plugins
+
 # copy nginx config files
 COPY ./nginx/crawlab.conf /etc/nginx/conf.d
 
-# working directory
-WORKDIR /app/backend
-
-# timezone environment
-ENV TZ Asia/Shanghai
-
-# language environment
-ENV LC_ALL C.UTF-8
-ENV LANG C.UTF-8
-
-# frontend port
-EXPOSE 8080
-
-# backend port
-EXPOSE 8000
-
 # start backend
 CMD ["/bin/bash", "/app/bin/docker-init.sh"]

Dockerfile.cn

@@ -24,9 +24,6 @@ FROM ubuntu:20.04
 # set as non-interactive
 ENV DEBIAN_FRONTEND noninteractive
 
-# set CRAWLAB_IS_DOCKER
-ENV CRAWLAB_IS_DOCKER Y
-
 # install packages
 RUN chmod 777 /tmp \
 	&& apt-get update \
@@ -67,6 +64,9 @@ COPY --from=frontend-build /app/dist /app/dist
 # copy nginx config files
 COPY ./nginx/crawlab.conf /etc/nginx/conf.d
 
+# install plugins
+RUN /bin/bash /app/bin/docker-install-plugins.sh
+
 # working directory
 WORKDIR /app/backend
 

Dockerfile.local

@@ -24,9 +24,6 @@ FROM ubuntu:20.04
 # set as non-interactive
 ENV DEBIAN_FRONTEND noninteractive
 
-# set CRAWLAB_IS_DOCKER
-ENV CRAWLAB_IS_DOCKER Y
-
 # install packages
 RUN chmod 777 /tmp \
 	&& apt-get update \
@@ -67,6 +64,9 @@ COPY --from=frontend-build /app/dist /app/dist
 # copy nginx config files
 COPY ./nginx/crawlab.conf /etc/nginx/conf.d
 
+# install plugins
+RUN /bin/bash /app/bin/docker-install-plugins.sh
+
 # working directory
 WORKDIR /app/backend
 

backend/Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:1.16 AS build
+
+WORKDIR /go/src/app
+COPY . .
+
+ENV GO111MODULE on
+#ENV GOPROXY https://goproxy.io
+
+RUN go mod tidy \
+  && go install -v ./...
+
+FROM alpine:3.14
+
+# copy files
+COPY --from=build /go/bin/crawlab /go/bin/crawlab

backend/go.mod

@@ -2,4 +2,4 @@ module crawlab
 
 go 1.16
 
-require github.com/crawlab-team/crawlab-core v0.6.0
+require github.com/crawlab-team/crawlab-core v0.6.0-7

backend/go.sum

@@ -122,8 +122,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k=
-github.com/crawlab-team/crawlab-core v0.6.0 h1:CGxdztjcIkozZp2EHXFN1brvKxIAdD0Xe1l4c+SFNE8=
+github.com/crawlab-team/crawlab-core v0.6.0-7 h1:23kTlKOZ3OqfSmD+SYJiwlqP1Cu1NJtPmbR8KeIGvgw=
-github.com/crawlab-team/crawlab-core v0.6.0/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
+github.com/crawlab-team/crawlab-core v0.6.0-7/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
 github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78=
 github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k=
 github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU=

frontend/Dockerfile

@@ -0,0 +1,13 @@
+FROM node:12 AS build
+
+ADD . /app
+WORKDIR /app
+RUN rm /app/.npmrc
+
+# install frontend
+RUN yarn install && yarn run build:docker
+
+FROM alpine:3.14
+
+# copy files
+COPY --from=build /app/dist /app/dist

frontend/package.json

@@ -23,7 +23,7 @@
     "@fortawesome/vue-fontawesome": "^3.0.0-5",
     "atom-material-icons": "^3.0.0",
     "codemirror": "^5.59.1",
-    "crawlab-ui": "0.6.0",
+    "crawlab-ui": "0.6.0-1",
     "echarts": "^5.1.2",
     "element-plus": "^1.3.0-beta.10",
     "vue": "^3.2",

frontend/yarn.lock

@@ -3472,10 +3472,10 @@ cosmiconfig@^5.0.0:
     js-yaml "^3.13.1"
     parse-json "^4.0.0"
 
-crawlab-ui@0.6.0:
+crawlab-ui@0.6.0-1:
-  version "0.6.0"
+  version "0.6.0-1"
-  resolved "https://registry.npmmirror.com/crawlab-ui/-/crawlab-ui-0.6.0.tgz#b0d67f33a8a3deef640707742a9a03588c9c1e76"
+  resolved "https://registry.npmjs.org/crawlab-ui/-/crawlab-ui-0.6.0-1.tgz#df6dc5473e7cee89738d8d459e5352dd7252671a"
-  integrity sha512-ORMgyA++RLlKs/7qiQN9cvBQ839m9K0TLkLmdAo+BvUxvPTAPhyVeoBV0BOrqiiehRVj3xrO4DH7Cl+I3YrIig==
+  integrity sha512-vDw9nvvq0+kFBLJVzaraawJg2s+mSjk6OmhQc/ocdr8UESCWHhKZf0i4pw28gzPY627pxmDhXuTfmNZ13iEtXg==
   dependencies:
     "@element-plus/icons" "^0.0.11"
     "@fortawesome/fontawesome-common-types" "^6.1.0"