diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml new file mode 100644 index 00000000..2fab69c4 --- /dev/null +++ b/.github/workflows/docker-crawlab.yml @@ -0,0 +1,107 @@ +name: "Docker Image CI: crawlab" + +on: + push: + branches: [ develop, main ] + #pull_request: + # branches: [ main ] + release: + types: [ published ] + +env: + IMAGE_PATH_CRAWLAB_BACKEND: backend + IMAGE_PATH_CRAWLAB_FRONTEND: frontend + IMAGE_NAME_CRAWLAB: crawlabteam/crawlab + IMAGE_NAME_CRAWLAB_BACKEND: crawlabteam/crawlab-backend + IMAGE_NAME_CRAWLAB_FRONTEND: crawlabteam/crawlab-frontend + +jobs: + setup: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v18.7 + - name: Check changed files + run: | + for file in ${{ steps.changed-files.outputs.all_changed_files }}; do + if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_BACKEND}/.* ]]; then + echo "is_matched_backend=1" >> $GITHUB_ENV + break + fi + if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_FRONTEND}/.* ]]; then + echo "is_matched_frontend=1" >> $GITHUB_ENV + break + fi + done + - name: Get version + run: | + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "main" ] && VERSION=latest + + echo "IMAGE_VERSION=$VERSION" >> $GITHUB_ENV + + build-backend: + if: ${{ env.is_matched_backend == '1' }} + needs: [ setup ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v18.7 + - name: Build image + run: | + cd $IMAGE_PATH_CRAWLAB_BACKEND + docker build . --file Dockerfile --tag image + - name: Log into registry + run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + - name: Push image + run: | + IMAGE_NAME=$IMAGE_NAME_CRAWLAB_BACKEND + docker tag image $IMAGE_NAME:$IMAGE_VERSION + docker push $IMAGE_NAME:$IMAGE_VERSION + + build-frontend: + if: ${{ env.is_matched_frontend == '1' }} + needs: [ setup ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v18.7 + - name: Build image + run: | + cd $IMAGE_PATH_CRAWLAB_FRONTEND + docker build . --file Dockerfile --tag image + - name: Log into registry + run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + - name: Push image + run: | + IMAGE_NAME=$IMAGE_NAME_CRAWLAB_FRONTEND + docker tag image $IMAGE_NAME:$IMAGE_VERSION + docker push $IMAGE_NAME:$IMAGE_VERSION + + build: + if: ${{ env.is_matched_backend == '1' || env.is_matched_frontend == '1' }} + needs: [ build-backend, build-frontend ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Build image + run: docker build . --file Dockerfile --tag image + - name: Log into registry + run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + - name: Push image + run: | + IMAGE_ID=$IMAGE_NAME_CRAWLAB + docker tag image $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION diff --git a/.github/workflows/dockerpush.yml b/.github/workflows/dockerpush.yml deleted file mode 100644 index e153f529..00000000 --- a/.github/workflows/dockerpush.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Docker Image CI - -on: - push: - branches: [ develop, main ] - #pull_request: - # branches: [ main ] - release: - types: [ published ] - -env: - IMAGE_NAME: crawlabteam/crawlab - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Build image - run: docker build . --file Dockerfile --tag image - - - name: Log into registry - run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - - - name: Push image - run: | - IMAGE_ID=$IMAGE_NAME - - # Strip git ref prefix from version - VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - - # Strip "v" prefix from tag name - [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') - - # Use Docker `latest` tag convention - [ "$VERSION" == "main" ] && VERSION=latest - - echo IMAGE_ID=$IMAGE_ID - echo VERSION=$VERSION - - docker tag image $IMAGE_ID:$VERSION - docker push $IMAGE_ID:$VERSION - - if [[ $VERSION == "latest" ]]; then - docker tag image $IMAGE_ID:main - docker push $IMAGE_ID:main - fi diff --git a/Dockerfile b/Dockerfile index b5c2dc54..18053697 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,52 +1,9 @@ -FROM golang:1.16 AS backend-build +FROM crawlabteam/crawlab-backend:latest AS backend-build -WORKDIR /go/src/app -COPY ./backend . - -ENV GO111MODULE on -#ENV GOPROXY https://goproxy.io - -RUN go mod tidy \ - && go install -v ./... - -FROM node:12 AS frontend-build - -ADD ./frontend /app -WORKDIR /app -RUN rm /app/.npmrc - -# install frontend -RUN yarn install && yarn run build:docker +FROM crawlabteam/crawlab-frontend:latest AS frontend-build # images -FROM ubuntu:20.04 - -# set as non-interactive -ENV DEBIAN_FRONTEND noninteractive - -# install packages -RUN chmod 777 /tmp \ - && apt-get update \ - && apt-get install -y curl git net-tools iputils-ping ntp ntpdate nginx wget dumb-init cloc - -# install python -RUN apt-get install -y python3 python3-pip \ - && ln -s /usr/bin/pip3 /usr/local/bin/pip \ - && ln -s /usr/bin/python3 /usr/local/bin/python - -# install golang -RUN curl -OL https://golang.org/dl/go1.16.7.linux-amd64.tar.gz \ - && tar -C /usr/local -xvf go1.16.7.linux-amd64.tar.gz \ - && ln -s /usr/local/go/bin/go /usr/local/bin/go - -# install seaweedfs -RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux_amd64.tar.gz \ - && tar -zxf linux_amd64.tar.gz \ - && cp weed /usr/local/bin - -# install backend -RUN pip install scrapy pymongo bs4 requests -RUN pip install crawlab-sdk==0.6.b20211224-1500 +FROM crawlabteam/crawlab-base:latest # add files COPY ./backend/conf /app/backend/conf @@ -67,24 +24,5 @@ COPY ./nginx/crawlab.conf /etc/nginx/conf.d # install plugins RUN /bin/bash /app/bin/docker-install-plugins.sh -# working directory -WORKDIR /app/backend - -# timezone environment -ENV TZ Asia/Shanghai - -# language environment -ENV LC_ALL C.UTF-8 -ENV LANG C.UTF-8 - -# docker -ENV CRAWLAB_DOCKER Y - -# frontend port -EXPOSE 8080 - -# backend port -EXPOSE 8000 - # start backend CMD ["/bin/bash", "/app/bin/docker-init.sh"] diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 00000000..d13a85a6 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,10 @@ +FROM golang:1.16 + +WORKDIR /go/src/app +COPY ./backend . + +ENV GO111MODULE on +#ENV GOPROXY https://goproxy.io + +RUN go mod tidy \ + && go install -v ./... diff --git a/backend/go.mod b/backend/go.mod index 74fbef94..cb1a57bf 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.0-3 +require github.com/crawlab-team/crawlab-core v0.6.0-4 diff --git a/backend/go.sum b/backend/go.sum index fe54f12e..3f8f5caa 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -122,8 +122,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k= -github.com/crawlab-team/crawlab-core v0.6.0-3 h1:K4QtGSl3p+naaYf8hqupKagEpRDz17qD/zwugbFPhOI= -github.com/crawlab-team/crawlab-core v0.6.0-3/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.0-4 h1:91Wsu+AS+0SEGgYlzhIjsRVrIFE5E0nU2kP0BJdBaRc= +github.com/crawlab-team/crawlab-core v0.6.0-4/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 00000000..b74e70f0 --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,8 @@ +FROM node:12 + +ADD ./frontend /app +WORKDIR /app +RUN rm /app/.npmrc + +# install frontend +RUN yarn install && yarn run build:docker