diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml new file mode 100644 index 00000000..f8c01315 --- /dev/null +++ b/.github/workflows/docker-crawlab.yml @@ -0,0 +1,145 @@ +name: "Docker Image CI: crawlab" + +on: + push: + branches: [ develop, main ] + #pull_request: + # branches: [ main ] + release: + types: [ published ] + +env: + IMAGE_PATH_CRAWLAB_BACKEND: backend + IMAGE_PATH_CRAWLAB_FRONTEND: frontend + IMAGE_NAME_CRAWLAB: crawlabteam/crawlab + IMAGE_NAME_CRAWLAB_BACKEND: crawlabteam/crawlab-backend + IMAGE_NAME_CRAWLAB_FRONTEND: crawlabteam/crawlab-frontend + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + is_matched_backend: ${{ steps.check_changed_files.outputs.is_matched_backend }} + is_matched_frontend: ${{ steps.check_changed_files.outputs.is_matched_frontend }} + is_matched_dockerfile: ${{ steps.check_changed_files.outputs.is_matched_dockerfile }} + version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v2 + - name: Get changed files + id: changed_files + uses: tj-actions/changed-files@v18.7 + - id: check_changed_files + name: Check changed files + run: | + # check changed files + is_matched_backend=0 + is_matched_frontend=0 + is_matched_dockerfile=0 + for file in ${{ steps.changed_files.outputs.all_changed_files }}; do + if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_BACKEND}/.* ]]; then + file_backend=$file + is_matched_backend=1 + fi + if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_FRONTEND}/.* ]]; then + file_frontend=$file + is_matched_frontend=1 + fi + if [[ $file == Dockerfile ]]; then + file_dockerfile=$file + is_matched_dockerfile=1 + fi + done + + # set outputs + echo "::set-output name=is_matched_backend::$is_matched_backend" + echo "::set-output name=is_matched_frontend::$is_matched_frontend" + echo "::set-output name=is_matched_dockerfile::$is_matched_dockerfile" + + # echo outputs + echo "is_matched_backend=$is_matched_backend, file_backend=$file_backend" + echo "is_matched_frontend=$is_matched_frontend, file_frontend=$file_frontend" + echo "is_matched_dockerfile=$is_matched_dockerfile, file_dockerfile=$file_dockerfile" + - id: version + name: Get version + run: | + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "main" ] && VERSION=latest + + echo "::set-output name=version::$VERSION" + + build-backend: + needs: [ setup ] + if: needs.setup.outputs.is_matched_backend == '1' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v18.7 + - name: Build image + run: | + cd $IMAGE_PATH_CRAWLAB_BACKEND + docker build . --file Dockerfile --tag image + - name: Log into registry + run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + - name: Push image + run: | + IMAGE_VERSION=${{needs.setup.outputs.version}} + IMAGE_NAME=$IMAGE_NAME_CRAWLAB_BACKEND + docker tag image $IMAGE_NAME:$IMAGE_VERSION + docker push $IMAGE_NAME:$IMAGE_VERSION + + build-frontend: + needs: [ setup ] + if: needs.setup.outputs.is_matched_frontend == '1' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v18.7 + - name: Build image + run: | + cd $IMAGE_PATH_CRAWLAB_FRONTEND + docker build . --file Dockerfile --tag image + - name: Log into registry + run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + - name: Push image + run: | + IMAGE_VERSION=${{needs.setup.outputs.version}} + IMAGE_NAME=$IMAGE_NAME_CRAWLAB_FRONTEND + docker tag image $IMAGE_NAME:$IMAGE_VERSION + docker push $IMAGE_NAME:$IMAGE_VERSION + + build-crawlab: + if: ${{ always() }} + needs: [ setup, build-backend, build-frontend ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Update Dockerfile + run: | + IMAGE_VERSION=${{needs.setup.outputs.version}} + if [[ $IMAGE_VERSION != "latest" ]]; then + for n in crawlab-backend crawlab-frontend; do + IMAGE_NAME=$n + sed -i "s/${IMAGE_NAME}:latest/${IMAGE_NAME}:${IMAGE_VERSION}/" Dockerfile + done + fi + + - name: Build image + run: docker build . --file Dockerfile --tag image + - name: Log into registry + run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + - name: Push image + run: | + IMAGE_VERSION=${{needs.setup.outputs.version}} + IMAGE_ID=$IMAGE_NAME_CRAWLAB + docker tag image $IMAGE_ID:$IMAGE_VERSION + docker push $IMAGE_ID:$IMAGE_VERSION diff --git a/.github/workflows/dockerpush.yml b/.github/workflows/dockerpush.yml deleted file mode 100644 index e153f529..00000000 --- a/.github/workflows/dockerpush.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Docker Image CI - -on: - push: - branches: [ develop, main ] - #pull_request: - # branches: [ main ] - release: - types: [ published ] - -env: - IMAGE_NAME: crawlabteam/crawlab - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Build image - run: docker build . --file Dockerfile --tag image - - - name: Log into registry - run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - - - name: Push image - run: | - IMAGE_ID=$IMAGE_NAME - - # Strip git ref prefix from version - VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - - # Strip "v" prefix from tag name - [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') - - # Use Docker `latest` tag convention - [ "$VERSION" == "main" ] && VERSION=latest - - echo IMAGE_ID=$IMAGE_ID - echo VERSION=$VERSION - - docker tag image $IMAGE_ID:$VERSION - docker push $IMAGE_ID:$VERSION - - if [[ $VERSION == "latest" ]]; then - docker tag image $IMAGE_ID:main - docker push $IMAGE_ID:main - fi diff --git a/Dockerfile b/Dockerfile index 68df191a..d05fbee4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,55 +1,11 @@ -FROM golang:1.16 AS backend-build +FROM crawlabteam/crawlab-backend:latest AS backend-build -WORKDIR /go/src/app -COPY ./backend . +FROM crawlabteam/crawlab-frontend:latest AS frontend-build -ENV GO111MODULE on -#ENV GOPROXY https://goproxy.io - -RUN go mod tidy \ - && go install -v ./... - -FROM node:12 AS frontend-build - -ADD ./frontend /app -WORKDIR /app -RUN rm /app/.npmrc - -# install frontend -RUN yarn install && yarn run build:docker +FROM crawlabteam/crawlab-public-plugins:latest AS public-plugins-build # images -FROM ubuntu:20.04 - -# set as non-interactive -ENV DEBIAN_FRONTEND noninteractive - -# set CRAWLAB_IS_DOCKER -ENV CRAWLAB_IS_DOCKER Y - -# install packages -RUN chmod 777 /tmp \ - && apt-get update \ - && apt-get install -y curl git net-tools iputils-ping ntp ntpdate nginx wget dumb-init cloc - -# install python -RUN apt-get install -y python3 python3-pip \ - && ln -s /usr/bin/pip3 /usr/local/bin/pip \ - && ln -s /usr/bin/python3 /usr/local/bin/python - -# install golang -RUN curl -OL https://golang.org/dl/go1.16.7.linux-amd64.tar.gz \ - && tar -C /usr/local -xvf go1.16.7.linux-amd64.tar.gz \ - && ln -s /usr/local/go/bin/go /usr/local/bin/go - -# install seaweedfs -RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux_amd64.tar.gz \ - && tar -zxf linux_amd64.tar.gz \ - && cp weed /usr/local/bin - -# install backend -RUN pip install scrapy pymongo bs4 requests -RUN pip install crawlab-sdk==0.6.b20211224-1500 +FROM crawlabteam/crawlab-base:latest # add files COPY ./backend/conf /app/backend/conf @@ -64,24 +20,11 @@ RUN cp /opt/bin/crawlab /usr/local/bin/crawlab-server # copy frontend files COPY --from=frontend-build /app/dist /app/dist +# copy public-plugins files +COPY --from=public-plugins-build /app/plugins /app/plugins + # copy nginx config files COPY ./nginx/crawlab.conf /etc/nginx/conf.d -# working directory -WORKDIR /app/backend - -# timezone environment -ENV TZ Asia/Shanghai - -# language environment -ENV LC_ALL C.UTF-8 -ENV LANG C.UTF-8 - -# frontend port -EXPOSE 8080 - -# backend port -EXPOSE 8000 - # start backend CMD ["/bin/bash", "/app/bin/docker-init.sh"] diff --git a/Dockerfile.cn b/Dockerfile.cn index 6a359e11..4b9320b4 100644 --- a/Dockerfile.cn +++ b/Dockerfile.cn @@ -24,9 +24,6 @@ FROM ubuntu:20.04 # set as non-interactive ENV DEBIAN_FRONTEND noninteractive -# set CRAWLAB_IS_DOCKER -ENV CRAWLAB_IS_DOCKER Y - # install packages RUN chmod 777 /tmp \ && apt-get update \ @@ -67,6 +64,9 @@ COPY --from=frontend-build /app/dist /app/dist # copy nginx config files COPY ./nginx/crawlab.conf /etc/nginx/conf.d +# install plugins +RUN /bin/bash /app/bin/docker-install-plugins.sh + # working directory WORKDIR /app/backend diff --git a/Dockerfile.local b/Dockerfile.local index aeffb969..b8000c3d 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -24,9 +24,6 @@ FROM ubuntu:20.04 # set as non-interactive ENV DEBIAN_FRONTEND noninteractive -# set CRAWLAB_IS_DOCKER -ENV CRAWLAB_IS_DOCKER Y - # install packages RUN chmod 777 /tmp \ && apt-get update \ @@ -67,6 +64,9 @@ COPY --from=frontend-build /app/dist /app/dist # copy nginx config files COPY ./nginx/crawlab.conf /etc/nginx/conf.d +# install plugins +RUN /bin/bash /app/bin/docker-install-plugins.sh + # working directory WORKDIR /app/backend diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 00000000..6cddfb80 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,15 @@ +FROM golang:1.16 AS build + +WORKDIR /go/src/app +COPY . . + +ENV GO111MODULE on +#ENV GOPROXY https://goproxy.io + +RUN go mod tidy \ + && go install -v ./... + +FROM alpine:3.14 + +# copy files +COPY --from=build /go/bin/crawlab /go/bin/crawlab diff --git a/backend/go.mod b/backend/go.mod index a2ada1f2..48f0d69e 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,4 +2,4 @@ module crawlab go 1.16 -require github.com/crawlab-team/crawlab-core v0.6.0 +require github.com/crawlab-team/crawlab-core v0.6.0-7 diff --git a/backend/go.sum b/backend/go.sum index 0ada3474..57afa5db 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -122,8 +122,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k= -github.com/crawlab-team/crawlab-core v0.6.0 h1:CGxdztjcIkozZp2EHXFN1brvKxIAdD0Xe1l4c+SFNE8= -github.com/crawlab-team/crawlab-core v0.6.0/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= +github.com/crawlab-team/crawlab-core v0.6.0-7 h1:23kTlKOZ3OqfSmD+SYJiwlqP1Cu1NJtPmbR8KeIGvgw= +github.com/crawlab-team/crawlab-core v0.6.0-7/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 00000000..6e9e1fae --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,13 @@ +FROM node:12 AS build + +ADD . /app +WORKDIR /app +RUN rm /app/.npmrc + +# install frontend +RUN yarn install && yarn run build:docker + +FROM alpine:3.14 + +# copy files +COPY --from=build /app/dist /app/dist diff --git a/frontend/package.json b/frontend/package.json index ed46a80f..2d5ac7be 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -23,7 +23,7 @@ "@fortawesome/vue-fontawesome": "^3.0.0-5", "atom-material-icons": "^3.0.0", "codemirror": "^5.59.1", - "crawlab-ui": "0.6.0", + "crawlab-ui": "0.6.0-1", "echarts": "^5.1.2", "element-plus": "^1.3.0-beta.10", "vue": "^3.2", diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 9d3bda49..57a51f7c 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -3472,10 +3472,10 @@ cosmiconfig@^5.0.0: js-yaml "^3.13.1" parse-json "^4.0.0" -crawlab-ui@0.6.0: - version "0.6.0" - resolved "https://registry.npmmirror.com/crawlab-ui/-/crawlab-ui-0.6.0.tgz#b0d67f33a8a3deef640707742a9a03588c9c1e76" - integrity sha512-ORMgyA++RLlKs/7qiQN9cvBQ839m9K0TLkLmdAo+BvUxvPTAPhyVeoBV0BOrqiiehRVj3xrO4DH7Cl+I3YrIig== +crawlab-ui@0.6.0-1: + version "0.6.0-1" + resolved "https://registry.npmjs.org/crawlab-ui/-/crawlab-ui-0.6.0-1.tgz#df6dc5473e7cee89738d8d459e5352dd7252671a" + integrity sha512-vDw9nvvq0+kFBLJVzaraawJg2s+mSjk6OmhQc/ocdr8UESCWHhKZf0i4pw28gzPY627pxmDhXuTfmNZ13iEtXg== dependencies: "@element-plus/icons" "^0.0.11" "@fortawesome/fontawesome-common-types" "^6.1.0"