git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1117 from crawlab-team/develop

Browse Source 
Develop
This commit is contained in:
Marvin Zhang
2022-06-03 21:51:17 +08:00
committed by GitHub
parent 7fcc590053 854c92a986
commit 63b2f42833
11 changed files with 194 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
.github/workflows/docker-crawlab.yml vendored Normal file
View File

@@ -0,0 +1,145 @@
name: "Docker Image CI: crawlab"
on:
push:
branches: [ develop, main ]
#pull_request:
# branches: [ main ]
release:
types: [ published ]
env:
IMAGE_PATH_CRAWLAB_BACKEND: backend
IMAGE_PATH_CRAWLAB_FRONTEND: frontend
IMAGE_NAME_CRAWLAB: crawlabteam/crawlab
IMAGE_NAME_CRAWLAB_BACKEND: crawlabteam/crawlab-backend
IMAGE_NAME_CRAWLAB_FRONTEND: crawlabteam/crawlab-frontend
jobs:
setup:
runs-on: ubuntu-latest
outputs:
is_matched_backend: ${{ steps.check_changed_files.outputs.is_matched_backend }}
is_matched_frontend: ${{ steps.check_changed_files.outputs.is_matched_frontend }}
is_matched_dockerfile: ${{ steps.check_changed_files.outputs.is_matched_dockerfile }}
version: ${{ steps.version.outputs.version }}
steps:
- uses: actions/checkout@v2
- name: Get changed files
id: changed_files
uses: tj-actions/changed-files@v18.7
- id: check_changed_files
name: Check changed files
run: |
# check changed files
is_matched_backend=0
is_matched_frontend=0
is_matched_dockerfile=0
for file in ${{ steps.changed_files.outputs.all_changed_files }}; do
if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_BACKEND}/.* ]]; then
file_backend=$file
is_matched_backend=1
fi
if [[ $file =~ ^${IMAGE_PATH_CRAWLAB_FRONTEND}/.* ]]; then
file_frontend=$file
is_matched_frontend=1
fi
if [[ $file == Dockerfile ]]; then
file_dockerfile=$file
is_matched_dockerfile=1
fi
done
# set outputs
echo "::set-output name=is_matched_backend::$is_matched_backend"
echo "::set-output name=is_matched_frontend::$is_matched_frontend"
echo "::set-output name=is_matched_dockerfile::$is_matched_dockerfile"
# echo outputs
echo "is_matched_backend=$is_matched_backend, file_backend=$file_backend"
echo "is_matched_frontend=$is_matched_frontend, file_frontend=$file_frontend"
echo "is_matched_dockerfile=$is_matched_dockerfile, file_dockerfile=$file_dockerfile"
- id: version
name: Get version
run: |
# Strip git ref prefix from version
VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
# Strip "v" prefix from tag name
[[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
# Use Docker `latest` tag convention
[ "$VERSION" == "main" ] && VERSION=latest
echo "::set-output name=version::$VERSION"
build-backend:
needs: [ setup ]
if: needs.setup.outputs.is_matched_backend == '1'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@v18.7
- name: Build image
run: |
cd $IMAGE_PATH_CRAWLAB_BACKEND
docker build . --file Dockerfile --tag image
- name: Log into registry
run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
- name: Push image
run: |
IMAGE_VERSION=${{needs.setup.outputs.version}}
IMAGE_NAME=$IMAGE_NAME_CRAWLAB_BACKEND
docker tag image $IMAGE_NAME:$IMAGE_VERSION
docker push $IMAGE_NAME:$IMAGE_VERSION
build-frontend:
needs: [ setup ]
if: needs.setup.outputs.is_matched_frontend == '1'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@v18.7
- name: Build image
run: |
cd $IMAGE_PATH_CRAWLAB_FRONTEND
docker build . --file Dockerfile --tag image
- name: Log into registry
run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
- name: Push image
run: |
IMAGE_VERSION=${{needs.setup.outputs.version}}
IMAGE_NAME=$IMAGE_NAME_CRAWLAB_FRONTEND
docker tag image $IMAGE_NAME:$IMAGE_VERSION
docker push $IMAGE_NAME:$IMAGE_VERSION
build-crawlab:
if: ${{ always() }}
needs: [ setup, build-backend, build-frontend ]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Update Dockerfile
run: |
IMAGE_VERSION=${{needs.setup.outputs.version}}
if [[ $IMAGE_VERSION != "latest" ]]; then
for n in crawlab-backend crawlab-frontend; do
IMAGE_NAME=$n
sed -i "s/${IMAGE_NAME}:latest/${IMAGE_NAME}:${IMAGE_VERSION}/" Dockerfile
done
fi
- name: Build image
run: docker build . --file Dockerfile --tag image
- name: Log into registry
run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
- name: Push image
run: |
IMAGE_VERSION=${{needs.setup.outputs.version}}
IMAGE_ID=$IMAGE_NAME_CRAWLAB
docker tag image $IMAGE_ID:$IMAGE_VERSION
docker push $IMAGE_ID:$IMAGE_VERSION

50
.github/workflows/dockerpush.yml vendored
View File

@@ -1,50 +0,0 @@
name: Docker Image CI
on:
push:
branches: [ develop, main ]
#pull_request:
# branches: [ main ]
release:
types: [ published ]
env:
IMAGE_NAME: crawlabteam/crawlab
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Build image
run: docker build . --file Dockerfile --tag image
- name: Log into registry
run: echo ${{ secrets.DOCKER_PASSWORD}} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
- name: Push image
run: |
IMAGE_ID=$IMAGE_NAME
# Strip git ref prefix from version
VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
# Strip "v" prefix from tag name
[[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
# Use Docker `latest` tag convention
[ "$VERSION" == "main" ] && VERSION=latest
echo IMAGE_ID=$IMAGE_ID
echo VERSION=$VERSION
docker tag image $IMAGE_ID:$VERSION
docker push $IMAGE_ID:$VERSION
if [[ $VERSION == "latest" ]]; then
docker tag image $IMAGE_ID:main
docker push $IMAGE_ID:main
fi

71
Dockerfile
View File

@@ -1,55 +1,11 @@
FROM golang:1.16 AS backend-build
FROM crawlabteam/crawlab-backend:latest AS backend-build
WORKDIR /go/src/app
COPY ./backend .
FROM crawlabteam/crawlab-frontend:latest AS frontend-build
ENV GO111MODULE on
#ENV GOPROXY https://goproxy.io
RUN go mod tidy \
&& go install -v ./...
FROM node:12 AS frontend-build
ADD ./frontend /app
WORKDIR /app
RUN rm /app/.npmrc
# install frontend
RUN yarn install && yarn run build:docker
FROM crawlabteam/crawlab-public-plugins:latest AS public-plugins-build
# images
FROM ubuntu:20.04
# set as non-interactive
ENV DEBIAN_FRONTEND noninteractive
# set CRAWLAB_IS_DOCKER
ENV CRAWLAB_IS_DOCKER Y
# install packages
RUN chmod 777 /tmp \
&& apt-get update \
&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate nginx wget dumb-init cloc
# install python
RUN apt-get install -y python3 python3-pip \
&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
&& ln -s /usr/bin/python3 /usr/local/bin/python
# install golang
RUN curl -OL https://golang.org/dl/go1.16.7.linux-amd64.tar.gz \
&& tar -C /usr/local -xvf go1.16.7.linux-amd64.tar.gz \
&& ln -s /usr/local/go/bin/go /usr/local/bin/go
# install seaweedfs
RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux_amd64.tar.gz \
&& tar -zxf linux_amd64.tar.gz \
&& cp weed /usr/local/bin
# install backend
RUN pip install scrapy pymongo bs4 requests
RUN pip install crawlab-sdk==0.6.b20211224-1500
FROM crawlabteam/crawlab-base:latest
# add files
COPY ./backend/conf /app/backend/conf
@@ -64,24 +20,11 @@ RUN cp /opt/bin/crawlab /usr/local/bin/crawlab-server
# copy frontend files
COPY --from=frontend-build /app/dist /app/dist
# copy public-plugins files
COPY --from=public-plugins-build /app/plugins /app/plugins
# copy nginx config files
COPY ./nginx/crawlab.conf /etc/nginx/conf.d
# working directory
WORKDIR /app/backend
# timezone environment
ENV TZ Asia/Shanghai
# language environment
ENV LC_ALL C.UTF-8
ENV LANG C.UTF-8
# frontend port
EXPOSE 8080
# backend port
EXPOSE 8000
# start backend
CMD ["/bin/bash", "/app/bin/docker-init.sh"]

6
Dockerfile.cn
View File

@@ -24,9 +24,6 @@ FROM ubuntu:20.04
# set as non-interactive
ENV DEBIAN_FRONTEND noninteractive
# set CRAWLAB_IS_DOCKER
ENV CRAWLAB_IS_DOCKER Y
# install packages
RUN chmod 777 /tmp \
&& apt-get update \
@@ -67,6 +64,9 @@ COPY --from=frontend-build /app/dist /app/dist
# copy nginx config files
COPY ./nginx/crawlab.conf /etc/nginx/conf.d
# install plugins
RUN /bin/bash /app/bin/docker-install-plugins.sh
# working directory
WORKDIR /app/backend

6
Dockerfile.local
View File

@@ -24,9 +24,6 @@ FROM ubuntu:20.04
# set as non-interactive
ENV DEBIAN_FRONTEND noninteractive
# set CRAWLAB_IS_DOCKER
ENV CRAWLAB_IS_DOCKER Y
# install packages
RUN chmod 777 /tmp \
&& apt-get update \
@@ -67,6 +64,9 @@ COPY --from=frontend-build /app/dist /app/dist
# copy nginx config files
COPY ./nginx/crawlab.conf /etc/nginx/conf.d
# install plugins
RUN /bin/bash /app/bin/docker-install-plugins.sh
# working directory
WORKDIR /app/backend

15
backend/Dockerfile Normal file
View File

@@ -0,0 +1,15 @@
FROM golang:1.16 AS build
WORKDIR /go/src/app
COPY . .
ENV GO111MODULE on
#ENV GOPROXY https://goproxy.io
RUN go mod tidy \
&& go install -v ./...
FROM alpine:3.14
# copy files
COPY --from=build /go/bin/crawlab /go/bin/crawlab

2
backend/go.mod
View File

@@ -2,4 +2,4 @@ module crawlab
go 1.16
require github.com/crawlab-team/crawlab-core v0.6.0
require github.com/crawlab-team/crawlab-core v0.6.0-7

4
backend/go.sum
View File

@@ -122,8 +122,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k=
github.com/crawlab-team/crawlab-core v0.6.0 h1:CGxdztjcIkozZp2EHXFN1brvKxIAdD0Xe1l4c+SFNE8=
github.com/crawlab-team/crawlab-core v0.6.0/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
github.com/crawlab-team/crawlab-core v0.6.0-7 h1:23kTlKOZ3OqfSmD+SYJiwlqP1Cu1NJtPmbR8KeIGvgw=
github.com/crawlab-team/crawlab-core v0.6.0-7/go.mod h1:KqfjSkEclVY39nC58bsq3MLcuXbDnsPp/ClcBDkqOF0=
github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78=
github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300 h1:2EymVIiOspX28qNC1Qon3W1fzXKQ8hi6ho3QtXB4w6k=
github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU=

13
frontend/Dockerfile Normal file
View File

@@ -0,0 +1,13 @@
FROM node:12 AS build
ADD . /app
WORKDIR /app
RUN rm /app/.npmrc
# install frontend
RUN yarn install && yarn run build:docker
FROM alpine:3.14
# copy files
COPY --from=build /app/dist /app/dist

2
frontend/package.json
View File

@@ -23,7 +23,7 @@
"@fortawesome/vue-fontawesome": "^3.0.0-5",
"atom-material-icons": "^3.0.0",
"codemirror": "^5.59.1",
"crawlab-ui": "0.6.0",
"crawlab-ui": "0.6.0-1",
"echarts": "^5.1.2",
"element-plus": "^1.3.0-beta.10",
"vue": "^3.2",

8
frontend/yarn.lock
View File

@@ -3472,10 +3472,10 @@ cosmiconfig@^5.0.0:
js-yaml "^3.13.1"
parse-json "^4.0.0"
crawlab-ui@0.6.0:
version "0.6.0"
resolved "https://registry.npmmirror.com/crawlab-ui/-/crawlab-ui-0.6.0.tgz#b0d67f33a8a3deef640707742a9a03588c9c1e76"
integrity sha512-ORMgyA++RLlKs/7qiQN9cvBQ839m9K0TLkLmdAo+BvUxvPTAPhyVeoBV0BOrqiiehRVj3xrO4DH7Cl+I3YrIig==
crawlab-ui@0.6.0-1:
version "0.6.0-1"
resolved "https://registry.npmjs.org/crawlab-ui/-/crawlab-ui-0.6.0-1.tgz#df6dc5473e7cee89738d8d459e5352dd7252671a"
integrity sha512-vDw9nvvq0+kFBLJVzaraawJg2s+mSjk6OmhQc/ocdr8UESCWHhKZf0i4pw28gzPY627pxmDhXuTfmNZ13iEtXg==
dependencies:
"@element-plus/icons" "^0.0.11"
"@fortawesome/fontawesome-common-types" "^6.1.0"