From 1e2cad899d2c494366ca0e16b1d891be640cfb9b Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 20 Nov 2024 14:21:05 +0800 Subject: [PATCH] ci: updated dockerfiles --- .env.local | 2 + .github/workflows/docker-crawlab.yml | 2 + Dockerfile | 8 ++- backend/.dockerignore | 41 +++++++++++++ docker-compose.dev.yml | 58 +++++++++++++++++++ docker/base-image/Dockerfile | 3 - .../install/chromedriver/chromedriver.sh | 30 +++++----- docker/base-image/install/node/node.sh | 24 ++++++-- docker/base-image/install/python/python.sh | 23 ++++++-- .../base-image/install/seaweedfs/seaweedfs.sh | 5 -- frontend/.dockerignore | 33 ++++++++++- 11 files changed, 194 insertions(+), 35 deletions(-) create mode 100644 .env.local create mode 100644 backend/.dockerignore create mode 100644 docker-compose.dev.yml delete mode 100644 docker/base-image/install/seaweedfs/seaweedfs.sh diff --git a/.env.local b/.env.local new file mode 100644 index 00000000..82e90d3f --- /dev/null +++ b/.env.local @@ -0,0 +1,2 @@ +CRAWLAB_TAG=develop +CRAWLAB_PORT=8082 diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 8817ce11..e1e1763a 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -268,6 +268,8 @@ jobs: with: context: . file: ./Dockerfile + build-args: | + CRAWLAB_TAG=${{ needs.setup.outputs.version }} push: true tags: | ${{ env.IMAGE_NAME_CRAWLAB }}:${{ needs.setup.outputs.version }} diff --git a/Dockerfile b/Dockerfile index d0b80f68..adbc2281 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,10 @@ -FROM crawlabteam/crawlab-backend:latest AS backend-build +ARG CRAWLAB_TAG=latest -FROM crawlabteam/crawlab-frontend:latest AS frontend-build +FROM crawlabteam/crawlab-backend:${CRAWLAB_TAG} AS backend-build -FROM crawlabteam/crawlab-base:latest +FROM crawlabteam/crawlab-frontend:${CRAWLAB_TAG} AS frontend-build + +FROM crawlabteam/crawlab-base:${CRAWLAB_TAG} # copy backend files RUN mkdir -p /opt/bin diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 00000000..739d5f19 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,41 @@ +# Go build artifacts +*.exe +*.exe~ +*.dll +*.so +*.dylib +*.test +*.out + +# Go workspace file +go.work + +# IDE directories +.idea/ +.vscode/ + +# Dependencies +vendor/ + +# Build output +bin/ +dist/ + +# Debug files +debug/ + +# Environment files +.env +*.env + +# System files +.DS_Store +Thumbs.db + +# Test coverage +coverage.txt +*.cover + +# Temporary files +*.tmp +*~ diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..95631922 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,58 @@ +version: '3.3' +services: + backend: + build: + context: ./backend + image: crawlabteam/crawlab-backend:${CRAWLAB_TAG:-develop} + env_file: + - .env.local + + frontend: + build: + context: ./frontend + image: crawlabteam/crawlab-frontend:${CRAWLAB_TAG:-develop} + env_file: + - .env.local + + base-image: + build: + context: ./docker/base-image + image: crawlabteam/crawlab-base:${CRAWLAB_TAG:-develop} + env_file: + - .env.local + + crawlab: + build: + context: . + image: crawlabteam/crawlab:${CRAWLAB_TAG:-develop} + depends_on: + - backend + - frontend + - base-image + env_file: + - .env.local + + master: + image: crawlabteam/crawlab:${CRAWLAB_TAG:-develop} + environment: + CRAWLAB_NODE_MASTER: "Y" + CRAWLAB_MONGO_HOST: "mongo" + ports: + - "${CRAWLAB_PORT:-8082}:8080" + depends_on: + - mongo + env_file: + - .env.local + + worker: + image: crawlabteam/crawlab:${CRAWLAB_TAG:-develop} + environment: + CRAWLAB_NODE_MASTER: "N" + CRAWLAB_MASTER_HOST: "master" + depends_on: + - master + env_file: + - .env.local + + mongo: + image: mongo:5 diff --git a/docker/base-image/Dockerfile b/docker/base-image/Dockerfile index 846a946e..a56f451a 100644 --- a/docker/base-image/Dockerfile +++ b/docker/base-image/Dockerfile @@ -22,9 +22,6 @@ RUN bash /app/install/node/node.sh # install java #RUN bash /app/install/java/java.sh -# install seaweedfs -RUN bash /app/install/seaweedfs/seaweedfs.sh - # install chromedriver RUN bash /app/install/chromedriver/chromedriver.sh diff --git a/docker/base-image/install/chromedriver/chromedriver.sh b/docker/base-image/install/chromedriver/chromedriver.sh index 35502efe..2677b376 100644 --- a/docker/base-image/install/chromedriver/chromedriver.sh +++ b/docker/base-image/install/chromedriver/chromedriver.sh @@ -1,21 +1,25 @@ #!/bin/bash # version -version="106.0.5249.61" +version="stable" # deps -apt-get install -y unzip xvfb libxi6 libgconf-2-4 +apt-get install -y xvfb libxi6 libgconf-2-4 -# chrome -wget -q "http://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_${version}-1_amd64.deb" -apt-get -y install "./google-chrome-stable_${version}-1_amd64.deb" -echo `google-chrome --version` -rm -f "./google-chrome-stable_${version}-1_amd64.deb" +# install chrome +npx @puppeteer/browsers install chrome@${version} +# verify chrome version +if [[ ! "$(google-chrome --version)" =~ ^Google\ Chrome\ ${version} ]]; then + echo "ERROR: chrome version does not match. expected: \"Google Chrome ${version}\", but actual is \"$(google-chrome --version)\"" + exit 1 +fi -# chromedriver -wget "https://chromedriver.storage.googleapis.com/${version}/chromedriver_linux64.zip" -unzip chromedriver_linux64.zip -mv chromedriver /usr/local/bin/chromedriver -chown root:root /usr/local/bin/chromedriver -chmod +x /usr/local/bin/chromedriver +# install chromedriver +npx @puppeteer/browsers install chromedriver@${version} + +# verify chromedriver version +if [[ ! "$(chromedriver --version)" =~ ^ChromeDriver\ ${version} ]]; then + echo "ERROR: chromedriver version does not match. expected: \"ChromeDriver ${version}\", but actual is \"$(chromedriver --version)\"" + exit 1 +fi diff --git a/docker/base-image/install/node/node.sh b/docker/base-image/install/node/node.sh index 4465b495..12af4299 100644 --- a/docker/base-image/install/node/node.sh +++ b/docker/base-image/install/node/node.sh @@ -1,9 +1,25 @@ #!/bin/bash -# install node -curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh \ - && bash /tmp/nodesource_setup.sh \ - && apt-get install -y nodejs +# installs nvm (Node Version Manager) +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.0/install.sh | bash + +# download and install Node.js (you may need to restart the terminal) +nvm install 22 + +# set node version +nvm use 22 + +# verifies the right Node.js version is in the environment +if [[ ! "$(node -v)" =~ ^v22 ]]; then + echo "Node.js version is not v22.x" + exit 1 +fi + +# verifies the right npm version is in the environment +if [[ ! "$(npm -v)" =~ ^10 ]]; then + echo "npm version is not 10.x" + exit 1 +fi # install node dependencies npm install -g \ diff --git a/docker/base-image/install/python/python.sh b/docker/base-image/install/python/python.sh index 829303c8..8ce8299f 100644 --- a/docker/base-image/install/python/python.sh +++ b/docker/base-image/install/python/python.sh @@ -1,11 +1,22 @@ #!/bin/bash -# install python -apt-get update \ - && apt install software-properties-common -y \ - && add-apt-repository ppa:deadsnakes/ppa -y \ - && apt install python3.10 -y \ - && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 +# install pyenv +curl https://pyenv.run | bash + +# add pyenv to path +echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc +echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc +echo 'eval "$(pyenv init -)"' >> ~/.bashrc +source ~/.bashrc + +# install python build dependencies +apt-get install -y make build-essential libssl-dev zlib1g-dev \ +libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ +libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev + +# install python 3.10 via pyenv +pyenv install 3.10 +pyenv global 3.10 # alias rm /usr/local/bin/pip | true diff --git a/docker/base-image/install/seaweedfs/seaweedfs.sh b/docker/base-image/install/seaweedfs/seaweedfs.sh deleted file mode 100644 index fb21ec75..00000000 --- a/docker/base-image/install/seaweedfs/seaweedfs.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -wget https://github.com/seaweedfs/seaweedfs/releases/download/3.47/linux_amd64.tar.gz \ - && tar -zxf linux_amd64.tar.gz \ - && cp weed /usr/local/bin diff --git a/frontend/.dockerignore b/frontend/.dockerignore index 8b178651..9bc955c7 100644 --- a/frontend/.dockerignore +++ b/frontend/.dockerignore @@ -1,2 +1,33 @@ # Ignore the .npmrc file -.npmrc \ No newline at end of file +.npmrc + +# Ignore node_modules +node_modules + +# local env files +.env.local +.env.*.local + +# Editor directories and files +.idea +.vscode +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? + +# Log files +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* + +# Ignore stats.html +stats.html + +# Ignore .turbo +.turbo + +# Ignore tmp directory +tmp/