ci: updated base-image

2026-01-30 18:00:56 +01:00 · 2024-11-23 10:51:46 +08:00
parent b4c233f122
commit f016d279d1
15 changed files with 258 additions and 134 deletions
--- a/docker/base-image/Dockerfile
+++ b/docker/base-image/Dockerfile
@@ -1,6 +1,9 @@
-# Build stage
-FROM ubuntu:22.04 AS builder
+FROM ubuntu:22.04
+
+# Non-interactive mode
 ENV DEBIAN_FRONTEND=noninteractive
+
+# Install dependencies
 COPY ./install /app/install
 RUN bash /app/install/deps/deps.sh && \
    bash /app/install/python/python.sh && \
@@ -8,21 +11,13 @@ RUN bash /app/install/deps/deps.sh && \
    bash /app/install/node/node.sh && \
    bash /app/install/browser/browser.sh

-# Final stage
-FROM ubuntu:22.04
-ENV DEBIAN_FRONTEND=noninteractive
-COPY --from=builder /usr/local/bin /usr/local/bin
-COPY --from=builder /usr/local/lib /usr/local/lib
-COPY --from=builder /usr/lib /usr/lib
-COPY --from=builder /app/install /app/install
-
-# working directory
+# Working directory
 WORKDIR /app/backend

-# node path
+# Node path
 ENV NODE_PATH=/usr/lib/node_modules

-# timezone environment
+# Timezone environment
 ENV TZ=Asia/Shanghai

 # language environment
--- a/docker/base-image/install/browser/browser.sh
+++ b/docker/base-image/install/browser/browser.sh
@@ -1,9 +1,7 @@
 #!/bin/bash

-# Source nvm environment
-export NVM_DIR="$HOME/.nvm"
-[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
-[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"
+# Source nvm environment from profile.d instead
+source /etc/profile.d/node-env.sh

 # Version - using "stable" for installation but not for verification
 version="stable"
@@ -34,7 +32,7 @@ echo "Chrome binary path: $CHROME_BIN"
 # Update version variable for ChromeDriver
 version="$ACTUAL_VERSION"

-# Add chrome to PATH
+# Create symbolic links
 ln -sf "$CHROME_BIN" /usr/local/bin/google-chrome

 # Verify chrome is installed (with more detailed error message)
@@ -54,7 +52,7 @@ echo "ChromeDriver installation output: $CHROMEDRIVER_OUTPUT"
 CHROMEDRIVER_BIN=$(echo "$CHROMEDRIVER_OUTPUT" | awk '{print $2}')
 echo "ChromeDriver binary path: $CHROMEDRIVER_BIN"

-# Add chromedriver to PATH
+# Create symbolic links
 ln -sf "$CHROMEDRIVER_BIN" /usr/local/bin/chromedriver

 # Verify chromedriver is installed
@@ -99,3 +97,18 @@ python3 test.py
 # Clean up
 cd -
 rm -rf "$TEST_DIR"
+
+# After successful Chrome and ChromeDriver installation, create persistent env config
+cat > /etc/profile.d/browser-env.sh << 'EOF'
+# Chrome and ChromeDriver paths
+export CHROME_BIN="$(which google-chrome)"
+export CHROMEDRIVER_BIN="$(which chromedriver)"
+# Common Chrome flags for headless environments
+export CHROME_FLAGS="--headless --no-sandbox --disable-dev-shm-usage"
+EOF
+
+# Make the file executable
+chmod +x /etc/profile.d/browser-env.sh
+
+# Source it immediately
+source /etc/profile.d/browser-env.sh
--- a/docker/base-image/install/deps/deps.sh
+++ b/docker/base-image/install/deps/deps.sh
@@ -1,24 +1,39 @@
 #!/bin/bash

-# ensure directory mode of /tmp
+# Ensure directory mode of /tmp is world-writable (readable, writable, executable by all users)
+# This is important for temporary file operations in containerized environments
 chmod 777 /tmp

-# update
+# Update the package index files from the repositories
+# This ensures we get the latest versions of packages
 apt-get update

-# common deps
+# Install common dependencies with detailed explanations
+# -y flag means "yes" to all prompts (non-interactive installation)
 apt-get install -y \
-	curl \
-	git \
-	net-tools \
-	iputils-ping \
-	ntp \
-	ntpdate \
-	nginx \
-	wget \
-	dumb-init \
-	cloc \
-	unzip \
-	build-essential \
-	gnupg2 \
-	libc6
+    # Network and File Transfer Utilities
+    curl \       # Modern HTTP client, useful for API requests and downloads
+    wget \       # Another download utility, often used in scripts
+    
+    # Version Control
+    git \        # Distributed version control system
+    
+    # Network Diagnostics and Monitoring
+    net-tools \  # Traditional networking tools (netstat, ifconfig, etc.)
+    iputils-ping \ # Tools for testing network connectivity (ping)
+    
+    # Time Synchronization
+    ntp \        # Network Time Protocol daemon for time sync
+    ntpdate \    # Client for one-time NTP sync
+    
+    # Web Server
+    nginx \      # High-performance HTTP server and reverse proxy
+    
+    # File Operations
+    unzip \      # Extract .zip archives
+    
+    # Security and Encryption
+    gnupg2 \     # GNU Privacy Guard for encryption and signing
+    
+    # System Libraries
+    libc6        # GNU C Library - essential for running C programs
--- a/docker/base-image/install/go/go.sh
+++ b/docker/base-image/install/go/go.sh
@@ -5,15 +5,18 @@ version="1.22.9"
 # Install goenv
 git clone https://github.com/go-nv/goenv.git ~/.goenv

-# Add goenv to path
-echo 'export GOENV_ROOT="$HOME/.goenv"' >> ~/.bashrc
-echo 'export PATH="$GOENV_ROOT/bin:$PATH"' >> ~/.bashrc
-echo 'eval "$(goenv init -)"' >> ~/.bashrc
-
-# Ensure changes take effect immediately
+# Create persistent environment config
+cat > /etc/profile.d/goenv.sh << 'EOF'
 export GOENV_ROOT="$HOME/.goenv"
 export PATH="$GOENV_ROOT/bin:$PATH"
 eval "$(goenv init -)"
+EOF
+
+# Make the file executable
+chmod +x /etc/profile.d/goenv.sh
+
+# Source it immediately for the rest of the installation
+source /etc/profile.d/goenv.sh

 # Install go
 goenv install ${version}
--- a/docker/base-image/install/java/java.sh
+++ b/docker/base-image/install/java/java.sh
@@ -5,10 +5,24 @@ version="11.0.12-open"
 # Install SDKMAN!
 curl -s "https://get.sdkman.io" | bash

-# Source SDKMAN!
-source "$HOME/.sdkman/bin/sdkman-init.sh"
+# Create persistent environment config for SDKMAN
+cat > /etc/profile.d/sdkman.sh << 'EOF'
+# SDKMAN configuration
+export SDKMAN_DIR="$HOME/.sdkman"
+[[ -s "$SDKMAN_DIR/bin/sdkman-init.sh" ]] && source "$SDKMAN_DIR/bin/sdkman-init.sh"

-# Install Java 11 (you can specify vendor, e.g., 11.0.12-open for OpenJDK)
+# Java environment variables
+export JAVA_HOME="$SDKMAN_DIR/candidates/java/current"
+export PATH="$JAVA_HOME/bin:$PATH"
+EOF
+
+# Make the file executable
+chmod +x /etc/profile.d/sdkman.sh
+
+# Source it immediately for the rest of the installation
+source /etc/profile.d/sdkman.sh
+
+# Install Java 11 (OpenJDK)
 sdk install java ${version}

 # Set Java 11 as default
@@ -18,18 +32,18 @@ sdk default java ${version}
 ln -sf "$(sdkman which java)" /usr/local/bin/java
 ln -sf "$(sdkman which javac)" /usr/local/bin/javac

-# Verify
+# Verify installations
 java_version=$(java -version)
 if [[ $java_version =~ "${version}" ]]; then
-	:
+    :
 else
-	echo "ERROR: java version does not match. expect \"${version}\", but actual is \"${java_version}\""
-	exit 1
+    echo "ERROR: java version does not match. expect \"${version}\", but actual is \"${java_version}\""
+    exit 1
 fi
 javac_version=$(javac -version)
 if [[ $javac_version =~ "${version}" ]]; then
-	:
+    :
 else
-	echo "ERROR: javac version does not match. expect \"${version}\", but actual is \"${javac_version}\""
-	exit 1
+    echo "ERROR: javac version does not match. expect \"${version}\", but actual is \"${javac_version}\""
+    exit 1
 fi
--- a/docker/base-image/install/node/node.sh
+++ b/docker/base-image/install/node/node.sh
@@ -5,15 +5,16 @@ version="22"
 # Install nvm (Node Version Manager)
 curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.0/install.sh | bash

-# Add nvm to path
-echo 'export NVM_DIR="$HOME/.nvm"' >> ~/.bashrc
-echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm' >> ~/.bashrc
-echo '[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"  # This loads nvm bash_completion' >> ~/.bashrc
+# Create a file in /etc/profile.d/
+cat > /etc/profile.d/node-env.sh << 'EOF'
+export NVM_DIR="/root/.nvm"
+[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"
+export NODE_PATH=/usr/lib/node_modules
+EOF

-# Ensure changes take effect immediately
-export NVM_DIR="$HOME/.nvm"
-[[ -s "$NVM_DIR/nvm.sh" ]] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
-[[ -s "$NVM_DIR/bash_completion" ]] && \. "$NVM_DIR/bash_completion"  # This loads nvm bash_completion
+# Make the file executable
+chmod +x /etc/profile.d/node-env.sh

 # Download and install Node.js (you may need to restart the terminal)
 nvm install ${version}
--- a/docker/base-image/install/python/python.sh
+++ b/docker/base-image/install/python/python.sh
@@ -25,17 +25,19 @@ apt-get install -y \
 # Install pyenv
 curl https://pyenv.run | bash

-# Add pyenv to path
-echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc
-echo '[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc
-echo 'eval "$(pyenv init -)"' >> ~/.bashrc
-echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc
-
-# Ensure changes take effect immediately
+# Create a file in /etc/profile.d/
+cat > /etc/profile.d/pyenv.sh << 'EOF'
 export PYENV_ROOT="$HOME/.pyenv"
 [[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"
 eval "$(pyenv init -)"
 eval "$(pyenv virtualenv-init -)"
+EOF
+
+# Make the file executable
+chmod +x /etc/profile.d/pyenv.sh
+
+# Source it immediately for the rest of the installation
+source /etc/profile.d/pyenv.sh

 # Install python ${version} via pyenv
 pyenv install ${version}
--- a/docker/ghcr/docker-compose.yml
+++ b/docker/ghcr/docker-compose.yml
@@ -0,0 +1,34 @@
+version: '3.3'
+services:
+  master:
+    image: ghcr.io/crawlab-team/crawlab:${CRAWLAB_TAG:-develop}
+    container_name: crawlab_ghcr_master
+    environment:
+      CRAWLAB_NODE_MASTER: "Y"
+      CRAWLAB_MONGO_HOST: "mongo"
+    ports:
+      - "8080:8080"
+    depends_on:
+      - mongo
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health", "||", "exit", "1"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+
+  worker:
+    image: ghcr.io/crawlab-team/crawlab:${CRAWLAB_TAG:-develop}
+    container_name: crawlab_ghcr_worker
+    environment:
+      CRAWLAB_NODE_MASTER: "N"
+      CRAWLAB_MASTER_HOST: "master"
+    depends_on:
+      - master
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health", "||", "exit", "1"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+
+  mongo:
+    image: mongo:5