mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-21 17:21:09 +01:00
ci: updated base-image
This commit is contained in:
Marvin Zhang
@@ -6,10 +6,7 @@ ENV DEBIAN_FRONTEND=noninteractive
|
||||
# Install dependencies
|
||||
COPY ./install /app/install
|
||||
RUN bash /app/install/deps/deps.sh && \
|
||||
bash /app/install/python/python.sh && \
|
||||
bash /app/install/go/go.sh && \
|
||||
bash /app/install/node/node.sh && \
|
||||
bash /app/install/browser/browser.sh
|
||||
bash /app/install/python/python.sh
|
||||
|
||||
# Working directory
|
||||
WORKDIR /app/backend
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Source nvm environment from profile.d instead
|
||||
source /etc/profile.d/node-env.sh
|
||||
|
||||
# Version - using "stable" for installation but not for verification
|
||||
version="stable"
|
||||
|
||||
# Install dependencies
|
||||
apt-get install -y \
|
||||
xvfb \
|
||||
libxi6 \
|
||||
libgconf-2-4 \
|
||||
libglib2.0-0 \
|
||||
libnss3 \
|
||||
libx11-6
|
||||
|
||||
# Install puppeteer browsers package globally first
|
||||
npm install -g @puppeteer/browsers
|
||||
|
||||
# Install chrome with auto-yes and capture the output
|
||||
INSTALL_OUTPUT=$(npx -y @puppeteer/browsers install chrome@${version} --install-deps)
|
||||
echo "Installation output: $INSTALL_OUTPUT"
|
||||
|
||||
# Extract the actual version and path from the output
|
||||
ACTUAL_VERSION=$(echo "$INSTALL_OUTPUT" | grep -o 'chrome@[^ ]*' | cut -d'@' -f2)
|
||||
CHROME_BIN=$(echo "$INSTALL_OUTPUT" | awk '{print $2}')
|
||||
|
||||
echo "Detected Chrome version: $ACTUAL_VERSION"
|
||||
echo "Chrome binary path: $CHROME_BIN"
|
||||
|
||||
# Update version variable for ChromeDriver
|
||||
version="$ACTUAL_VERSION"
|
||||
|
||||
# Create symbolic links
|
||||
ln -sf "$CHROME_BIN" /usr/local/bin/google-chrome
|
||||
|
||||
# Verify chrome is installed (with more detailed error message)
|
||||
if ! command -v google-chrome &> /dev/null; then
|
||||
echo "ERROR: Chrome is not installed properly"
|
||||
echo "Chrome installation path: $(find /chrome -type f -name chrome 2>/dev/null)"
|
||||
echo "PATH environment: $PATH"
|
||||
echo "CHROME_PATH environment: $CHROME_BIN"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Install chromedriver with auto-yes and capture the output
|
||||
CHROMEDRIVER_OUTPUT=$(npx -y @puppeteer/browsers install chromedriver@${version})
|
||||
echo "ChromeDriver installation output: $CHROMEDRIVER_OUTPUT"
|
||||
|
||||
# Extract ChromeDriver path from the output
|
||||
CHROMEDRIVER_BIN=$(echo "$CHROMEDRIVER_OUTPUT" | awk '{print $2}')
|
||||
echo "ChromeDriver binary path: $CHROMEDRIVER_BIN"
|
||||
|
||||
# Create symbolic links
|
||||
ln -sf "$CHROMEDRIVER_BIN" /usr/local/bin/chromedriver
|
||||
|
||||
# Verify chromedriver is installed
|
||||
if ! command -v chromedriver &> /dev/null; then
|
||||
echo "ERROR: ChromeDriver is not installed properly"
|
||||
echo "ChromeDriver installation path: $CHROMEDRIVER_BIN"
|
||||
echo "PATH environment: $PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Print installed versions for reference
|
||||
echo "Installed Chrome version: $(google-chrome --version)"
|
||||
echo "Installed ChromeDriver version: $(chromedriver --version)"
|
||||
|
||||
# Create a temporary directory for the test
|
||||
TEST_DIR=$(mktemp -d)
|
||||
cd "$TEST_DIR"
|
||||
|
||||
# Create a simple test script
|
||||
cat > test.py << 'EOL'
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
|
||||
try:
|
||||
chrome_options = Options()
|
||||
chrome_options.add_argument('--headless')
|
||||
chrome_options.add_argument('--no-sandbox')
|
||||
chrome_options.add_argument('--disable-dev-shm-usage')
|
||||
|
||||
driver = webdriver.Chrome(options=chrome_options)
|
||||
driver.get('https://www.google.com')
|
||||
print("ChromeDriver test successful!")
|
||||
driver.quit()
|
||||
except Exception as e:
|
||||
print(f"ChromeDriver test failed: {str(e)}")
|
||||
exit(1)
|
||||
EOL
|
||||
|
||||
# Run the test
|
||||
python3 test.py
|
||||
|
||||
# Clean up
|
||||
cd -
|
||||
rm -rf "$TEST_DIR"
|
||||
|
||||
# After successful Chrome and ChromeDriver installation, create persistent env config
|
||||
cat > /etc/profile.d/browser-env.sh << 'EOF'
|
||||
# Chrome and ChromeDriver paths
|
||||
export CHROME_BIN="$(which google-chrome)"
|
||||
export CHROMEDRIVER_BIN="$(which chromedriver)"
|
||||
# Common Chrome flags for headless environments
|
||||
export CHROME_FLAGS="--headless --no-sandbox --disable-dev-shm-usage"
|
||||
EOF
|
||||
|
||||
# Make the file executable
|
||||
chmod +x /etc/profile.d/browser-env.sh
|
||||
|
||||
# Source it immediately
|
||||
source /etc/profile.d/browser-env.sh
|
||||
@@ -1,36 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
version="1.22.9"
|
||||
|
||||
# Install goenv
|
||||
git clone https://github.com/go-nv/goenv.git ~/.goenv
|
||||
|
||||
# Create persistent environment config
|
||||
cat > /etc/profile.d/goenv.sh << 'EOF'
|
||||
export GOENV_ROOT="$HOME/.goenv"
|
||||
export PATH="$GOENV_ROOT/bin:$PATH"
|
||||
eval "$(goenv init -)"
|
||||
EOF
|
||||
|
||||
# Make the file executable
|
||||
chmod +x /etc/profile.d/goenv.sh
|
||||
|
||||
# Source it immediately for the rest of the installation
|
||||
source /etc/profile.d/goenv.sh
|
||||
|
||||
# Install go
|
||||
goenv install ${version}
|
||||
goenv global ${version}
|
||||
|
||||
# Verify
|
||||
go_version=$(go version)
|
||||
if [[ $go_version =~ "go${version}" ]]; then
|
||||
:
|
||||
else
|
||||
echo "ERROR: go version does not match. expect \"go${version}\", but actual is \"${go_version}\""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create symbolic links
|
||||
ln -sf "$(goenv which go)" /usr/local/bin/go
|
||||
ln -sf "$(goenv which gofmt)" /usr/local/bin/gofmt
|
||||
@@ -1,49 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
version="11.0.12-open"
|
||||
|
||||
# Install SDKMAN!
|
||||
curl -s "https://get.sdkman.io" | bash
|
||||
|
||||
# Create persistent environment config for SDKMAN
|
||||
cat > /etc/profile.d/sdkman.sh << 'EOF'
|
||||
# SDKMAN configuration
|
||||
export SDKMAN_DIR="$HOME/.sdkman"
|
||||
[[ -s "$SDKMAN_DIR/bin/sdkman-init.sh" ]] && source "$SDKMAN_DIR/bin/sdkman-init.sh"
|
||||
|
||||
# Java environment variables
|
||||
export JAVA_HOME="$SDKMAN_DIR/candidates/java/current"
|
||||
export PATH="$JAVA_HOME/bin:$PATH"
|
||||
EOF
|
||||
|
||||
# Make the file executable
|
||||
chmod +x /etc/profile.d/sdkman.sh
|
||||
|
||||
# Source it immediately for the rest of the installation
|
||||
source /etc/profile.d/sdkman.sh
|
||||
|
||||
# Install Java 11 (OpenJDK)
|
||||
sdk install java ${version}
|
||||
|
||||
# Set Java 11 as default
|
||||
sdk default java ${version}
|
||||
|
||||
# Create symbolic links
|
||||
ln -sf "$(sdkman which java)" /usr/local/bin/java
|
||||
ln -sf "$(sdkman which javac)" /usr/local/bin/javac
|
||||
|
||||
# Verify installations
|
||||
java_version=$(java -version)
|
||||
if [[ $java_version =~ "${version}" ]]; then
|
||||
:
|
||||
else
|
||||
echo "ERROR: java version does not match. expect \"${version}\", but actual is \"${java_version}\""
|
||||
exit 1
|
||||
fi
|
||||
javac_version=$(javac -version)
|
||||
if [[ $javac_version =~ "${version}" ]]; then
|
||||
:
|
||||
else
|
||||
echo "ERROR: javac version does not match. expect \"${version}\", but actual is \"${javac_version}\""
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,54 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
version="22"
|
||||
|
||||
# Install nvm (Node Version Manager)
|
||||
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.0/install.sh | bash
|
||||
|
||||
# Create a file in /etc/profile.d/
|
||||
cat > /etc/profile.d/node-env.sh << 'EOF'
|
||||
export NVM_DIR="/root/.nvm"
|
||||
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
|
||||
[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"
|
||||
export NODE_PATH=/usr/lib/node_modules
|
||||
EOF
|
||||
|
||||
# Make the file executable
|
||||
chmod +x /etc/profile.d/node-env.sh
|
||||
|
||||
# Source the file to apply the environment variables
|
||||
source /etc/profile.d/node-env.sh
|
||||
|
||||
# Download and install Node.js (you may need to restart the terminal)
|
||||
nvm install ${version}
|
||||
|
||||
# Set node version and make it the default
|
||||
nvm use ${version}
|
||||
nvm alias default ${version}
|
||||
|
||||
# Verify the right Node.js version is in the environment
|
||||
if [[ ! "$(node -v)" =~ ^v${version} ]]; then
|
||||
echo "Node.js version is not v${version}.x"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Install node dependencies
|
||||
npm install -g \
|
||||
npm@latest \
|
||||
yarn \
|
||||
pnpm \
|
||||
crawlab-sdk@latest \
|
||||
puppeteer \
|
||||
playwright \
|
||||
playwright-chromium \
|
||||
crawlee
|
||||
|
||||
# Create symbolic links
|
||||
ln -sf "$(nvm which node)" /usr/local/bin/node
|
||||
ln -sf "$(nvm which npm)" /usr/local/bin/npm
|
||||
ln -sf "$(nvm which yarn)" /usr/local/bin/yarn
|
||||
ln -sf "$(nvm which pnpm)" /usr/local/bin/pnpm
|
||||
|
||||
# Clean up
|
||||
npm cache clean --force && \
|
||||
rm -rf ~/.npm
|
||||
Reference in New Issue
Block a user