diff --git a/docker/base-image/Dockerfile b/docker/base-image/Dockerfile index 7e761eed..83cdf15a 100644 --- a/docker/base-image/Dockerfile +++ b/docker/base-image/Dockerfile @@ -6,10 +6,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Install dependencies COPY ./install /app/install RUN bash /app/install/deps/deps.sh && \ - bash /app/install/python/python.sh && \ - bash /app/install/go/go.sh && \ - bash /app/install/node/node.sh && \ - bash /app/install/browser/browser.sh + bash /app/install/python/python.sh # Working directory WORKDIR /app/backend diff --git a/docker/base-image/install/browser/browser.sh b/docker/base-image/install/browser/browser.sh deleted file mode 100644 index bf6c0159..00000000 --- a/docker/base-image/install/browser/browser.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash - -# Source nvm environment from profile.d instead -source /etc/profile.d/node-env.sh - -# Version - using "stable" for installation but not for verification -version="stable" - -# Install dependencies -apt-get install -y \ - xvfb \ - libxi6 \ - libgconf-2-4 \ - libglib2.0-0 \ - libnss3 \ - libx11-6 - -# Install puppeteer browsers package globally first -npm install -g @puppeteer/browsers - -# Install chrome with auto-yes and capture the output -INSTALL_OUTPUT=$(npx -y @puppeteer/browsers install chrome@${version} --install-deps) -echo "Installation output: $INSTALL_OUTPUT" - -# Extract the actual version and path from the output -ACTUAL_VERSION=$(echo "$INSTALL_OUTPUT" | grep -o 'chrome@[^ ]*' | cut -d'@' -f2) -CHROME_BIN=$(echo "$INSTALL_OUTPUT" | awk '{print $2}') - -echo "Detected Chrome version: $ACTUAL_VERSION" -echo "Chrome binary path: $CHROME_BIN" - -# Update version variable for ChromeDriver -version="$ACTUAL_VERSION" - -# Create symbolic links -ln -sf "$CHROME_BIN" /usr/local/bin/google-chrome - -# Verify chrome is installed (with more detailed error message) -if ! command -v google-chrome &> /dev/null; then - echo "ERROR: Chrome is not installed properly" - echo "Chrome installation path: $(find /chrome -type f -name chrome 2>/dev/null)" - echo "PATH environment: $PATH" - echo "CHROME_PATH environment: $CHROME_BIN" - exit 1 -fi - -# Install chromedriver with auto-yes and capture the output -CHROMEDRIVER_OUTPUT=$(npx -y @puppeteer/browsers install chromedriver@${version}) -echo "ChromeDriver installation output: $CHROMEDRIVER_OUTPUT" - -# Extract ChromeDriver path from the output -CHROMEDRIVER_BIN=$(echo "$CHROMEDRIVER_OUTPUT" | awk '{print $2}') -echo "ChromeDriver binary path: $CHROMEDRIVER_BIN" - -# Create symbolic links -ln -sf "$CHROMEDRIVER_BIN" /usr/local/bin/chromedriver - -# Verify chromedriver is installed -if ! command -v chromedriver &> /dev/null; then - echo "ERROR: ChromeDriver is not installed properly" - echo "ChromeDriver installation path: $CHROMEDRIVER_BIN" - echo "PATH environment: $PATH" - exit 1 -fi - -# Print installed versions for reference -echo "Installed Chrome version: $(google-chrome --version)" -echo "Installed ChromeDriver version: $(chromedriver --version)" - -# Create a temporary directory for the test -TEST_DIR=$(mktemp -d) -cd "$TEST_DIR" - -# Create a simple test script -cat > test.py << 'EOL' -from selenium import webdriver -from selenium.webdriver.chrome.options import Options - -try: - chrome_options = Options() - chrome_options.add_argument('--headless') - chrome_options.add_argument('--no-sandbox') - chrome_options.add_argument('--disable-dev-shm-usage') - - driver = webdriver.Chrome(options=chrome_options) - driver.get('https://www.google.com') - print("ChromeDriver test successful!") - driver.quit() -except Exception as e: - print(f"ChromeDriver test failed: {str(e)}") - exit(1) -EOL - -# Run the test -python3 test.py - -# Clean up -cd - -rm -rf "$TEST_DIR" - -# After successful Chrome and ChromeDriver installation, create persistent env config -cat > /etc/profile.d/browser-env.sh << 'EOF' -# Chrome and ChromeDriver paths -export CHROME_BIN="$(which google-chrome)" -export CHROMEDRIVER_BIN="$(which chromedriver)" -# Common Chrome flags for headless environments -export CHROME_FLAGS="--headless --no-sandbox --disable-dev-shm-usage" -EOF - -# Make the file executable -chmod +x /etc/profile.d/browser-env.sh - -# Source it immediately -source /etc/profile.d/browser-env.sh diff --git a/docker/base-image/install/go/go.sh b/docker/base-image/install/go/go.sh deleted file mode 100644 index b58e4268..00000000 --- a/docker/base-image/install/go/go.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -version="1.22.9" - -# Install goenv -git clone https://github.com/go-nv/goenv.git ~/.goenv - -# Create persistent environment config -cat > /etc/profile.d/goenv.sh << 'EOF' -export GOENV_ROOT="$HOME/.goenv" -export PATH="$GOENV_ROOT/bin:$PATH" -eval "$(goenv init -)" -EOF - -# Make the file executable -chmod +x /etc/profile.d/goenv.sh - -# Source it immediately for the rest of the installation -source /etc/profile.d/goenv.sh - -# Install go -goenv install ${version} -goenv global ${version} - -# Verify -go_version=$(go version) -if [[ $go_version =~ "go${version}" ]]; then - : -else - echo "ERROR: go version does not match. expect \"go${version}\", but actual is \"${go_version}\"" - exit 1 -fi - -# Create symbolic links -ln -sf "$(goenv which go)" /usr/local/bin/go -ln -sf "$(goenv which gofmt)" /usr/local/bin/gofmt \ No newline at end of file diff --git a/docker/base-image/install/java/java.sh b/docker/base-image/install/java/java.sh deleted file mode 100644 index 8fc32e83..00000000 --- a/docker/base-image/install/java/java.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -version="11.0.12-open" - -# Install SDKMAN! -curl -s "https://get.sdkman.io" | bash - -# Create persistent environment config for SDKMAN -cat > /etc/profile.d/sdkman.sh << 'EOF' -# SDKMAN configuration -export SDKMAN_DIR="$HOME/.sdkman" -[[ -s "$SDKMAN_DIR/bin/sdkman-init.sh" ]] && source "$SDKMAN_DIR/bin/sdkman-init.sh" - -# Java environment variables -export JAVA_HOME="$SDKMAN_DIR/candidates/java/current" -export PATH="$JAVA_HOME/bin:$PATH" -EOF - -# Make the file executable -chmod +x /etc/profile.d/sdkman.sh - -# Source it immediately for the rest of the installation -source /etc/profile.d/sdkman.sh - -# Install Java 11 (OpenJDK) -sdk install java ${version} - -# Set Java 11 as default -sdk default java ${version} - -# Create symbolic links -ln -sf "$(sdkman which java)" /usr/local/bin/java -ln -sf "$(sdkman which javac)" /usr/local/bin/javac - -# Verify installations -java_version=$(java -version) -if [[ $java_version =~ "${version}" ]]; then - : -else - echo "ERROR: java version does not match. expect \"${version}\", but actual is \"${java_version}\"" - exit 1 -fi -javac_version=$(javac -version) -if [[ $javac_version =~ "${version}" ]]; then - : -else - echo "ERROR: javac version does not match. expect \"${version}\", but actual is \"${javac_version}\"" - exit 1 -fi \ No newline at end of file diff --git a/docker/base-image/install/node/node.sh b/docker/base-image/install/node/node.sh deleted file mode 100644 index 850378d5..00000000 --- a/docker/base-image/install/node/node.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -version="22" - -# Install nvm (Node Version Manager) -curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.0/install.sh | bash - -# Create a file in /etc/profile.d/ -cat > /etc/profile.d/node-env.sh << 'EOF' -export NVM_DIR="/root/.nvm" -[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" -[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" -export NODE_PATH=/usr/lib/node_modules -EOF - -# Make the file executable -chmod +x /etc/profile.d/node-env.sh - -# Source the file to apply the environment variables -source /etc/profile.d/node-env.sh - -# Download and install Node.js (you may need to restart the terminal) -nvm install ${version} - -# Set node version and make it the default -nvm use ${version} -nvm alias default ${version} - -# Verify the right Node.js version is in the environment -if [[ ! "$(node -v)" =~ ^v${version} ]]; then - echo "Node.js version is not v${version}.x" - exit 1 -fi - -# Install node dependencies -npm install -g \ - npm@latest \ - yarn \ - pnpm \ - crawlab-sdk@latest \ - puppeteer \ - playwright \ - playwright-chromium \ - crawlee - -# Create symbolic links -ln -sf "$(nvm which node)" /usr/local/bin/node -ln -sf "$(nvm which npm)" /usr/local/bin/npm -ln -sf "$(nvm which yarn)" /usr/local/bin/yarn -ln -sf "$(nvm which pnpm)" /usr/local/bin/pnpm - -# Clean up -npm cache clean --force && \ -rm -rf ~/.npm \ No newline at end of file