mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-21 17:21:09 +01:00
ci: updated workflow
This commit is contained in:
4
.github/workflows/docker-crawlab.yml
vendored
4
.github/workflows/docker-crawlab.yml
vendored
@@ -141,6 +141,7 @@ jobs:
|
||||
run: echo "failed=true" >> $GITHUB_OUTPUT
|
||||
|
||||
test_backend:
|
||||
name: Test backend
|
||||
needs: [ setup ]
|
||||
if: needs.setup.outputs.backend_changed == 'true' || needs.setup.outputs.workflow_changed == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
@@ -178,6 +179,7 @@ jobs:
|
||||
run: echo "failed=true" >> $GITHUB_OUTPUT
|
||||
|
||||
build_backend:
|
||||
name: Build backend
|
||||
needs: [ setup, test_backend ]
|
||||
if: needs.test_backend.result == 'success' || needs.setup.outputs.workflow_changed == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
@@ -203,6 +205,7 @@ jobs:
|
||||
run: echo "failed=true" >> $GITHUB_OUTPUT
|
||||
|
||||
build_frontend:
|
||||
name: Build frontend
|
||||
needs: [ setup ]
|
||||
if: needs.setup.outputs.frontend_changed == 'true' || needs.setup.outputs.workflow_changed == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
@@ -230,6 +233,7 @@ jobs:
|
||||
run: echo "failed=true" >> $GITHUB_OUTPUT
|
||||
|
||||
build_crawlab:
|
||||
name: Build crawlab
|
||||
needs: [setup, build_base_image, test_backend, build_backend, build_frontend]
|
||||
if: |
|
||||
always() &&
|
||||
|
||||
@@ -2,28 +2,24 @@
|
||||
|
||||
version="1.22.9"
|
||||
|
||||
# install goenv
|
||||
# Install goenv
|
||||
git clone https://github.com/go-nv/goenv.git ~/.goenv
|
||||
|
||||
# add goenv to path
|
||||
# Add goenv to path
|
||||
echo 'export GOENV_ROOT="$HOME/.goenv"' >> ~/.bashrc
|
||||
echo 'export PATH="$GOENV_ROOT/bin:$PATH"' >> ~/.bashrc
|
||||
echo 'eval "$(goenv init -)"' >> ~/.bashrc
|
||||
|
||||
# ensure changes take effect immediately
|
||||
# Ensure changes take effect immediately
|
||||
export GOENV_ROOT="$HOME/.goenv"
|
||||
export PATH="$GOENV_ROOT/bin:$PATH"
|
||||
eval "$(goenv init -)"
|
||||
|
||||
# install go
|
||||
# Install go
|
||||
goenv install ${version}
|
||||
goenv global ${version}
|
||||
|
||||
# Create symbolic links
|
||||
ln -sf "$(goenv which go)" /usr/local/bin/go
|
||||
ln -sf "$(goenv which gofmt)" /usr/local/bin/gofmt
|
||||
|
||||
# verify
|
||||
# Verify
|
||||
go_version=$(go version)
|
||||
if [[ $go_version =~ "go${version}" ]]; then
|
||||
:
|
||||
@@ -31,3 +27,7 @@ else
|
||||
echo "ERROR: go version does not match. expect \"go${version}\", but actual is \"${go_version}\""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create symbolic links
|
||||
ln -sf "$(goenv which go)" /usr/local/bin/go
|
||||
ln -sf "$(goenv which gofmt)" /usr/local/bin/gofmt
|
||||
@@ -2,39 +2,33 @@
|
||||
|
||||
version="22"
|
||||
|
||||
# installs nvm (Node Version Manager)
|
||||
# Install nvm (Node Version Manager)
|
||||
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.0/install.sh | bash
|
||||
|
||||
# add nvm to path
|
||||
# Add nvm to path
|
||||
echo 'export NVM_DIR="$HOME/.nvm"' >> ~/.bashrc
|
||||
echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' >> ~/.bashrc
|
||||
echo '[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion' >> ~/.bashrc
|
||||
|
||||
# ensure changes take effect immediately
|
||||
# Ensure changes take effect immediately
|
||||
export NVM_DIR="$HOME/.nvm"
|
||||
[[ -s "$NVM_DIR/nvm.sh" ]] && \. "$NVM_DIR/nvm.sh" # This loads nvm
|
||||
[[ -s "$NVM_DIR/bash_completion" ]] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion
|
||||
|
||||
# download and install Node.js (you may need to restart the terminal)
|
||||
# Download and install Node.js (you may need to restart the terminal)
|
||||
nvm install ${version}
|
||||
|
||||
# set node version and make it the default
|
||||
# Set node version and make it the default
|
||||
nvm use ${version}
|
||||
nvm alias default ${version}
|
||||
|
||||
# Create symbolic links
|
||||
ln -sf "$(nvm which node)" /usr/local/bin/node
|
||||
ln -sf "$(nvm which npm)" /usr/local/bin/npm
|
||||
ln -sf "$(nvm which yarn)" /usr/local/bin/yarn
|
||||
ln -sf "$(nvm which pnpm)" /usr/local/bin/pnpm
|
||||
|
||||
# verifies the right Node.js version is in the environment
|
||||
# Verify the right Node.js version is in the environment
|
||||
if [[ ! "$(node -v)" =~ ^v${version} ]]; then
|
||||
echo "Node.js version is not v${version}.x"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# install node dependencies
|
||||
# Install node dependencies
|
||||
npm install -g \
|
||||
npm@latest \
|
||||
yarn \
|
||||
@@ -45,6 +39,12 @@ npm install -g \
|
||||
playwright-chromium \
|
||||
crawlee
|
||||
|
||||
# Create symbolic links
|
||||
ln -sf "$(nvm which node)" /usr/local/bin/node
|
||||
ln -sf "$(nvm which npm)" /usr/local/bin/npm
|
||||
ln -sf "$(nvm which yarn)" /usr/local/bin/yarn
|
||||
ln -sf "$(nvm which pnpm)" /usr/local/bin/pnpm
|
||||
|
||||
# Clean up
|
||||
npm cache clean --force && \
|
||||
rm -rf ~/.npm
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
version="3.12"
|
||||
|
||||
# install build dependencies
|
||||
# Install build dependencies
|
||||
apt-get install -y \
|
||||
make \
|
||||
build-essential \
|
||||
@@ -22,30 +22,26 @@ apt-get install -y \
|
||||
libffi-dev \
|
||||
liblzma-dev
|
||||
|
||||
# install pyenv
|
||||
# Install pyenv
|
||||
curl https://pyenv.run | bash
|
||||
|
||||
# add pyenv to path
|
||||
# Add pyenv to path
|
||||
echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc
|
||||
echo '[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc
|
||||
echo 'eval "$(pyenv init -)"' >> ~/.bashrc
|
||||
echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc
|
||||
|
||||
# ensure changes take effect immediately
|
||||
# Ensure changes take effect immediately
|
||||
export PYENV_ROOT="$HOME/.pyenv"
|
||||
[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"
|
||||
eval "$(pyenv init -)"
|
||||
eval "$(pyenv virtualenv-init -)"
|
||||
|
||||
# install python ${version} via pyenv
|
||||
# Install python ${version} via pyenv
|
||||
pyenv install ${version}
|
||||
pyenv global ${version}
|
||||
|
||||
# Create symbolic links
|
||||
ln -sf $(pyenv which python) /usr/local/bin/python
|
||||
ln -sf $(pyenv which pip) /usr/local/bin/pip
|
||||
|
||||
# verify
|
||||
# Verify
|
||||
python_version=$(python -V)
|
||||
if [[ $python_version =~ "Python ${version}" ]]; then
|
||||
:
|
||||
@@ -61,9 +57,13 @@ else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# install python dependencies
|
||||
# Install python dependencies
|
||||
pip install -r /app/install/python/requirements.txt
|
||||
|
||||
# Create symbolic links
|
||||
ln -sf $(pyenv which python) /usr/local/bin/python
|
||||
ln -sf $(pyenv which pip) /usr/local/bin/pip
|
||||
|
||||
# After pip install
|
||||
pip cache purge && \
|
||||
rm -rf ~/.cache/pip/* && \
|
||||
|
||||
Reference in New Issue
Block a user