git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

ci: updated base-image

Browse Source
This commit is contained in:
Marvin Zhang
2024-11-23 10:51:46 +08:00
parent b4c233f122
commit f016d279d1
15 changed files with 258 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/docker-crawlab.yml vendored
View File

@@ -305,7 +305,7 @@ jobs:
ports:
- 8080:8080
options: >-
--health-cmd "curl -f http://localhost:8000/health || exit 1"
--health-cmd "curl -f http://localhost:8080/api/health || exit 1"
--health-interval 30s
--health-timeout 10s
--health-retries 5
@@ -314,8 +314,10 @@ jobs:
env:
CRAWLAB_NODE_MASTER: N
CRAWLAB_MASTER_HOST: master
ports:
- 8001:8000
options: >-
--health-cmd "curl -f http://localhost:8000/health || exit 1"
--health-cmd "curl -f http://localhost:8001/health || exit 1"
--health-interval 30s
--health-timeout 10s
--health-retries 5

11
core/apps/api.go
View File

@@ -50,13 +50,14 @@ func (app *Api) Start() {
if err != nil {
panic(err)
}
log.Infof("api server listening on %s", address)
// serve
if err := http.Serve(app.ln, app.app); err != nil {
if !errors.Is(err, http.ErrServerClosed) {
log.Error("run server error:" + err.Error())
log.Errorf("run api server error: %v", err)
} else {
log.Info("server graceful down")
log.Info("api server graceful down")
}
}
}
@@ -70,7 +71,7 @@ func (app *Api) Stop() {
defer cancel()
if err := app.srv.Shutdown(ctx); err != nil {
log.Error("run server error:" + err.Error())
log.Errorf("shutdown api server error: %v", err)
}
}
@@ -89,11 +90,9 @@ func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (
}
func newApi() *Api {
api := &Api{
return &Api{
app: gin.New(),
}
api.Init()
return api
}
var api *Api

12
core/apps/server.go
View File

@@ -16,9 +16,6 @@ type Server struct {
// modules
nodeSvc interfaces.NodeService
api *Api
// internals
quit chan int
}
func (app *Server) Init() {
@@ -32,7 +29,7 @@ func (app *Server) Init() {
func (app *Server) Start() {
if utils.IsMaster() {
// start api
go app.api.Start()
go start(app.api)
}
// start node service
@@ -40,12 +37,11 @@ func (app *Server) Start() {
}
func (app *Server) Wait() {
<-app.quit
utils.DefaultWait()
}
func (app *Server) Stop() {
app.api.Stop()
app.quit <- 1
}
func (app *Server) GetApi() ApiApp {
@@ -70,9 +66,7 @@ func (app *Server) initPprof() {
func newServer() App {
// server
svr := &Server{
quit: make(chan int, 1),
}
svr := &Server{}
// master modules
if utils.IsMaster() {

5
core/apps/utils.go
View File

@@ -3,7 +3,6 @@ package apps
import (
"fmt"
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/trace"
)
@@ -18,10 +17,6 @@ func start(app App) {
app.Stop()
}
func DefaultWait() {
utils.DefaultWait()
}
func initModule(name string, fn func() error) (err error) {
if err := fn(); err != nil {
log.Error(fmt.Sprintf("init %s error: %s", name, err.Error()))

23
db/mongo/client_test.go
View File

@@ -1,23 +0,0 @@
package mongo
import (
"github.com/stretchr/testify/require"
"testing"
)
func setupMongoTest() (err error) {
return nil
}
func cleanupMongoTest() {
}
func TestMongoInitMongo(t *testing.T) {
err := setupMongoTest()
require.Nil(t, err)
_, err = GetMongoClient()
require.Nil(t, err)
cleanupMongoTest()
}

9
db/mongo/col.go
View File

@@ -3,7 +3,6 @@ package mongo
import (
"context"
"errors"
"github.com/apex/log"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
@@ -260,6 +259,10 @@ func (col *Col) MustCreateIndexes(indexModels []mongo.IndexModel) {
func (col *Col) DeleteIndex(name string) (err error) {
_, err = col.c.Indexes().DropOne(col.ctx, name)
if err != nil {
var e mongo.CommandError
if errors.As(err, &e) && e.HasErrorCode(26) {
return nil
}
log.Errorf("error deleting index: %v", err)
return err
}
@@ -269,6 +272,10 @@ func (col *Col) DeleteIndex(name string) (err error) {
func (col *Col) DeleteAllIndexes() (err error) {
_, err = col.c.Indexes().DropAll(col.ctx)
if err != nil {
var e mongo.CommandError
if errors.As(err, &e) && e.HasErrorCode(26) {
return nil
}
log.Errorf("error deleting all indexes: %v", err)
return err
}

115
db/mongo/col_test.go
View File

@@ -2,13 +2,15 @@ package mongo
import (
"fmt"
"strconv"
"testing"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"strconv"
"testing"
)
type ColTestObject struct {
@@ -49,30 +51,38 @@ func cleanupColTest(to *ColTestObject) {
}
func TestGetMongoCol(t *testing.T) {
// Test getting a MongoDB collection with default database
colName := "test_col"
// Get collection and verify the name matches
col := GetMongoCol(colName)
require.Equal(t, colName, col.c.Name())
}
func TestGetMongoColWithDb(t *testing.T) {
// Test getting a MongoDB collection with specified database
dbName := "test_db"
colName := "test_col"
// Get database instance first
db := GetMongoDb(dbName)
// Get collection with specific database and verify both names
col := GetMongoColWithDb(colName, db)
require.Equal(t, colName, col.c.Name())
require.Equal(t, dbName, col.db.Name())
}
func TestCol_Insert(t *testing.T) {
// Setup test environment
to, err := setupColTest()
require.Nil(t, err)
// Insert a single document
id, err := to.col.Insert(bson.M{"key": "value"})
require.Nil(t, err)
require.IsType(t, primitive.ObjectID{}, id)
// Verify the inserted document
var doc map[string]string
err = to.col.FindId(id).One(&doc)
require.Nil(t, err)
@@ -82,36 +92,48 @@ func TestCol_Insert(t *testing.T) {
}
func TestCol_InsertMany(t *testing.T) {
// Setup test environment
to, err := setupColTest()
require.Nil(t, err)
// Create test data: 10 documents with incrementing values
n := 10
var docs []interface{}
for i := 0; i < n; i++ {
docs = append(docs, bson.M{"key": fmt.Sprintf("value-%d", i)})
}
// Insert multiple documents at once
ids, err := to.col.InsertMany(docs)
require.Nil(t, err)
require.Equal(t, n, len(ids))
// Verify all documents were inserted correctly
// Retrieve all documents sorted by _id
var resDocs []map[string]string
err = to.col.Find(nil, &FindOptions{Sort: bson.D{{"_id", 1}}}).All(&resDocs)
require.Nil(t, err)
require.Equal(t, n, len(resDocs))
// Verify each document has correct value
for i, doc := range resDocs {
require.Equal(t, fmt.Sprintf("value-%d", i), doc["key"])
}
// Cleanup test environment
cleanupColTest(to)
}
func TestCol_UpdateId(t *testing.T) {
// Setup test environment
to, err := setupColTest()
require.Nil(t, err)
// Insert initial document
id, err := to.col.Insert(bson.M{"key": "old-value"})
require.Nil(t, err)
// Update document by ID using $set operator
err = to.col.UpdateId(id, bson.M{
"$set": bson.M{
"key": "new-value",
@@ -119,6 +141,7 @@ func TestCol_UpdateId(t *testing.T) {
})
require.Nil(t, err)
// Verify the update
var doc map[string]string
err = to.col.FindId(id).One(&doc)
require.Nil(t, err)
@@ -128,15 +151,18 @@ func TestCol_UpdateId(t *testing.T) {
}
func TestCol_Update(t *testing.T) {
// Setup test environment
to, err := setupColTest()
require.Nil(t, err)
// Prepare test documents
n := 10
var docs []interface{}
for i := 0; i < n; i++ {
docs = append(docs, bson.M{"key": fmt.Sprintf("old-value-%d", i)})
}
// Update all documents (nil query means update all)
err = to.col.Update(nil, bson.M{
"$set": bson.M{
"key": "new-value",
@@ -144,6 +170,7 @@ func TestCol_Update(t *testing.T) {
})
require.Nil(t, err)
// Verify all documents were updated
var resDocs []map[string]string
err = to.col.Find(nil, &FindOptions{Sort: bson.D{{"_id", 1}}}).All(&resDocs)
require.Nil(t, err)
@@ -155,20 +182,25 @@ func TestCol_Update(t *testing.T) {
}
func TestCol_ReplaceId(t *testing.T) {
// Setup test environment
to, err := setupColTest()
require.Nil(t, err)
// Insert initial document
id, err := to.col.Insert(bson.M{"key": "old-value"})
require.Nil(t, err)
// Get the document and modify it
var doc map[string]interface{}
err = to.col.FindId(id).One(&doc)
require.Nil(t, err)
doc["key"] = "new-value"
// Replace the entire document
err = to.col.ReplaceId(id, doc)
require.Nil(t, err)
// Verify the replacement
err = to.col.FindId(id).One(doc)
require.Nil(t, err)
require.Equal(t, "new-value", doc["key"])
@@ -199,15 +231,19 @@ func TestCol_Replace(t *testing.T) {
}
func TestCol_DeleteId(t *testing.T) {
// Setup test environment
to, err := setupColTest()
require.Nil(t, err)
// Insert a document to be deleted
id, err := to.col.Insert(bson.M{"key": "value"})
require.Nil(t, err)
// Delete the document by ID
err = to.col.DeleteId(id)
require.Nil(t, err)
// Verify deletion by checking collection count
total, err := to.col.Count(nil)
require.Nil(t, err)
require.Equal(t, 0, total)
@@ -261,9 +297,11 @@ func TestCol_FindId(t *testing.T) {
}
func TestCol_Find(t *testing.T) {
// Setup test environment
to, err := setupColTest()
require.Nil(t, err)
// Insert test documents
n := 10
var docs []interface{}
for i := 0; i < n; i++ {
@@ -276,26 +314,31 @@ func TestCol_Find(t *testing.T) {
require.Nil(t, err)
require.Equal(t, n, len(ids))
// Test 1: Find all documents
err = to.col.Find(nil, nil).All(&docs)
require.Nil(t, err)
require.Equal(t, n, len(docs))
// Test 2: Find documents with conditional query ($gte)
err = to.col.Find(bson.M{"key": bson.M{"$gte": fmt.Sprintf("value-%d", 5)}}, nil).All(&docs)
require.Nil(t, err)
require.Equal(t, n-5, len(docs))
// Test 3: Test skip functionality
err = to.col.Find(nil, &FindOptions{
Skip: 5,
}).All(&docs)
require.Nil(t, err)
require.Equal(t, n-5, len(docs))
// Test 4: Test limit functionality
err = to.col.Find(nil, &FindOptions{
Limit: 5,
}).All(&docs)
require.Nil(t, err)
require.Equal(t, 5, len(docs))
// Test 5: Test ascending sort
var resDocs []TestDocument
err = to.col.Find(nil, &FindOptions{
Sort: bson.D{{"key", 1}},
@@ -304,6 +347,7 @@ func TestCol_Find(t *testing.T) {
require.Greater(t, len(resDocs), 0)
require.Equal(t, "value-0", resDocs[0].Key)
// Test 6: Test descending sort
err = to.col.Find(nil, &FindOptions{
Sort: bson.D{{"key", -1}},
}).All(&resDocs)
@@ -311,23 +355,29 @@ func TestCol_Find(t *testing.T) {
require.Greater(t, len(resDocs), 0)
require.Equal(t, fmt.Sprintf("value-%d", n-1), resDocs[0].Key)
// Test 7: Test array query with $in operator
var resDocs2 []TestDocument
err = to.col.Find(bson.M{"tags": bson.M{"$in": []string{"test tag"}}}, nil).All(&resDocs2)
require.Nil(t, err)
require.Greater(t, len(resDocs2), 0)
// Cleanup test environment
cleanupColTest(to)
}
func TestCol_CreateIndex(t *testing.T) {
// Setup test environment
to, err := setupColTest()
require.Nil(t, err)
// Create a single index on the 'key' field
err = to.col.CreateIndex(mongo.IndexModel{
Keys: bson.D{{"key", 1}},
})
require.Nil(t, err)
// Verify index creation
// Should have 2 indexes: default _id index and our new index
indexes, err := to.col.ListIndexes()
require.Nil(t, err)
require.Equal(t, 2, len(indexes))
@@ -336,22 +386,12 @@ func TestCol_CreateIndex(t *testing.T) {
}
func TestCol_Aggregate(t *testing.T) {
// Setup test environment
to, err := setupColTest()
require.Nil(t, err)
n := 10
v := 2
var docs []interface{}
for i := 0; i < n; i++ {
docs = append(docs, TestDocument{
Key: fmt.Sprintf("%d", i%2),
Value: v,
})
}
ids, err := to.col.InsertMany(docs)
require.Nil(t, err)
require.Equal(t, n, len(ids))
// Define aggregation pipeline
var results []TestAggregateResult
pipeline := mongo.Pipeline{
{
{
@@ -376,7 +416,26 @@ func TestCol_Aggregate(t *testing.T) {
},
},
}
var results []TestAggregateResult
// Test empty collection aggregation
err = to.col.Aggregate(pipeline, nil).All(&results)
assert.Nil(t, err)
// Insert test documents
n := 10 // total documents
v := 2 // value for each document
var docs []interface{}
for i := 0; i < n; i++ {
docs = append(docs, TestDocument{
Key: fmt.Sprintf("%d", i%2), // alternates between "0" and "1"
Value: v,
})
}
ids, err := to.col.InsertMany(docs)
require.Nil(t, err)
require.Equal(t, n, len(ids))
// Run aggregation on populate
err = to.col.Aggregate(pipeline, nil).All(&results)
require.Nil(t, err)
require.Equal(t, 2, len(results))
@@ -392,6 +451,9 @@ func TestCol_CreateIndexes(t *testing.T) {
to, err := setupColTest()
require.Nil(t, err)
indexes, err := to.col.ListIndexes()
assert.Nil(t, err)
err = to.col.CreateIndexes([]mongo.IndexModel{
{
Keys: bson.D{{"key", 1}},
@@ -402,7 +464,7 @@ func TestCol_CreateIndexes(t *testing.T) {
})
require.Nil(t, err)
indexes, err := to.col.ListIndexes()
indexes, err = to.col.ListIndexes()
require.Nil(t, err)
require.Equal(t, 3, len(indexes))
@@ -413,6 +475,9 @@ func TestCol_DeleteIndex(t *testing.T) {
to, err := setupColTest()
require.Nil(t, err)
err = to.col.DeleteIndex("non-existent-index")
assert.Nil(t, err)
err = to.col.CreateIndex(mongo.IndexModel{
Keys: bson.D{{"key", 1}},
})
@@ -420,29 +485,35 @@ func TestCol_DeleteIndex(t *testing.T) {
indexes, err := to.col.ListIndexes()
require.Nil(t, err)
require.Equal(t, 2, len(indexes))
assert.Equal(t, 2, len(indexes))
for _, index := range indexes {
name, ok := index["name"].(string)
require.True(t, ok)
assert.True(t, ok)
if name != "_id_" {
err = to.col.DeleteIndex(name)
require.Nil(t, err)
assert.Nil(t, err)
break
}
}
indexes, err = to.col.ListIndexes()
require.Nil(t, err)
require.Equal(t, 1, len(indexes))
assert.Equal(t, 1, len(indexes))
cleanupColTest(to)
}
func TestCol_DeleteIndexes(t *testing.T) {
// Setup test environment
to, err := setupColTest()
require.Nil(t, err)
// Delete all existing indexes
err = to.col.DeleteAllIndexes()
assert.Nil(t, err)
// Create multiple test indexes
err = to.col.CreateIndexes([]mongo.IndexModel{
{
Keys: bson.D{{"key", 1}},
@@ -453,9 +524,11 @@ func TestCol_DeleteIndexes(t *testing.T) {
})
require.Nil(t, err)
// Delete all indexes except the default _id index
err = to.col.DeleteAllIndexes()
require.Nil(t, err)
// Verify only default _id index remains
indexes, err := to.col.ListIndexes()
require.Nil(t, err)
require.Equal(t, 1, len(indexes))

21
docker/base-image/Dockerfile
View File

@@ -1,6 +1,9 @@
# Build stage
FROM ubuntu:22.04 AS builder
FROM ubuntu:22.04
# Non-interactive mode
ENV DEBIAN_FRONTEND=noninteractive
# Install dependencies
COPY ./install /app/install
RUN bash /app/install/deps/deps.sh && \
bash /app/install/python/python.sh && \
@@ -8,21 +11,13 @@ RUN bash /app/install/deps/deps.sh && \
bash /app/install/node/node.sh && \
bash /app/install/browser/browser.sh
# Final stage
FROM ubuntu:22.04
ENV DEBIAN_FRONTEND=noninteractive
COPY --from=builder /usr/local/bin /usr/local/bin
COPY --from=builder /usr/local/lib /usr/local/lib
COPY --from=builder /usr/lib /usr/lib
COPY --from=builder /app/install /app/install
# working directory
# Working directory
WORKDIR /app/backend
# node path
# Node path
ENV NODE_PATH=/usr/lib/node_modules
# timezone environment
# Timezone environment
ENV TZ=Asia/Shanghai
# language environment

25
docker/base-image/install/browser/browser.sh
View File

@@ -1,9 +1,7 @@
#!/bin/bash
# Source nvm environment
export NVM_DIR="$HOME/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"
# Source nvm environment from profile.d instead
source /etc/profile.d/node-env.sh
# Version - using "stable" for installation but not for verification
version="stable"
@@ -34,7 +32,7 @@ echo "Chrome binary path: $CHROME_BIN"
# Update version variable for ChromeDriver
version="$ACTUAL_VERSION"
# Add chrome to PATH
# Create symbolic links
ln -sf "$CHROME_BIN" /usr/local/bin/google-chrome
# Verify chrome is installed (with more detailed error message)
@@ -54,7 +52,7 @@ echo "ChromeDriver installation output: $CHROMEDRIVER_OUTPUT"
CHROMEDRIVER_BIN=$(echo "$CHROMEDRIVER_OUTPUT" | awk '{print $2}')
echo "ChromeDriver binary path: $CHROMEDRIVER_BIN"
# Add chromedriver to PATH
# Create symbolic links
ln -sf "$CHROMEDRIVER_BIN" /usr/local/bin/chromedriver
# Verify chromedriver is installed
@@ -99,3 +97,18 @@ python3 test.py
# Clean up
cd -
rm -rf "$TEST_DIR"
# After successful Chrome and ChromeDriver installation, create persistent env config
cat > /etc/profile.d/browser-env.sh << 'EOF'
# Chrome and ChromeDriver paths
export CHROME_BIN="$(which google-chrome)"
export CHROMEDRIVER_BIN="$(which chromedriver)"
# Common Chrome flags for headless environments
export CHROME_FLAGS="--headless --no-sandbox --disable-dev-shm-usage"
EOF
# Make the file executable
chmod +x /etc/profile.d/browser-env.sh
# Source it immediately
source /etc/profile.d/browser-env.sh

49
docker/base-image/install/deps/deps.sh
View File

@@ -1,24 +1,39 @@
#!/bin/bash
# ensure directory mode of /tmp
# Ensure directory mode of /tmp is world-writable (readable, writable, executable by all users)
# This is important for temporary file operations in containerized environments
chmod 777 /tmp
# update
# Update the package index files from the repositories
# This ensures we get the latest versions of packages
apt-get update
# common deps
# Install common dependencies with detailed explanations
# -y flag means "yes" to all prompts (non-interactive installation)
apt-get install -y \
curl \
git \
net-tools \
iputils-ping \
ntp \
ntpdate \
nginx \
wget \
dumb-init \
cloc \
unzip \
build-essential \
gnupg2 \
libc6
# Network and File Transfer Utilities
curl \ # Modern HTTP client, useful for API requests and downloads
wget \ # Another download utility, often used in scripts
# Version Control
git \ # Distributed version control system
# Network Diagnostics and Monitoring
net-tools \ # Traditional networking tools (netstat, ifconfig, etc.)
iputils-ping \ # Tools for testing network connectivity (ping)
# Time Synchronization
ntp \ # Network Time Protocol daemon for time sync
ntpdate \ # Client for one-time NTP sync
# Web Server
nginx \ # High-performance HTTP server and reverse proxy
# File Operations
unzip \ # Extract .zip archives
# Security and Encryption
gnupg2 \ # GNU Privacy Guard for encryption and signing
# System Libraries
libc6 # GNU C Library - essential for running C programs

15
docker/base-image/install/go/go.sh
View File

@@ -5,15 +5,18 @@ version="1.22.9"
# Install goenv
git clone https://github.com/go-nv/goenv.git ~/.goenv
# Add goenv to path
echo 'export GOENV_ROOT="$HOME/.goenv"' >> ~/.bashrc
echo 'export PATH="$GOENV_ROOT/bin:$PATH"' >> ~/.bashrc
echo 'eval "$(goenv init -)"' >> ~/.bashrc
# Ensure changes take effect immediately
# Create persistent environment config
cat > /etc/profile.d/goenv.sh << 'EOF'
export GOENV_ROOT="$HOME/.goenv"
export PATH="$GOENV_ROOT/bin:$PATH"
eval "$(goenv init -)"
EOF
# Make the file executable
chmod +x /etc/profile.d/goenv.sh
# Source it immediately for the rest of the installation
source /etc/profile.d/goenv.sh
# Install go
goenv install ${version}

34
docker/base-image/install/java/java.sh
View File

@@ -5,10 +5,24 @@ version="11.0.12-open"
# Install SDKMAN!
curl -s "https://get.sdkman.io" | bash
# Source SDKMAN!
source "$HOME/.sdkman/bin/sdkman-init.sh"
# Create persistent environment config for SDKMAN
cat > /etc/profile.d/sdkman.sh << 'EOF'
# SDKMAN configuration
export SDKMAN_DIR="$HOME/.sdkman"
[[ -s "$SDKMAN_DIR/bin/sdkman-init.sh" ]] && source "$SDKMAN_DIR/bin/sdkman-init.sh"
# Install Java 11 (you can specify vendor, e.g., 11.0.12-open for OpenJDK)
# Java environment variables
export JAVA_HOME="$SDKMAN_DIR/candidates/java/current"
export PATH="$JAVA_HOME/bin:$PATH"
EOF
# Make the file executable
chmod +x /etc/profile.d/sdkman.sh
# Source it immediately for the rest of the installation
source /etc/profile.d/sdkman.sh
# Install Java 11 (OpenJDK)
sdk install java ${version}
# Set Java 11 as default
@@ -18,18 +32,18 @@ sdk default java ${version}
ln -sf "$(sdkman which java)" /usr/local/bin/java
ln -sf "$(sdkman which javac)" /usr/local/bin/javac
# Verify
# Verify installations
java_version=$(java -version)
if [[ $java_version =~ "${version}" ]]; then
:
:
else
echo "ERROR: java version does not match. expect \"${version}\", but actual is \"${java_version}\""
exit 1
echo "ERROR: java version does not match. expect \"${version}\", but actual is \"${java_version}\""
exit 1
fi
javac_version=$(javac -version)
if [[ $javac_version =~ "${version}" ]]; then
:
:
else
echo "ERROR: javac version does not match. expect \"${version}\", but actual is \"${javac_version}\""
exit 1
echo "ERROR: javac version does not match. expect \"${version}\", but actual is \"${javac_version}\""
exit 1
fi

17
docker/base-image/install/node/node.sh
View File

@@ -5,15 +5,16 @@ version="22"
# Install nvm (Node Version Manager)
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.0/install.sh | bash
# Add nvm to path
echo 'export NVM_DIR="$HOME/.nvm"' >> ~/.bashrc
echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' >> ~/.bashrc
echo '[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion' >> ~/.bashrc
# Create a file in /etc/profile.d/
cat > /etc/profile.d/node-env.sh << 'EOF'
export NVM_DIR="/root/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"
export NODE_PATH=/usr/lib/node_modules
EOF
# Ensure changes take effect immediately
export NVM_DIR="$HOME/.nvm"
[[ -s "$NVM_DIR/nvm.sh" ]] && \. "$NVM_DIR/nvm.sh" # This loads nvm
[[ -s "$NVM_DIR/bash_completion" ]] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion
# Make the file executable
chmod +x /etc/profile.d/node-env.sh
# Download and install Node.js (you may need to restart the terminal)
nvm install ${version}

16
docker/base-image/install/python/python.sh
View File

@@ -25,17 +25,19 @@ apt-get install -y \
# Install pyenv
curl https://pyenv.run | bash
# Add pyenv to path
echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc
echo '[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc
echo 'eval "$(pyenv init -)"' >> ~/.bashrc
echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc
# Ensure changes take effect immediately
# Create a file in /etc/profile.d/
cat > /etc/profile.d/pyenv.sh << 'EOF'
export PYENV_ROOT="$HOME/.pyenv"
[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"
eval "$(pyenv init -)"
eval "$(pyenv virtualenv-init -)"
EOF
# Make the file executable
chmod +x /etc/profile.d/pyenv.sh
# Source it immediately for the rest of the installation
source /etc/profile.d/pyenv.sh
# Install python ${version} via pyenv
pyenv install ${version}

34
docker/ghcr/docker-compose.yml Normal file
View File

@@ -0,0 +1,34 @@
version: '3.3'
services:
master:
image: ghcr.io/crawlab-team/crawlab:${CRAWLAB_TAG:-develop}
container_name: crawlab_ghcr_master
environment:
CRAWLAB_NODE_MASTER: "Y"
CRAWLAB_MONGO_HOST: "mongo"
ports:
- "8080:8080"
depends_on:
- mongo
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000/health", "||", "exit", "1"]
interval: 30s
timeout: 10s
retries: 5
worker:
image: ghcr.io/crawlab-team/crawlab:${CRAWLAB_TAG:-develop}
container_name: crawlab_ghcr_worker
environment:
CRAWLAB_NODE_MASTER: "N"
CRAWLAB_MASTER_HOST: "master"
depends_on:
- master
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000/health", "||", "exit", "1"]
interval: 30s
timeout: 10s
retries: 5
mongo:
image: mongo:5