diff --git a/.github/workflows/docker-crawlab.yml b/.github/workflows/docker-crawlab.yml index 22cdd58b..6aaaaf51 100644 --- a/.github/workflows/docker-crawlab.yml +++ b/.github/workflows/docker-crawlab.yml @@ -305,7 +305,7 @@ jobs: ports: - 8080:8080 options: >- - --health-cmd "curl -f http://localhost:8000/health || exit 1" + --health-cmd "curl -f http://localhost:8080/api/health || exit 1" --health-interval 30s --health-timeout 10s --health-retries 5 @@ -314,8 +314,10 @@ jobs: env: CRAWLAB_NODE_MASTER: N CRAWLAB_MASTER_HOST: master + ports: + - 8001:8000 options: >- - --health-cmd "curl -f http://localhost:8000/health || exit 1" + --health-cmd "curl -f http://localhost:8001/health || exit 1" --health-interval 30s --health-timeout 10s --health-retries 5 diff --git a/core/apps/api.go b/core/apps/api.go index c6b062ec..047ca535 100644 --- a/core/apps/api.go +++ b/core/apps/api.go @@ -50,13 +50,14 @@ func (app *Api) Start() { if err != nil { panic(err) } + log.Infof("api server listening on %s", address) // serve if err := http.Serve(app.ln, app.app); err != nil { if !errors.Is(err, http.ErrServerClosed) { - log.Error("run server error:" + err.Error()) + log.Errorf("run api server error: %v", err) } else { - log.Info("server graceful down") + log.Info("api server graceful down") } } } @@ -70,7 +71,7 @@ func (app *Api) Stop() { defer cancel() if err := app.srv.Shutdown(ctx); err != nil { - log.Error("run server error:" + err.Error()) + log.Errorf("shutdown api server error: %v", err) } } @@ -89,11 +90,9 @@ func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) ( } func newApi() *Api { - api := &Api{ + return &Api{ app: gin.New(), } - api.Init() - return api } var api *Api diff --git a/core/apps/server.go b/core/apps/server.go index b148bccc..7cc2bdf8 100644 --- a/core/apps/server.go +++ b/core/apps/server.go @@ -16,9 +16,6 @@ type Server struct { // modules nodeSvc interfaces.NodeService api *Api - - // internals - quit chan int } func (app *Server) Init() { @@ -32,7 +29,7 @@ func (app *Server) Init() { func (app *Server) Start() { if utils.IsMaster() { // start api - go app.api.Start() + go start(app.api) } // start node service @@ -40,12 +37,11 @@ func (app *Server) Start() { } func (app *Server) Wait() { - <-app.quit + utils.DefaultWait() } func (app *Server) Stop() { app.api.Stop() - app.quit <- 1 } func (app *Server) GetApi() ApiApp { @@ -70,9 +66,7 @@ func (app *Server) initPprof() { func newServer() App { // server - svr := &Server{ - quit: make(chan int, 1), - } + svr := &Server{} // master modules if utils.IsMaster() { diff --git a/core/apps/utils.go b/core/apps/utils.go index a524b467..57bc1da7 100644 --- a/core/apps/utils.go +++ b/core/apps/utils.go @@ -3,7 +3,6 @@ package apps import ( "fmt" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/trace" ) @@ -18,10 +17,6 @@ func start(app App) { app.Stop() } -func DefaultWait() { - utils.DefaultWait() -} - func initModule(name string, fn func() error) (err error) { if err := fn(); err != nil { log.Error(fmt.Sprintf("init %s error: %s", name, err.Error())) diff --git a/db/mongo/client_test.go b/db/mongo/client_test.go deleted file mode 100644 index 32f54531..00000000 --- a/db/mongo/client_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package mongo - -import ( - "github.com/stretchr/testify/require" - "testing" -) - -func setupMongoTest() (err error) { - return nil -} - -func cleanupMongoTest() { -} - -func TestMongoInitMongo(t *testing.T) { - err := setupMongoTest() - require.Nil(t, err) - - _, err = GetMongoClient() - require.Nil(t, err) - - cleanupMongoTest() -} diff --git a/db/mongo/col.go b/db/mongo/col.go index 925270f0..d5288595 100644 --- a/db/mongo/col.go +++ b/db/mongo/col.go @@ -3,7 +3,6 @@ package mongo import ( "context" "errors" - "github.com/apex/log" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -260,6 +259,10 @@ func (col *Col) MustCreateIndexes(indexModels []mongo.IndexModel) { func (col *Col) DeleteIndex(name string) (err error) { _, err = col.c.Indexes().DropOne(col.ctx, name) if err != nil { + var e mongo.CommandError + if errors.As(err, &e) && e.HasErrorCode(26) { + return nil + } log.Errorf("error deleting index: %v", err) return err } @@ -269,6 +272,10 @@ func (col *Col) DeleteIndex(name string) (err error) { func (col *Col) DeleteAllIndexes() (err error) { _, err = col.c.Indexes().DropAll(col.ctx) if err != nil { + var e mongo.CommandError + if errors.As(err, &e) && e.HasErrorCode(26) { + return nil + } log.Errorf("error deleting all indexes: %v", err) return err } diff --git a/db/mongo/col_test.go b/db/mongo/col_test.go index d9f1462e..467ed672 100644 --- a/db/mongo/col_test.go +++ b/db/mongo/col_test.go @@ -2,13 +2,15 @@ package mongo import ( "fmt" + "strconv" + "testing" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" - "strconv" - "testing" ) type ColTestObject struct { @@ -49,30 +51,38 @@ func cleanupColTest(to *ColTestObject) { } func TestGetMongoCol(t *testing.T) { + // Test getting a MongoDB collection with default database colName := "test_col" + // Get collection and verify the name matches col := GetMongoCol(colName) require.Equal(t, colName, col.c.Name()) } func TestGetMongoColWithDb(t *testing.T) { + // Test getting a MongoDB collection with specified database dbName := "test_db" colName := "test_col" + // Get database instance first db := GetMongoDb(dbName) + // Get collection with specific database and verify both names col := GetMongoColWithDb(colName, db) require.Equal(t, colName, col.c.Name()) require.Equal(t, dbName, col.db.Name()) } func TestCol_Insert(t *testing.T) { + // Setup test environment to, err := setupColTest() require.Nil(t, err) + // Insert a single document id, err := to.col.Insert(bson.M{"key": "value"}) require.Nil(t, err) require.IsType(t, primitive.ObjectID{}, id) + // Verify the inserted document var doc map[string]string err = to.col.FindId(id).One(&doc) require.Nil(t, err) @@ -82,36 +92,48 @@ func TestCol_Insert(t *testing.T) { } func TestCol_InsertMany(t *testing.T) { + // Setup test environment to, err := setupColTest() require.Nil(t, err) + // Create test data: 10 documents with incrementing values n := 10 var docs []interface{} for i := 0; i < n; i++ { docs = append(docs, bson.M{"key": fmt.Sprintf("value-%d", i)}) } + + // Insert multiple documents at once ids, err := to.col.InsertMany(docs) require.Nil(t, err) require.Equal(t, n, len(ids)) + // Verify all documents were inserted correctly + // Retrieve all documents sorted by _id var resDocs []map[string]string err = to.col.Find(nil, &FindOptions{Sort: bson.D{{"_id", 1}}}).All(&resDocs) require.Nil(t, err) require.Equal(t, n, len(resDocs)) + + // Verify each document has correct value for i, doc := range resDocs { require.Equal(t, fmt.Sprintf("value-%d", i), doc["key"]) } + // Cleanup test environment cleanupColTest(to) } func TestCol_UpdateId(t *testing.T) { + // Setup test environment to, err := setupColTest() require.Nil(t, err) + // Insert initial document id, err := to.col.Insert(bson.M{"key": "old-value"}) require.Nil(t, err) + // Update document by ID using $set operator err = to.col.UpdateId(id, bson.M{ "$set": bson.M{ "key": "new-value", @@ -119,6 +141,7 @@ func TestCol_UpdateId(t *testing.T) { }) require.Nil(t, err) + // Verify the update var doc map[string]string err = to.col.FindId(id).One(&doc) require.Nil(t, err) @@ -128,15 +151,18 @@ func TestCol_UpdateId(t *testing.T) { } func TestCol_Update(t *testing.T) { + // Setup test environment to, err := setupColTest() require.Nil(t, err) + // Prepare test documents n := 10 var docs []interface{} for i := 0; i < n; i++ { docs = append(docs, bson.M{"key": fmt.Sprintf("old-value-%d", i)}) } + // Update all documents (nil query means update all) err = to.col.Update(nil, bson.M{ "$set": bson.M{ "key": "new-value", @@ -144,6 +170,7 @@ func TestCol_Update(t *testing.T) { }) require.Nil(t, err) + // Verify all documents were updated var resDocs []map[string]string err = to.col.Find(nil, &FindOptions{Sort: bson.D{{"_id", 1}}}).All(&resDocs) require.Nil(t, err) @@ -155,20 +182,25 @@ func TestCol_Update(t *testing.T) { } func TestCol_ReplaceId(t *testing.T) { + // Setup test environment to, err := setupColTest() require.Nil(t, err) + // Insert initial document id, err := to.col.Insert(bson.M{"key": "old-value"}) require.Nil(t, err) + // Get the document and modify it var doc map[string]interface{} err = to.col.FindId(id).One(&doc) require.Nil(t, err) doc["key"] = "new-value" + // Replace the entire document err = to.col.ReplaceId(id, doc) require.Nil(t, err) + // Verify the replacement err = to.col.FindId(id).One(doc) require.Nil(t, err) require.Equal(t, "new-value", doc["key"]) @@ -199,15 +231,19 @@ func TestCol_Replace(t *testing.T) { } func TestCol_DeleteId(t *testing.T) { + // Setup test environment to, err := setupColTest() require.Nil(t, err) + // Insert a document to be deleted id, err := to.col.Insert(bson.M{"key": "value"}) require.Nil(t, err) + // Delete the document by ID err = to.col.DeleteId(id) require.Nil(t, err) + // Verify deletion by checking collection count total, err := to.col.Count(nil) require.Nil(t, err) require.Equal(t, 0, total) @@ -261,9 +297,11 @@ func TestCol_FindId(t *testing.T) { } func TestCol_Find(t *testing.T) { + // Setup test environment to, err := setupColTest() require.Nil(t, err) + // Insert test documents n := 10 var docs []interface{} for i := 0; i < n; i++ { @@ -276,26 +314,31 @@ func TestCol_Find(t *testing.T) { require.Nil(t, err) require.Equal(t, n, len(ids)) + // Test 1: Find all documents err = to.col.Find(nil, nil).All(&docs) require.Nil(t, err) require.Equal(t, n, len(docs)) + // Test 2: Find documents with conditional query ($gte) err = to.col.Find(bson.M{"key": bson.M{"$gte": fmt.Sprintf("value-%d", 5)}}, nil).All(&docs) require.Nil(t, err) require.Equal(t, n-5, len(docs)) + // Test 3: Test skip functionality err = to.col.Find(nil, &FindOptions{ Skip: 5, }).All(&docs) require.Nil(t, err) require.Equal(t, n-5, len(docs)) + // Test 4: Test limit functionality err = to.col.Find(nil, &FindOptions{ Limit: 5, }).All(&docs) require.Nil(t, err) require.Equal(t, 5, len(docs)) + // Test 5: Test ascending sort var resDocs []TestDocument err = to.col.Find(nil, &FindOptions{ Sort: bson.D{{"key", 1}}, @@ -304,6 +347,7 @@ func TestCol_Find(t *testing.T) { require.Greater(t, len(resDocs), 0) require.Equal(t, "value-0", resDocs[0].Key) + // Test 6: Test descending sort err = to.col.Find(nil, &FindOptions{ Sort: bson.D{{"key", -1}}, }).All(&resDocs) @@ -311,23 +355,29 @@ func TestCol_Find(t *testing.T) { require.Greater(t, len(resDocs), 0) require.Equal(t, fmt.Sprintf("value-%d", n-1), resDocs[0].Key) + // Test 7: Test array query with $in operator var resDocs2 []TestDocument err = to.col.Find(bson.M{"tags": bson.M{"$in": []string{"test tag"}}}, nil).All(&resDocs2) require.Nil(t, err) require.Greater(t, len(resDocs2), 0) + // Cleanup test environment cleanupColTest(to) } func TestCol_CreateIndex(t *testing.T) { + // Setup test environment to, err := setupColTest() require.Nil(t, err) + // Create a single index on the 'key' field err = to.col.CreateIndex(mongo.IndexModel{ Keys: bson.D{{"key", 1}}, }) require.Nil(t, err) + // Verify index creation + // Should have 2 indexes: default _id index and our new index indexes, err := to.col.ListIndexes() require.Nil(t, err) require.Equal(t, 2, len(indexes)) @@ -336,22 +386,12 @@ func TestCol_CreateIndex(t *testing.T) { } func TestCol_Aggregate(t *testing.T) { + // Setup test environment to, err := setupColTest() require.Nil(t, err) - n := 10 - v := 2 - var docs []interface{} - for i := 0; i < n; i++ { - docs = append(docs, TestDocument{ - Key: fmt.Sprintf("%d", i%2), - Value: v, - }) - } - ids, err := to.col.InsertMany(docs) - require.Nil(t, err) - require.Equal(t, n, len(ids)) - + // Define aggregation pipeline + var results []TestAggregateResult pipeline := mongo.Pipeline{ { { @@ -376,7 +416,26 @@ func TestCol_Aggregate(t *testing.T) { }, }, } - var results []TestAggregateResult + + // Test empty collection aggregation + err = to.col.Aggregate(pipeline, nil).All(&results) + assert.Nil(t, err) + + // Insert test documents + n := 10 // total documents + v := 2 // value for each document + var docs []interface{} + for i := 0; i < n; i++ { + docs = append(docs, TestDocument{ + Key: fmt.Sprintf("%d", i%2), // alternates between "0" and "1" + Value: v, + }) + } + ids, err := to.col.InsertMany(docs) + require.Nil(t, err) + require.Equal(t, n, len(ids)) + + // Run aggregation on populate err = to.col.Aggregate(pipeline, nil).All(&results) require.Nil(t, err) require.Equal(t, 2, len(results)) @@ -392,6 +451,9 @@ func TestCol_CreateIndexes(t *testing.T) { to, err := setupColTest() require.Nil(t, err) + indexes, err := to.col.ListIndexes() + assert.Nil(t, err) + err = to.col.CreateIndexes([]mongo.IndexModel{ { Keys: bson.D{{"key", 1}}, @@ -402,7 +464,7 @@ func TestCol_CreateIndexes(t *testing.T) { }) require.Nil(t, err) - indexes, err := to.col.ListIndexes() + indexes, err = to.col.ListIndexes() require.Nil(t, err) require.Equal(t, 3, len(indexes)) @@ -413,6 +475,9 @@ func TestCol_DeleteIndex(t *testing.T) { to, err := setupColTest() require.Nil(t, err) + err = to.col.DeleteIndex("non-existent-index") + assert.Nil(t, err) + err = to.col.CreateIndex(mongo.IndexModel{ Keys: bson.D{{"key", 1}}, }) @@ -420,29 +485,35 @@ func TestCol_DeleteIndex(t *testing.T) { indexes, err := to.col.ListIndexes() require.Nil(t, err) - require.Equal(t, 2, len(indexes)) + assert.Equal(t, 2, len(indexes)) for _, index := range indexes { name, ok := index["name"].(string) - require.True(t, ok) + assert.True(t, ok) if name != "_id_" { err = to.col.DeleteIndex(name) - require.Nil(t, err) + assert.Nil(t, err) break } } indexes, err = to.col.ListIndexes() require.Nil(t, err) - require.Equal(t, 1, len(indexes)) + assert.Equal(t, 1, len(indexes)) cleanupColTest(to) } func TestCol_DeleteIndexes(t *testing.T) { + // Setup test environment to, err := setupColTest() require.Nil(t, err) + // Delete all existing indexes + err = to.col.DeleteAllIndexes() + assert.Nil(t, err) + + // Create multiple test indexes err = to.col.CreateIndexes([]mongo.IndexModel{ { Keys: bson.D{{"key", 1}}, @@ -453,9 +524,11 @@ func TestCol_DeleteIndexes(t *testing.T) { }) require.Nil(t, err) + // Delete all indexes except the default _id index err = to.col.DeleteAllIndexes() require.Nil(t, err) + // Verify only default _id index remains indexes, err := to.col.ListIndexes() require.Nil(t, err) require.Equal(t, 1, len(indexes)) diff --git a/docker/base-image/Dockerfile b/docker/base-image/Dockerfile index 4aa4b72f..7e761eed 100644 --- a/docker/base-image/Dockerfile +++ b/docker/base-image/Dockerfile @@ -1,6 +1,9 @@ -# Build stage -FROM ubuntu:22.04 AS builder +FROM ubuntu:22.04 + +# Non-interactive mode ENV DEBIAN_FRONTEND=noninteractive + +# Install dependencies COPY ./install /app/install RUN bash /app/install/deps/deps.sh && \ bash /app/install/python/python.sh && \ @@ -8,21 +11,13 @@ RUN bash /app/install/deps/deps.sh && \ bash /app/install/node/node.sh && \ bash /app/install/browser/browser.sh -# Final stage -FROM ubuntu:22.04 -ENV DEBIAN_FRONTEND=noninteractive -COPY --from=builder /usr/local/bin /usr/local/bin -COPY --from=builder /usr/local/lib /usr/local/lib -COPY --from=builder /usr/lib /usr/lib -COPY --from=builder /app/install /app/install - -# working directory +# Working directory WORKDIR /app/backend -# node path +# Node path ENV NODE_PATH=/usr/lib/node_modules -# timezone environment +# Timezone environment ENV TZ=Asia/Shanghai # language environment diff --git a/docker/base-image/install/browser/browser.sh b/docker/base-image/install/browser/browser.sh index bc5cd12f..bf6c0159 100644 --- a/docker/base-image/install/browser/browser.sh +++ b/docker/base-image/install/browser/browser.sh @@ -1,9 +1,7 @@ #!/bin/bash -# Source nvm environment -export NVM_DIR="$HOME/.nvm" -[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" -[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" +# Source nvm environment from profile.d instead +source /etc/profile.d/node-env.sh # Version - using "stable" for installation but not for verification version="stable" @@ -34,7 +32,7 @@ echo "Chrome binary path: $CHROME_BIN" # Update version variable for ChromeDriver version="$ACTUAL_VERSION" -# Add chrome to PATH +# Create symbolic links ln -sf "$CHROME_BIN" /usr/local/bin/google-chrome # Verify chrome is installed (with more detailed error message) @@ -54,7 +52,7 @@ echo "ChromeDriver installation output: $CHROMEDRIVER_OUTPUT" CHROMEDRIVER_BIN=$(echo "$CHROMEDRIVER_OUTPUT" | awk '{print $2}') echo "ChromeDriver binary path: $CHROMEDRIVER_BIN" -# Add chromedriver to PATH +# Create symbolic links ln -sf "$CHROMEDRIVER_BIN" /usr/local/bin/chromedriver # Verify chromedriver is installed @@ -99,3 +97,18 @@ python3 test.py # Clean up cd - rm -rf "$TEST_DIR" + +# After successful Chrome and ChromeDriver installation, create persistent env config +cat > /etc/profile.d/browser-env.sh << 'EOF' +# Chrome and ChromeDriver paths +export CHROME_BIN="$(which google-chrome)" +export CHROMEDRIVER_BIN="$(which chromedriver)" +# Common Chrome flags for headless environments +export CHROME_FLAGS="--headless --no-sandbox --disable-dev-shm-usage" +EOF + +# Make the file executable +chmod +x /etc/profile.d/browser-env.sh + +# Source it immediately +source /etc/profile.d/browser-env.sh diff --git a/docker/base-image/install/deps/deps.sh b/docker/base-image/install/deps/deps.sh index d7e7510b..1e6f7a6c 100644 --- a/docker/base-image/install/deps/deps.sh +++ b/docker/base-image/install/deps/deps.sh @@ -1,24 +1,39 @@ #!/bin/bash -# ensure directory mode of /tmp +# Ensure directory mode of /tmp is world-writable (readable, writable, executable by all users) +# This is important for temporary file operations in containerized environments chmod 777 /tmp -# update +# Update the package index files from the repositories +# This ensures we get the latest versions of packages apt-get update -# common deps +# Install common dependencies with detailed explanations +# -y flag means "yes" to all prompts (non-interactive installation) apt-get install -y \ - curl \ - git \ - net-tools \ - iputils-ping \ - ntp \ - ntpdate \ - nginx \ - wget \ - dumb-init \ - cloc \ - unzip \ - build-essential \ - gnupg2 \ - libc6 + # Network and File Transfer Utilities + curl \ # Modern HTTP client, useful for API requests and downloads + wget \ # Another download utility, often used in scripts + + # Version Control + git \ # Distributed version control system + + # Network Diagnostics and Monitoring + net-tools \ # Traditional networking tools (netstat, ifconfig, etc.) + iputils-ping \ # Tools for testing network connectivity (ping) + + # Time Synchronization + ntp \ # Network Time Protocol daemon for time sync + ntpdate \ # Client for one-time NTP sync + + # Web Server + nginx \ # High-performance HTTP server and reverse proxy + + # File Operations + unzip \ # Extract .zip archives + + # Security and Encryption + gnupg2 \ # GNU Privacy Guard for encryption and signing + + # System Libraries + libc6 # GNU C Library - essential for running C programs diff --git a/docker/base-image/install/go/go.sh b/docker/base-image/install/go/go.sh index c6a31465..b58e4268 100644 --- a/docker/base-image/install/go/go.sh +++ b/docker/base-image/install/go/go.sh @@ -5,15 +5,18 @@ version="1.22.9" # Install goenv git clone https://github.com/go-nv/goenv.git ~/.goenv -# Add goenv to path -echo 'export GOENV_ROOT="$HOME/.goenv"' >> ~/.bashrc -echo 'export PATH="$GOENV_ROOT/bin:$PATH"' >> ~/.bashrc -echo 'eval "$(goenv init -)"' >> ~/.bashrc - -# Ensure changes take effect immediately +# Create persistent environment config +cat > /etc/profile.d/goenv.sh << 'EOF' export GOENV_ROOT="$HOME/.goenv" export PATH="$GOENV_ROOT/bin:$PATH" eval "$(goenv init -)" +EOF + +# Make the file executable +chmod +x /etc/profile.d/goenv.sh + +# Source it immediately for the rest of the installation +source /etc/profile.d/goenv.sh # Install go goenv install ${version} diff --git a/docker/base-image/install/java/java.sh b/docker/base-image/install/java/java.sh index 69f51353..8fc32e83 100644 --- a/docker/base-image/install/java/java.sh +++ b/docker/base-image/install/java/java.sh @@ -5,10 +5,24 @@ version="11.0.12-open" # Install SDKMAN! curl -s "https://get.sdkman.io" | bash -# Source SDKMAN! -source "$HOME/.sdkman/bin/sdkman-init.sh" +# Create persistent environment config for SDKMAN +cat > /etc/profile.d/sdkman.sh << 'EOF' +# SDKMAN configuration +export SDKMAN_DIR="$HOME/.sdkman" +[[ -s "$SDKMAN_DIR/bin/sdkman-init.sh" ]] && source "$SDKMAN_DIR/bin/sdkman-init.sh" -# Install Java 11 (you can specify vendor, e.g., 11.0.12-open for OpenJDK) +# Java environment variables +export JAVA_HOME="$SDKMAN_DIR/candidates/java/current" +export PATH="$JAVA_HOME/bin:$PATH" +EOF + +# Make the file executable +chmod +x /etc/profile.d/sdkman.sh + +# Source it immediately for the rest of the installation +source /etc/profile.d/sdkman.sh + +# Install Java 11 (OpenJDK) sdk install java ${version} # Set Java 11 as default @@ -18,18 +32,18 @@ sdk default java ${version} ln -sf "$(sdkman which java)" /usr/local/bin/java ln -sf "$(sdkman which javac)" /usr/local/bin/javac -# Verify +# Verify installations java_version=$(java -version) if [[ $java_version =~ "${version}" ]]; then - : + : else - echo "ERROR: java version does not match. expect \"${version}\", but actual is \"${java_version}\"" - exit 1 + echo "ERROR: java version does not match. expect \"${version}\", but actual is \"${java_version}\"" + exit 1 fi javac_version=$(javac -version) if [[ $javac_version =~ "${version}" ]]; then - : + : else - echo "ERROR: javac version does not match. expect \"${version}\", but actual is \"${javac_version}\"" - exit 1 + echo "ERROR: javac version does not match. expect \"${version}\", but actual is \"${javac_version}\"" + exit 1 fi \ No newline at end of file diff --git a/docker/base-image/install/node/node.sh b/docker/base-image/install/node/node.sh index 8e894310..74437462 100644 --- a/docker/base-image/install/node/node.sh +++ b/docker/base-image/install/node/node.sh @@ -5,15 +5,16 @@ version="22" # Install nvm (Node Version Manager) curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.0/install.sh | bash -# Add nvm to path -echo 'export NVM_DIR="$HOME/.nvm"' >> ~/.bashrc -echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' >> ~/.bashrc -echo '[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion' >> ~/.bashrc +# Create a file in /etc/profile.d/ +cat > /etc/profile.d/node-env.sh << 'EOF' +export NVM_DIR="/root/.nvm" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" +export NODE_PATH=/usr/lib/node_modules +EOF -# Ensure changes take effect immediately -export NVM_DIR="$HOME/.nvm" -[[ -s "$NVM_DIR/nvm.sh" ]] && \. "$NVM_DIR/nvm.sh" # This loads nvm -[[ -s "$NVM_DIR/bash_completion" ]] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion +# Make the file executable +chmod +x /etc/profile.d/node-env.sh # Download and install Node.js (you may need to restart the terminal) nvm install ${version} diff --git a/docker/base-image/install/python/python.sh b/docker/base-image/install/python/python.sh index 445f5433..88e9c4f5 100644 --- a/docker/base-image/install/python/python.sh +++ b/docker/base-image/install/python/python.sh @@ -25,17 +25,19 @@ apt-get install -y \ # Install pyenv curl https://pyenv.run | bash -# Add pyenv to path -echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc -echo '[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc -echo 'eval "$(pyenv init -)"' >> ~/.bashrc -echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc - -# Ensure changes take effect immediately +# Create a file in /etc/profile.d/ +cat > /etc/profile.d/pyenv.sh << 'EOF' export PYENV_ROOT="$HOME/.pyenv" [[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" +EOF + +# Make the file executable +chmod +x /etc/profile.d/pyenv.sh + +# Source it immediately for the rest of the installation +source /etc/profile.d/pyenv.sh # Install python ${version} via pyenv pyenv install ${version} diff --git a/docker/ghcr/docker-compose.yml b/docker/ghcr/docker-compose.yml new file mode 100644 index 00000000..e33a868f --- /dev/null +++ b/docker/ghcr/docker-compose.yml @@ -0,0 +1,34 @@ +version: '3.3' +services: + master: + image: ghcr.io/crawlab-team/crawlab:${CRAWLAB_TAG:-develop} + container_name: crawlab_ghcr_master + environment: + CRAWLAB_NODE_MASTER: "Y" + CRAWLAB_MONGO_HOST: "mongo" + ports: + - "8080:8080" + depends_on: + - mongo + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health", "||", "exit", "1"] + interval: 30s + timeout: 10s + retries: 5 + + worker: + image: ghcr.io/crawlab-team/crawlab:${CRAWLAB_TAG:-develop} + container_name: crawlab_ghcr_worker + environment: + CRAWLAB_NODE_MASTER: "N" + CRAWLAB_MASTER_HOST: "master" + depends_on: + - master + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health", "||", "exit", "1"] + interval: 30s + timeout: 10s + retries: 5 + + mongo: + image: mongo:5