git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

ci: updated workflow

Browse Source
This commit is contained in:
Marvin Zhang
2024-11-22 20:44:26 +08:00
parent 1984831056
commit cd2abe9547
10 changed files with 101 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
.github/workflows/docker-crawlab.yml vendored
View File

@@ -26,6 +26,7 @@ env:
jobs:
setup:
name: Setup
runs-on: ubuntu-latest
outputs:
backend_changed: ${{ steps.check_changed_files.outputs.backend_changed }}
@@ -112,6 +113,7 @@ jobs:
fi
build_base_image:
name: Build base image
needs: [ setup ]
if: needs.setup.outputs.base_image_changed == 'true' || needs.setup.outputs.workflow_changed == 'true'
runs-on: ubuntu-latest
@@ -149,15 +151,18 @@ jobs:
image: mongo:5
ports:
- 27017:27017
strategy:
matrix:
package: [core, db]
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: 'core/go.mod'
cache-dependency-path: 'core/go.sum'
go-version-file: '${{ matrix.package }}/go.mod'
cache-dependency-path: '${{ matrix.package }}/go.sum'
- name: Run tests
working-directory: core
working-directory: ${{ matrix.package }}
run: |
# Find all directories containing *_test.go files
test_dirs=$(find . -name "*_test.go" -exec dirname {} \; | sort -u)
@@ -275,6 +280,7 @@ jobs:
${{ env.IMAGE_NAME_CRAWLAB }}:${{ needs.setup.outputs.version }}
test_crawlab:
name: Test crawlab
needs: [setup, build_crawlab]
if: ${{ always() && needs.build_crawlab.result == 'success' }}
runs-on: ubuntu-latest
@@ -332,6 +338,7 @@ jobs:
destination_dir: playwright-report/${{ needs.setup.outputs.version }}
push_images:
name: Push images
if: ${{ always() && needs.test_crawlab.result == 'success' }}
needs: [setup, test_crawlab]
runs-on: ubuntu-latest

45
db/mongo/col.go
View File

@@ -3,7 +3,8 @@ package mongo
import (
"context"
"errors"
"github.com/crawlab-team/crawlab/trace"
"github.com/apex/log"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
@@ -53,18 +54,21 @@ type Col struct {
func (col *Col) Insert(doc interface{}) (id primitive.ObjectID, err error) {
res, err := col.c.InsertOne(col.ctx, doc)
if err != nil {
return primitive.NilObjectID, trace.TraceError(err)
log.Errorf("error inserting document: %v", err)
return primitive.NilObjectID, err
}
if id, ok := res.InsertedID.(primitive.ObjectID); ok {
return id, nil
}
return primitive.NilObjectID, trace.TraceError(errors.New("InsertedID is not ObjectID"))
err = errors.New("InsertedID is not ObjectID")
log.Errorf("error inserting document: %v", err)
return primitive.NilObjectID, err
}
func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err error) {
res, err := col.c.InsertMany(col.ctx, docs)
if err != nil {
return nil, trace.TraceError(err)
return nil, err
}
for _, v := range res.InsertedIDs {
switch v.(type) {
@@ -72,7 +76,9 @@ func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err er
id := v.(primitive.ObjectID)
ids = append(ids, id)
default:
return nil, trace.TraceError(errors.New("InsertedID is not ObjectID"))
err = errors.New("InsertedID is not ObjectID")
log.Errorf("error inserting document: %v", err)
return nil, err
}
}
return ids, nil
@@ -81,7 +87,8 @@ func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err er
func (col *Col) UpdateId(id primitive.ObjectID, update interface{}) (err error) {
_, err = col.c.UpdateOne(col.ctx, bson.M{"_id": id}, update)
if err != nil {
return trace.TraceError(err)
log.Errorf("error updating document: %v", err)
return err
}
return nil
}
@@ -97,7 +104,8 @@ func (col *Col) UpdateWithOptions(query bson.M, update interface{}, opts *option
_, err = col.c.UpdateMany(col.ctx, query, update, opts)
}
if err != nil {
return trace.TraceError(err)
log.Errorf("error updating document: %v", err)
return err
}
return nil
}
@@ -117,7 +125,8 @@ func (col *Col) ReplaceWithOptions(query bson.M, doc interface{}, opts *options.
_, err = col.c.ReplaceOne(col.ctx, query, doc, opts)
}
if err != nil {
return trace.TraceError(err)
log.Errorf("error replacing document: %v", err)
return err
}
return nil
}
@@ -125,7 +134,8 @@ func (col *Col) ReplaceWithOptions(query bson.M, doc interface{}, opts *options.
func (col *Col) DeleteId(id primitive.ObjectID) (err error) {
_, err = col.c.DeleteOne(col.ctx, bson.M{"_id": id})
if err != nil {
return trace.TraceError(err)
log.Errorf("error deleting document: %v", err)
return err
}
return nil
}
@@ -141,7 +151,8 @@ func (col *Col) DeleteWithOptions(query bson.M, opts *options.DeleteOptions) (er
_, err = col.c.DeleteMany(col.ctx, query, opts)
}
if err != nil {
return trace.TraceError(err)
log.Errorf("error deleting document: %v", err)
return err
}
return nil
}
@@ -223,7 +234,8 @@ func (col *Col) Aggregate(pipeline mongo.Pipeline, opts *options.AggregateOption
func (col *Col) CreateIndex(indexModel mongo.IndexModel) (err error) {
_, err = col.c.Indexes().CreateOne(col.ctx, indexModel)
if err != nil {
return trace.TraceError(err)
log.Errorf("error creating index: %v", err)
return err
}
return nil
}
@@ -231,7 +243,8 @@ func (col *Col) CreateIndex(indexModel mongo.IndexModel) (err error) {
func (col *Col) CreateIndexes(indexModels []mongo.IndexModel) (err error) {
_, err = col.c.Indexes().CreateMany(col.ctx, indexModels)
if err != nil {
return trace.TraceError(err)
log.Errorf("error creating indexes: %v", err)
return err
}
return nil
}
@@ -247,7 +260,8 @@ func (col *Col) MustCreateIndexes(indexModels []mongo.IndexModel) {
func (col *Col) DeleteIndex(name string) (err error) {
_, err = col.c.Indexes().DropOne(col.ctx, name)
if err != nil {
return trace.TraceError(err)
log.Errorf("error deleting index: %v", err)
return err
}
return nil
}
@@ -255,7 +269,8 @@ func (col *Col) DeleteIndex(name string) (err error) {
func (col *Col) DeleteAllIndexes() (err error) {
_, err = col.c.Indexes().DropAll(col.ctx)
if err != nil {
return trace.TraceError(err)
log.Errorf("error deleting all indexes: %v", err)
return err
}
return nil
}
@@ -263,9 +278,11 @@ func (col *Col) DeleteAllIndexes() (err error) {
func (col *Col) ListIndexes() (indexes []map[string]interface{}, err error) {
cur, err := col.c.Indexes().List(col.ctx)
if err != nil {
log.Errorf("error listing indexes: %v", err)
return nil, err
}
if err := cur.All(col.ctx, &indexes); err != nil {
log.Errorf("error listing indexes: %v", err)
return nil, err
}
return indexes, nil

3
db/mongo/col_test.go
View File

@@ -59,7 +59,8 @@ func TestGetMongoColWithDb(t *testing.T) {
dbName := "test_db"
colName := "test_col"
col := GetMongoColWithDb(colName, dbName)
db := GetMongoDb(dbName)
col := GetMongoColWithDb(colName, db)
require.Equal(t, colName, col.c.Name())
require.Equal(t, dbName, col.db.Name())
}

35
db/mongo/db.go
View File

@@ -1,35 +1,24 @@
package mongo
import (
"github.com/crawlab-team/crawlab/trace"
"github.com/apex/log"
"github.com/spf13/viper"
"go.mongodb.org/mongo-driver/mongo"
)
func GetMongoDb(dbName string, opts ...DbOption) (db *mongo.Database) {
func GetMongoDb(dbName string) *mongo.Database {
// Use default database name if not provided
if dbName == "" {
dbName = viper.GetString("mongo.db")
}
if dbName == "" {
dbName = "test"
}
_opts := &DbOptions{}
for _, op := range opts {
op(_opts)
}
var c *mongo.Client
if _opts.client == nil {
var err error
c, err = GetMongoClient()
if err != nil {
trace.PrintError(err)
return nil
if dbName = viper.GetString("mongo.db"); dbName == "" {
dbName = "test"
}
} else {
c = _opts.client
}
return c.Database(dbName, nil)
c, err := GetMongoClient()
if err != nil {
log.Errorf("error getting mongo client: %v", err)
return nil
}
return c.Database(dbName)
}

15
db/mongo/db_options.go
View File

@@ -1,15 +0,0 @@
package mongo
import "go.mongodb.org/mongo-driver/mongo"
type DbOption func(options *DbOptions)
type DbOptions struct {
client *mongo.Client
}
func WithDbClient(c *mongo.Client) DbOption {
return func(options *DbOptions) {
options.client = c
}
}

17
db/mongo/db_test.go
View File

@@ -1,17 +0,0 @@
package mongo
import (
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
"testing"
)
func TestMongoGetDb(t *testing.T) {
dbName := "test_db"
viper.Set("mongo.db", dbName)
err := InitMongo()
require.Nil(t, err)
db := GetMongoDb("")
require.Equal(t, dbName, db.Name())
}

4
docker/base-image/install/go/go.sh
View File

@@ -19,6 +19,10 @@ eval "$(goenv init -)"
goenv install ${version}
goenv global ${version}
# Create symbolic links
ln -sf "$(goenv which go)" /usr/local/bin/go
ln -sf "$(goenv which gofmt)" /usr/local/bin/gofmt
# verify
go_version=$(go version)
if [[ $go_version =~ "go${version}" ]]; then

39
docker/base-image/install/java/java.sh
View File

@@ -1,10 +1,35 @@
#!/bin/bash
# Exit on error
set -e
version="11.0.12-open"
# Update package list and install OpenJDK 11
DEBIAN_FRONTEND=noninteractive apt-get update && \
apt-get install -y --no-install-recommends openjdk-11-jdk && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Install SDKMAN!
curl -s "https://get.sdkman.io" | bash
# Source SDKMAN!
source "$HOME/.sdkman/bin/sdkman-init.sh"
# Install Java 11 (you can specify vendor, e.g., 11.0.12-open for OpenJDK)
sdk install java ${version}
# Set Java 11 as default
sdk default java ${version}
# Create symbolic links
ln -sf "$(sdkman which java)" /usr/local/bin/java
ln -sf "$(sdkman which javac)" /usr/local/bin/javac
# Verify
java_version=$(java -version)
if [[ $java_version =~ "${version}" ]]; then
:
else
echo "ERROR: java version does not match. expect \"${version}\", but actual is \"${java_version}\""
exit 1
fi
javac_version=$(javac -version)
if [[ $javac_version =~ "${version}" ]]; then
:
else
echo "ERROR: javac version does not match. expect \"${version}\", but actual is \"${javac_version}\""
exit 1
fi

6
docker/base-image/install/node/node.sh
View File

@@ -22,6 +22,12 @@ nvm install ${version}
nvm use ${version}
nvm alias default ${version}
# Create symbolic links
ln -sf "$(nvm which node)" /usr/local/bin/node
ln -sf "$(nvm which npm)" /usr/local/bin/npm
ln -sf "$(nvm which yarn)" /usr/local/bin/yarn
ln -sf "$(nvm which pnpm)" /usr/local/bin/pnpm
# verifies the right Node.js version is in the environment
if [[ ! "$(node -v)" =~ ^v${version} ]]; then
echo "Node.js version is not v${version}.x"

4
docker/base-image/install/python/python.sh
View File

@@ -41,6 +41,10 @@ eval "$(pyenv virtualenv-init -)"
pyenv install ${version}
pyenv global ${version}
# Create symbolic links
ln -sf $(pyenv which python) /usr/local/bin/python
ln -sf $(pyenv which pip) /usr/local/bin/pip
# verify
python_version=$(python -V)
if [[ $python_version =~ "Python ${version}" ]]; then