git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-26 17:49:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

ci: updated workflow

Browse Source
This commit is contained in:
Marvin Zhang
2024-11-22 20:44:26 +08:00
parent 6c221971cc
commit bfd2e214f2
10 changed files with 101 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
db/mongo/col.go
View File

@@ -3,7 +3,8 @@ package mongo
import (
"context"
"errors"
"github.com/crawlab-team/crawlab/trace"
"github.com/apex/log"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
@@ -53,18 +54,21 @@ type Col struct {
func (col *Col) Insert(doc interface{}) (id primitive.ObjectID, err error) {
res, err := col.c.InsertOne(col.ctx, doc)
if err != nil {
return primitive.NilObjectID, trace.TraceError(err)
log.Errorf("error inserting document: %v", err)
return primitive.NilObjectID, err
}
if id, ok := res.InsertedID.(primitive.ObjectID); ok {
return id, nil
}
return primitive.NilObjectID, trace.TraceError(errors.New("InsertedID is not ObjectID"))
err = errors.New("InsertedID is not ObjectID")
log.Errorf("error inserting document: %v", err)
return primitive.NilObjectID, err
}
func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err error) {
res, err := col.c.InsertMany(col.ctx, docs)
if err != nil {
return nil, trace.TraceError(err)
return nil, err
}
for _, v := range res.InsertedIDs {
switch v.(type) {
@@ -72,7 +76,9 @@ func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err er
id := v.(primitive.ObjectID)
ids = append(ids, id)
default:
return nil, trace.TraceError(errors.New("InsertedID is not ObjectID"))
err = errors.New("InsertedID is not ObjectID")
log.Errorf("error inserting document: %v", err)
return nil, err
}
}
return ids, nil
@@ -81,7 +87,8 @@ func (col *Col) InsertMany(docs []interface{}) (ids []primitive.ObjectID, err er
func (col *Col) UpdateId(id primitive.ObjectID, update interface{}) (err error) {
_, err = col.c.UpdateOne(col.ctx, bson.M{"_id": id}, update)
if err != nil {
return trace.TraceError(err)
log.Errorf("error updating document: %v", err)
return err
}
return nil
}
@@ -97,7 +104,8 @@ func (col *Col) UpdateWithOptions(query bson.M, update interface{}, opts *option
_, err = col.c.UpdateMany(col.ctx, query, update, opts)
}
if err != nil {
return trace.TraceError(err)
log.Errorf("error updating document: %v", err)
return err
}
return nil
}
@@ -117,7 +125,8 @@ func (col *Col) ReplaceWithOptions(query bson.M, doc interface{}, opts *options.
_, err = col.c.ReplaceOne(col.ctx, query, doc, opts)
}
if err != nil {
return trace.TraceError(err)
log.Errorf("error replacing document: %v", err)
return err
}
return nil
}
@@ -125,7 +134,8 @@ func (col *Col) ReplaceWithOptions(query bson.M, doc interface{}, opts *options.
func (col *Col) DeleteId(id primitive.ObjectID) (err error) {
_, err = col.c.DeleteOne(col.ctx, bson.M{"_id": id})
if err != nil {
return trace.TraceError(err)
log.Errorf("error deleting document: %v", err)
return err
}
return nil
}
@@ -141,7 +151,8 @@ func (col *Col) DeleteWithOptions(query bson.M, opts *options.DeleteOptions) (er
_, err = col.c.DeleteMany(col.ctx, query, opts)
}
if err != nil {
return trace.TraceError(err)
log.Errorf("error deleting document: %v", err)
return err
}
return nil
}
@@ -223,7 +234,8 @@ func (col *Col) Aggregate(pipeline mongo.Pipeline, opts *options.AggregateOption
func (col *Col) CreateIndex(indexModel mongo.IndexModel) (err error) {
_, err = col.c.Indexes().CreateOne(col.ctx, indexModel)
if err != nil {
return trace.TraceError(err)
log.Errorf("error creating index: %v", err)
return err
}
return nil
}
@@ -231,7 +243,8 @@ func (col *Col) CreateIndex(indexModel mongo.IndexModel) (err error) {
func (col *Col) CreateIndexes(indexModels []mongo.IndexModel) (err error) {
_, err = col.c.Indexes().CreateMany(col.ctx, indexModels)
if err != nil {
return trace.TraceError(err)
log.Errorf("error creating indexes: %v", err)
return err
}
return nil
}
@@ -247,7 +260,8 @@ func (col *Col) MustCreateIndexes(indexModels []mongo.IndexModel) {
func (col *Col) DeleteIndex(name string) (err error) {
_, err = col.c.Indexes().DropOne(col.ctx, name)
if err != nil {
return trace.TraceError(err)
log.Errorf("error deleting index: %v", err)
return err
}
return nil
}
@@ -255,7 +269,8 @@ func (col *Col) DeleteIndex(name string) (err error) {
func (col *Col) DeleteAllIndexes() (err error) {
_, err = col.c.Indexes().DropAll(col.ctx)
if err != nil {
return trace.TraceError(err)
log.Errorf("error deleting all indexes: %v", err)
return err
}
return nil
}
@@ -263,9 +278,11 @@ func (col *Col) DeleteAllIndexes() (err error) {
func (col *Col) ListIndexes() (indexes []map[string]interface{}, err error) {
cur, err := col.c.Indexes().List(col.ctx)
if err != nil {
log.Errorf("error listing indexes: %v", err)
return nil, err
}
if err := cur.All(col.ctx, &indexes); err != nil {
log.Errorf("error listing indexes: %v", err)
return nil, err
}
return indexes, nil

3
db/mongo/col_test.go
View File

@@ -59,7 +59,8 @@ func TestGetMongoColWithDb(t *testing.T) {
dbName := "test_db"
colName := "test_col"
col := GetMongoColWithDb(colName, dbName)
db := GetMongoDb(dbName)
col := GetMongoColWithDb(colName, db)
require.Equal(t, colName, col.c.Name())
require.Equal(t, dbName, col.db.Name())
}

35
db/mongo/db.go
View File

@@ -1,35 +1,24 @@
package mongo
import (
"github.com/crawlab-team/crawlab/trace"
"github.com/apex/log"
"github.com/spf13/viper"
"go.mongodb.org/mongo-driver/mongo"
)
func GetMongoDb(dbName string, opts ...DbOption) (db *mongo.Database) {
func GetMongoDb(dbName string) *mongo.Database {
// Use default database name if not provided
if dbName == "" {
dbName = viper.GetString("mongo.db")
}
if dbName == "" {
dbName = "test"
}
_opts := &DbOptions{}
for _, op := range opts {
op(_opts)
}
var c *mongo.Client
if _opts.client == nil {
var err error
c, err = GetMongoClient()
if err != nil {
trace.PrintError(err)
return nil
if dbName = viper.GetString("mongo.db"); dbName == "" {
dbName = "test"
}
} else {
c = _opts.client
}
return c.Database(dbName, nil)
c, err := GetMongoClient()
if err != nil {
log.Errorf("error getting mongo client: %v", err)
return nil
}
return c.Database(dbName)
}

15
db/mongo/db_options.go
View File

@@ -1,15 +0,0 @@
package mongo
import "go.mongodb.org/mongo-driver/mongo"
type DbOption func(options *DbOptions)
type DbOptions struct {
client *mongo.Client
}
func WithDbClient(c *mongo.Client) DbOption {
return func(options *DbOptions) {
options.client = c
}
}

17
db/mongo/db_test.go
View File

@@ -1,17 +0,0 @@
package mongo
import (
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
"testing"
)
func TestMongoGetDb(t *testing.T) {
dbName := "test_db"
viper.Set("mongo.db", dbName)
err := InitMongo()
require.Nil(t, err)
db := GetMongoDb("")
require.Equal(t, dbName, db.Name())
}