git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: removed unnecessary code

Browse Source
This commit is contained in:
Marvin Zhang
2024-07-11 12:45:29 +08:00
parent f2f298d8f8
commit aca0c0ebce
57 changed files with 423 additions and 430 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
core/utils/cockroachdb.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/upper/db/v4"
"github.com/upper/db/v4/adapter/mssql"
"time"
@@ -59,13 +60,13 @@ func getCockroachdbSession(ctx context.Context, ds *models.DataSource) (s db.Ses
return s, err
}
func GetCockroachdbSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
func GetCockroachdbSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getCockroachdbSessionV2(ctx, ds)
}
func getCockroachdbSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) {
func getCockroachdbSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) {
// normalize settings
host := ds.Host
port := ds.Port

5
core/utils/es.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/cenkalti/backoff/v4"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/trace"
"github.com/elastic/go-elasticsearch/v8"
@@ -83,13 +84,13 @@ func getElasticsearchClient(ctx context.Context, ds *models.DataSource) (c *elas
return c, err
}
func GetElasticsearchClientWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *elasticsearch.Client, err error) {
func GetElasticsearchClientWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *elasticsearch.Client, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getElasticsearchClientV2(ctx, ds)
}
func getElasticsearchClientV2(ctx context.Context, ds *models.DataSourceV2) (c *elasticsearch.Client, err error) {
func getElasticsearchClientV2(ctx context.Context, ds *models2.DataSourceV2) (c *elasticsearch.Client, err error) {
// normalize settings
host := ds.Host
port := ds.Port

5
core/utils/kafka.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/segmentio/kafka-go"
"time"
)
@@ -40,13 +41,13 @@ func getKafkaConnection(ctx context.Context, ds *models.DataSource) (c *kafka.Co
return kafka.DialLeader(ctx, network, address, topic, partition)
}
func GetKafkaConnectionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *kafka.Conn, err error) {
func GetKafkaConnectionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *kafka.Conn, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getKafkaConnectionV2(ctx, ds)
}
func getKafkaConnectionV2(ctx context.Context, ds *models.DataSourceV2) (c *kafka.Conn, err error) {
func getKafkaConnectionV2(ctx context.Context, ds *models2.DataSourceV2) (c *kafka.Conn, err error) {
// normalize settings
host := ds.Host
port := ds.Port

5
core/utils/mongo.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
@@ -54,7 +55,7 @@ func GetMongoClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c
return getMongoClient(ctx, ds)
}
func GetMongoClientWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *mongo2.Client, err error) {
func GetMongoClientWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *mongo2.Client, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getMongoClientV2(ctx, ds)
@@ -99,7 +100,7 @@ func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Clien
return mongo.GetMongoClient(opts...)
}
func getMongoClientV2(ctx context.Context, ds *models.DataSourceV2) (c *mongo2.Client, err error) {
func getMongoClientV2(ctx context.Context, ds *models2.DataSourceV2) (c *mongo2.Client, err error) {
// normalize settings
if ds.Host == "" {
ds.Host = constants.DefaultHost

5
core/utils/mssql.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/upper/db/v4"
"github.com/upper/db/v4/adapter/mssql"
"time"
@@ -59,13 +60,13 @@ func getMssqlSession(ctx context.Context, ds *models.DataSource) (s db.Session,
return s, err
}
func GetMssqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
func GetMssqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getMssqlSessionV2(ctx, ds)
}
func getMssqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) {
func getMssqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) {
// normalize settings
host := ds.Host
port := ds.Port

5
core/utils/mysql.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/upper/db/v4"
"github.com/upper/db/v4/adapter/mysql"
"time"
@@ -59,13 +60,13 @@ func getMysqlSession(ctx context.Context, ds *models.DataSource) (s db.Session,
return s, err
}
func GetMysqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
func GetMysqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getMysqlSessionV2(ctx, ds)
}
func getMysqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) {
func getMysqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) {
// normalize settings
host := ds.Host
port := ds.Port

5
core/utils/postgresql.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/upper/db/v4"
"github.com/upper/db/v4/adapter/postgresql"
"time"
@@ -59,13 +60,13 @@ func getPostgresqlSession(ctx context.Context, ds *models.DataSource) (s db.Sess
return s, err
}
func GetPostgresqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
func GetPostgresqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getPostgresqlSessionV2(ctx, ds)
}
func getPostgresqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) {
func getPostgresqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) {
// normalize settings
host := ds.Host
port := ds.Port

5
core/utils/sqlite.go
View File

@@ -3,6 +3,7 @@ package utils
import (
"context"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/upper/db/v4"
"github.com/upper/db/v4/adapter/sqlite"
"time"
@@ -44,13 +45,13 @@ func getSqliteSession(ctx context.Context, ds *models.DataSource) (s db.Session,
return s, err
}
func GetSqliteSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
func GetSqliteSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return getSqliteSessionV2(ctx, ds)
}
func getSqliteSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) {
func getSqliteSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) {
// connect settings
settings := sqlite.ConnectionURL{
Database: ds.Database,