feat: added modules

core/ds/cockroachdb.go

@@ -0,0 +1,70 @@
+package ds
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type CockroachdbService struct {
+	SqlService
+}
+
+func NewDataSourceCockroachdbService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
+	// service
+	svc := &CockroachdbService{}
+
+	// dependency injection
+	svc.modelSvc, err = service.GetService()
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// data source
+	if dsId.IsZero() {
+		svc.ds = &models.DataSource{}
+	} else {
+		svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
+		if err != nil {
+			return nil, trace.TraceError(err)
+		}
+	}
+
+	// data source defaults
+	if svc.ds.Host == "" {
+		svc.ds.Host = constants.DefaultHost
+	}
+	if svc.ds.Port == "" {
+		svc.ds.Port = constants.DefaultCockroachdbPort
+	}
+
+	// data source password
+	pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id)
+	if err == nil {
+		svc.ds.Password, err = utils.DecryptAES(pwd.Password)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// data collection
+	svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// session
+	svc.s, err = utils.GetCockroachdbSession(svc.ds)
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// collection
+	svc.col = svc.s.Collection(svc.dc.Name)
+
+	return svc, nil
+}

core/ds/default.go

@@ -0,0 +1 @@
+package ds

core/ds/es.go

@@ -0,0 +1,220 @@
+package ds
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/crawlab-team/crawlab-db/generic"
+	"github.com/crawlab-team/crawlab/core/constants"
+	constants2 "github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/entity"
+	entity2 "github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"github.com/elastic/go-elasticsearch/v8"
+	"github.com/elastic/go-elasticsearch/v8/esapi"
+	"github.com/google/uuid"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"strings"
+	"sync"
+	"time"
+)
+
+type ElasticsearchService struct {
+	// dependencies
+	modelSvc service.ModelService
+
+	// internals
+	dc *models.DataCollection // models.DataCollection
+	ds *models.DataSource     // models.DataSource
+	c  *elasticsearch.Client  // elasticsearch.Client
+	t  time.Time
+}
+
+func (svc *ElasticsearchService) Insert(records ...interface{}) (err error) {
+	// wait group
+	var wg sync.WaitGroup
+	wg.Add(len(records))
+
+	// iterate records
+	for _, r := range records {
+		// async operation
+		go func(r interface{}) {
+			switch r.(type) {
+			case entity.Result:
+				// convert type to entity.Result
+				d := r.(entity.Result)
+
+				// get document id
+				id := d.GetValue("id")
+				var docId string
+				switch id.(type) {
+				case string:
+					docId = id.(string)
+				}
+				if docId == "" {
+					docId = uuid.New().String() // generate new uuid if id is empty
+				}
+
+				// collection
+				d[constants2.DataCollectionKey] = svc.dc.Name
+
+				// index request
+				req := esapi.IndexRequest{
+					Index:      svc.getIndexName(),
+					DocumentID: docId,
+					Body:       strings.NewReader(d.String()),
+				}
+
+				// perform request
+				res, err := req.Do(context.Background(), svc.c)
+				if err != nil {
+					trace.PrintError(err)
+					wg.Done()
+					return
+				}
+				defer res.Body.Close()
+				if res.IsError() {
+					trace.PrintError(errors.New(fmt.Sprintf("[ElasticsearchService] [%s] error inserting record: %v", res.Status(), r)))
+				}
+
+				// release
+				wg.Done()
+			default:
+				wg.Done()
+				return
+			}
+		}(r)
+	}
+
+	// wait
+	wg.Wait()
+
+	return nil
+}
+
+func (svc *ElasticsearchService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) {
+	data, err := svc.getListResponse(query, opts, false)
+	if err != nil {
+		return nil, err
+	}
+	for _, hit := range data.Hits.Hits {
+		results = append(results, hit.Source)
+	}
+	return results, nil
+}
+
+func (svc *ElasticsearchService) Count(query generic.ListQuery) (n int, err error) {
+	data, err := svc.getListResponse(query, nil, true)
+	if err != nil {
+		return n, err
+	}
+	return int(data.Hits.Total.Value), nil
+}
+
+func (svc *ElasticsearchService) getListResponse(query generic.ListQuery, opts *generic.ListOptions, trackTotalHits bool) (data *entity2.ElasticsearchResponseData, err error) {
+	if opts == nil {
+		opts = &generic.ListOptions{}
+	}
+	query = append(query, generic.ListQueryCondition{
+		Key:   constants2.DataCollectionKey,
+		Op:    constants2.FilterOpEqual,
+		Value: svc.dc.Name,
+	})
+	res, err := svc.c.Search(
+		svc.c.Search.WithContext(context.Background()),
+		svc.c.Search.WithIndex(svc.getIndexName()),
+		svc.c.Search.WithBody(utils.GetElasticsearchQueryWithOptions(query, opts)),
+		svc.c.Search.WithTrackTotalHits(trackTotalHits),
+	)
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+	defer res.Body.Close()
+	if res.IsError() {
+		err = errors.New(fmt.Sprintf("[ElasticsearchService] [%s] error listing records: response=%s, query=%v opts=%v", res.Status(), res.String(), query, opts))
+		trace.PrintError(err)
+		return nil, err
+	}
+	data = &entity2.ElasticsearchResponseData{}
+	if err := json.NewDecoder(res.Body).Decode(data); err != nil {
+		return nil, trace.TraceError(err)
+	}
+	return data, nil
+}
+
+func (svc *ElasticsearchService) getIndexName() (index string) {
+	if svc.ds.Database == "" {
+		return svc.dc.Name
+	} else {
+		return svc.ds.Name
+	}
+}
+
+func NewDataSourceElasticsearchService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
+	// service
+	svc := &ElasticsearchService{}
+
+	// dependency injection
+	svc.modelSvc, err = service.GetService()
+	if err != nil {
+		return nil, err
+	}
+
+	// data source
+	if dsId.IsZero() {
+		svc.ds = &models.DataSource{}
+	} else {
+		svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// data source defaults
+	if svc.ds.Host == "" {
+		svc.ds.Host = constants.DefaultHost
+	}
+	if svc.ds.Port == "" {
+		svc.ds.Port = constants.DefaultElasticsearchPort
+	}
+
+	// data source password
+	pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id)
+	if err == nil {
+		svc.ds.Password, err = utils.DecryptAES(pwd.Password)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// data collection
+	svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
+	if err != nil {
+		return nil, err
+	}
+
+	// client
+	svc.c, err = utils.GetElasticsearchClient(svc.ds)
+	if err != nil {
+		return nil, err
+	}
+
+	return svc, nil
+}
+
+func (svc *ElasticsearchService) Index(fields []string) {
+	// TODO: implement me
+}
+
+func (svc *ElasticsearchService) SetTime(t time.Time) {
+	svc.t = t
+}
+
+func (svc *ElasticsearchService) GetTime() (t time.Time) {
+	return svc.t
+}

core/ds/kafka.go

@@ -0,0 +1,116 @@
+package ds
+
+import (
+	"github.com/cenkalti/backoff/v4"
+	"github.com/crawlab-team/crawlab-db/generic"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"github.com/segmentio/kafka-go"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type KafkaService struct {
+	// dependencies
+	modelSvc service.ModelService
+
+	// internals
+	dc *models.DataCollection // models.DataCollection
+	ds *models.DataSource     // models.DataSource
+	c  *kafka.Conn            // kafka.Conn
+	rb backoff.BackOff
+	t  time.Time
+}
+
+func (svc *KafkaService) Insert(records ...interface{}) (err error) {
+	var messages []kafka.Message
+	for _, r := range records {
+		switch r.(type) {
+		case entity.Result:
+			d := r.(entity.Result)
+			messages = append(messages, kafka.Message{
+				Topic: svc.ds.Database,
+				Key:   []byte(d.GetTaskId().Hex()),
+				Value: d.Bytes(),
+			})
+		}
+	}
+	_, err = svc.c.WriteMessages(messages...)
+	if err != nil {
+		return trace.TraceError(err)
+	}
+	return nil
+}
+
+func (svc *KafkaService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) {
+	// N/A
+	return nil, nil
+}
+
+func (svc *KafkaService) Count(query generic.ListQuery) (n int, err error) {
+	// N/A
+	return 0, nil
+}
+
+func NewDataSourceKafkaService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
+	// service
+	svc := &KafkaService{}
+
+	// dependency injection
+	svc.modelSvc, err = service.GetService()
+	if err != nil {
+		return nil, err
+	}
+
+	// data source
+	if dsId.IsZero() {
+		svc.ds = &models.DataSource{}
+	} else {
+		svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// data source defaults
+	if svc.ds.Host == "" {
+		svc.ds.Host = constants.DefaultHost
+	}
+	if svc.ds.Port == "" {
+		svc.ds.Port = constants.DefaultKafkaPort
+	}
+
+	// data source password
+	pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id)
+	if err == nil {
+		svc.ds.Password, err = utils.DecryptAES(pwd.Password)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// data collection
+	svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
+	if err != nil {
+		return nil, err
+	}
+
+	return svc, nil
+}
+
+func (svc *KafkaService) Index(fields []string) {
+	// TODO: implement me
+}
+
+func (svc *KafkaService) SetTime(t time.Time) {
+	svc.t = t
+}
+
+func (svc *KafkaService) GetTime() (t time.Time) {
+	return svc.t
+}

core/ds/mongo.go

@@ -0,0 +1,118 @@
+package ds
+
+import (
+	"github.com/crawlab-team/crawlab-db/generic"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	utils2 "github.com/crawlab-team/crawlab/core/utils"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"time"
+)
+
+type MongoService struct {
+	// dependencies
+	modelSvc service.ModelService
+
+	// internals
+	dc  *models.DataCollection // models.DataCollection
+	ds  *models.DataSource     // models.DataSource
+	c   *mongo2.Client
+	db  *mongo2.Database
+	col *mongo.Col
+	t   time.Time
+}
+
+func (svc *MongoService) Insert(records ...interface{}) (err error) {
+	_, err = svc.col.InsertMany(records)
+	return err
+}
+
+func (svc *MongoService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) {
+	var docs []models.Result
+	if err := svc.col.Find(utils.GetMongoQuery(query), utils.GetMongoOpts(opts)).All(&docs); err != nil {
+		return nil, err
+	}
+	for i := range docs {
+		results = append(results, &docs[i])
+	}
+	return results, nil
+}
+
+func (svc *MongoService) Count(query generic.ListQuery) (n int, err error) {
+	return svc.col.Count(utils.GetMongoQuery(query))
+}
+
+func NewDataSourceMongoService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
+	// service
+	svc := &MongoService{}
+
+	// dependency injection
+	svc.modelSvc, err = service.GetService()
+	if err != nil {
+		return nil, err
+	}
+
+	// data source
+	if dsId.IsZero() {
+		svc.ds = &models.DataSource{}
+	} else {
+		svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// data source defaults
+	if svc.ds.Host == "" {
+		svc.ds.Host = constants.DefaultHost
+	}
+	if svc.ds.Port == "" {
+		svc.ds.Port = constants.DefaultMongoPort
+	}
+
+	// data source password
+	pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id)
+	if err == nil {
+		svc.ds.Password, err = utils.DecryptAES(pwd.Password)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// data collection
+	svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
+	if err != nil {
+		return nil, err
+	}
+
+	// mongo client
+	svc.c, err = utils2.GetMongoClient(svc.ds)
+	if err != nil {
+		return nil, err
+	}
+
+	// mongo database
+	svc.db = mongo.GetMongoDb(svc.ds.Database, mongo.WithDbClient(svc.c))
+
+	// mongo col
+	svc.col = mongo.GetMongoColWithDb(svc.dc.Name, svc.db)
+
+	return svc, nil
+}
+
+func (svc *MongoService) Index(fields []string) {
+	// TODO: implement me
+}
+
+func (svc *MongoService) SetTime(t time.Time) {
+	svc.t = t
+}
+
+func (svc *MongoService) GetTime() (t time.Time) {
+	return svc.t
+}

core/ds/mssql.go

@@ -0,0 +1,71 @@
+package ds
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	utils2 "github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type MssqlService struct {
+	SqlService
+}
+
+func NewDataSourceMssqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
+	// service
+	svc := &MssqlService{}
+
+	// dependency injection
+	svc.modelSvc, err = service.GetService()
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// data source
+	if dsId.IsZero() {
+		svc.ds = &models.DataSource{}
+	} else {
+		svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
+		if err != nil {
+			return nil, trace.TraceError(err)
+		}
+	}
+
+	// data source defaults
+	if svc.ds.Host == "" {
+		svc.ds.Host = constants.DefaultHost
+	}
+	if svc.ds.Port == "" {
+		svc.ds.Port = constants.DefaultMssqlPort
+	}
+
+	// data source password
+	pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id)
+	if err == nil {
+		svc.ds.Password, err = utils.DecryptAES(pwd.Password)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// data collection
+	svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// session
+	svc.s, err = utils2.GetMssqlSession(svc.ds)
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// collection
+	svc.col = svc.s.Collection(svc.dc.Name)
+
+	return svc, nil
+}

core/ds/mssql_test.go

@@ -0,0 +1,8 @@
+package ds
+
+import "testing"
+
+func TestNewDataSourceMssqlService(t *testing.T) {
+	t.Run("insert", func(t *testing.T) {
+	})
+}

core/ds/mysql.go

@@ -0,0 +1,69 @@
+package ds
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	utils2 "github.com/crawlab-team/crawlab/core/utils"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type MysqlService struct {
+	SqlService
+}
+
+func NewDataSourceMysqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
+	// service
+	svc := &MysqlService{}
+
+	// dependency injection
+	svc.modelSvc, err = service.GetService()
+	if err != nil {
+		return nil, err
+	}
+
+	// data source
+	if dsId.IsZero() {
+		svc.ds = &models.DataSource{}
+	} else {
+		svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// data source defaults
+	if svc.ds.Host == "" {
+		svc.ds.Host = constants.DefaultHost
+	}
+	if svc.ds.Port == "" {
+		svc.ds.Port = constants.DefaultMysqlPort
+	}
+
+	// data source password
+	pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id)
+	if err == nil {
+		svc.ds.Password, err = utils.DecryptAES(pwd.Password)
+		if err != nil {
+			return nil, err
+		}
+	}
+	// data collection
+	svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
+	if err != nil {
+		return nil, err
+	}
+
+	// session
+	svc.s, err = utils2.GetMysqlSession(svc.ds)
+	if err != nil {
+		return nil, err
+	}
+
+	// collection
+	svc.col = svc.s.Collection(svc.dc.Name)
+
+	return svc, nil
+}

core/ds/options.go

@@ -0,0 +1,14 @@
+package ds
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"time"
+)
+
+type DataSourceServiceOption func(svc interfaces.DataSourceService)
+
+func WithMonitorInterval(duration time.Duration) DataSourceServiceOption {
+	return func(svc interfaces.DataSourceService) {
+		svc.SetMonitorInterval(duration)
+	}
+}

core/ds/postgresql.go

@@ -0,0 +1,71 @@
+package ds
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	utils2 "github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type PostgresqlService struct {
+	SqlService
+}
+
+func NewDataSourcePostgresqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
+	// service
+	svc := &PostgresqlService{}
+
+	// dependency injection
+	svc.modelSvc, err = service.GetService()
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// data source
+	if dsId.IsZero() {
+		svc.ds = &models.DataSource{}
+	} else {
+		svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
+		if err != nil {
+			return nil, trace.TraceError(err)
+		}
+	}
+
+	// data source defaults
+	if svc.ds.Host == "" {
+		svc.ds.Host = constants.DefaultHost
+	}
+	if svc.ds.Port == "" {
+		svc.ds.Port = constants.DefaultPostgresqlPort
+	}
+
+	// data source password
+	pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id)
+	if err == nil {
+		svc.ds.Password, err = utils.DecryptAES(pwd.Password)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// data collection
+	svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// session
+	svc.s, err = utils2.GetPostgresqlSession(svc.ds)
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// collection
+	svc.col = svc.s.Collection(svc.dc.Name)
+
+	return svc, nil
+}

core/ds/service.go

@@ -0,0 +1,305 @@
+package ds
+
+import (
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/constants"
+	constants2 "github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/result"
+	"github.com/crawlab-team/crawlab/core/utils"
+	utils2 "github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"sync"
+	"time"
+)
+
+type Service struct {
+	// dependencies
+	modelSvc service.ModelService
+
+	// internals
+	timeout         time.Duration
+	monitorInterval time.Duration
+	stopped         bool
+}
+
+func (svc *Service) Init() {
+	// result service registry
+	reg := result.GetResultServiceRegistry()
+
+	// register result services
+	reg.Register(constants.DataSourceTypeMongo, NewDataSourceMongoService)
+	reg.Register(constants.DataSourceTypeMysql, NewDataSourceMysqlService)
+	reg.Register(constants.DataSourceTypePostgresql, NewDataSourcePostgresqlService)
+	reg.Register(constants.DataSourceTypeMssql, NewDataSourceMssqlService)
+	reg.Register(constants.DataSourceTypeSqlite, NewDataSourceSqliteService)
+	reg.Register(constants.DataSourceTypeCockroachdb, NewDataSourceCockroachdbService)
+	reg.Register(constants.DataSourceTypeElasticSearch, NewDataSourceElasticsearchService)
+	reg.Register(constants.DataSourceTypeKafka, NewDataSourceKafkaService)
+}
+
+func (svc *Service) Start() {
+	// start monitoring
+	go svc.Monitor()
+}
+
+func (svc *Service) Wait() {
+	utils.DefaultWait()
+}
+
+func (svc *Service) Stop() {
+	svc.stopped = true
+}
+
+func (svc *Service) ChangePassword(id primitive.ObjectID, password string) (err error) {
+	p, err := svc.modelSvc.GetPasswordById(id)
+	if err == nil {
+		// exists, save
+		encryptedPassword, err := utils.EncryptAES(password)
+		if err != nil {
+			return err
+		}
+		p.Password = encryptedPassword
+		if err := delegate.NewModelDelegate(p).Save(); err != nil {
+			return err
+		}
+		return nil
+	} else if err.Error() == mongo.ErrNoDocuments.Error() {
+		// not exists, add
+		encryptedPassword, err := utils.EncryptAES(password)
+		if err != nil {
+			return err
+		}
+		p = &models.Password{
+			Id:       id,
+			Password: encryptedPassword,
+		}
+		if err := delegate.NewModelDelegate(p).Add(); err != nil {
+			return err
+		}
+		return nil
+	} else {
+		// error
+		return err
+	}
+}
+
+func (svc *Service) Monitor() {
+	for {
+		// return if stopped
+		if svc.stopped {
+			return
+		}
+
+		// monitor
+		if err := svc.monitor(); err != nil {
+			trace.PrintError(err)
+		}
+
+		// wait
+		time.Sleep(svc.monitorInterval)
+	}
+}
+
+func (svc *Service) CheckStatus(id primitive.ObjectID) (err error) {
+	ds, err := svc.modelSvc.GetDataSourceById(id)
+	if err != nil {
+		return err
+	}
+	return svc.checkStatus(ds, true)
+}
+
+func (svc *Service) SetTimeout(duration time.Duration) {
+	svc.timeout = duration
+}
+
+func (svc *Service) SetMonitorInterval(duration time.Duration) {
+	svc.monitorInterval = duration
+}
+
+func (svc *Service) monitor() (err error) {
+	// start
+	tic := time.Now()
+	log.Debugf("[DataSourceService] start monitoring")
+
+	// data source list
+	dsList, err := svc.modelSvc.GetDataSourceList(nil, nil)
+	if err != nil {
+		return err
+	}
+
+	// waiting group
+	wg := sync.WaitGroup{}
+	wg.Add(len(dsList))
+
+	// iterate data source list
+	for _, ds := range dsList {
+		// async operation
+		go func(ds models.DataSource) {
+			// check status and save
+			_ = svc.checkStatus(&ds, true)
+
+			// release
+			wg.Done()
+		}(ds)
+	}
+
+	// wait
+	wg.Wait()
+
+	// finish
+	toc := time.Now()
+	log.Debugf("[DataSourceService] finished monitoring. elapsed: %d ms", (toc.Sub(tic)).Milliseconds())
+
+	return nil
+}
+
+func (svc *Service) checkStatus(ds *models.DataSource, save bool) (err error) {
+	// password
+	if ds.Password == "" {
+		pwd, err := svc.modelSvc.GetPasswordById(ds.Id)
+		if err == nil {
+			ds.Password, err = utils.DecryptAES(pwd.Password)
+			if err != nil {
+				return err
+			}
+		} else if err.Error() != mongo.ErrNoDocuments.Error() {
+			return trace.TraceError(err)
+		}
+	}
+
+	// check status
+	if err := svc._checkStatus(ds); err != nil {
+		ds.Status = constants2.DataSourceStatusOffline
+		ds.Error = err.Error()
+	} else {
+		ds.Status = constants2.DataSourceStatusOnline
+		ds.Error = ""
+	}
+
+	// save
+	if save {
+		return svc._save(ds)
+	}
+
+	return nil
+}
+
+func (svc *Service) _save(ds *models.DataSource) (err error) {
+	log.Debugf("[DataSourceService] saving data source: name=%s, type=%s, status=%s, error=%s", ds.Name, ds.Type, ds.Status, ds.Error)
+	return delegate.NewModelDelegate(ds).Save()
+}
+
+func (svc *Service) _checkStatus(ds *models.DataSource) (err error) {
+	switch ds.Type {
+	case constants.DataSourceTypeMongo:
+		_, err := utils2.GetMongoClientWithTimeout(ds, svc.timeout)
+		if err != nil {
+			return err
+		}
+	case constants.DataSourceTypeMysql:
+		s, err := utils2.GetMysqlSessionWithTimeout(ds, svc.timeout)
+		if err != nil {
+			return err
+		}
+		if s != nil {
+			s.Close()
+		}
+	case constants.DataSourceTypePostgresql:
+		s, err := utils2.GetPostgresqlSessionWithTimeout(ds, svc.timeout)
+		if err != nil {
+			return err
+		}
+		if s != nil {
+			s.Close()
+		}
+	case constants.DataSourceTypeMssql:
+		s, err := utils2.GetMssqlSessionWithTimeout(ds, svc.timeout)
+		if err != nil {
+			return err
+		}
+		if s != nil {
+			s.Close()
+		}
+	case constants.DataSourceTypeSqlite:
+		s, err := utils2.GetSqliteSessionWithTimeout(ds, svc.timeout)
+		if err != nil {
+			return err
+		}
+		if s != nil {
+			s.Close()
+		}
+	case constants.DataSourceTypeCockroachdb:
+		s, err := utils2.GetCockroachdbSessionWithTimeout(ds, svc.timeout)
+		if err != nil {
+			return err
+		}
+		if s != nil {
+			s.Close()
+		}
+	case constants.DataSourceTypeElasticSearch:
+		_, err := utils2.GetElasticsearchClientWithTimeout(ds, svc.timeout)
+		if err != nil {
+			return err
+		}
+	case constants.DataSourceTypeKafka:
+		c, err := utils2.GetKafkaConnectionWithTimeout(ds, svc.timeout)
+		if err != nil {
+			return err
+		}
+		if c != nil {
+			c.Close()
+		}
+	default:
+		log.Warnf("[DataSourceService] invalid data source type: %s", ds.Type)
+	}
+	return nil
+}
+
+func NewDataSourceService(opts ...DataSourceServiceOption) (svc2 interfaces.DataSourceService, err error) {
+	// service
+	svc := &Service{
+		monitorInterval: 15 * time.Second,
+		timeout:         10 * time.Second,
+	}
+
+	// apply options
+	for _, opt := range opts {
+		opt(svc)
+	}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(modelSvc service.ModelService) {
+		svc.modelSvc = modelSvc
+	}); err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// initialize
+	svc.Init()
+
+	// start
+	svc.Start()
+
+	return svc, nil
+}
+
+var _dsSvc interfaces.DataSourceService
+
+func GetDataSourceService() (svc interfaces.DataSourceService, err error) {
+	if _dsSvc != nil {
+		return _dsSvc, nil
+	}
+	svc, err = NewDataSourceService()
+	if err != nil {
+		return nil, err
+	}
+	_dsSvc = svc
+	return svc, nil
+}

core/ds/sql.go

@@ -0,0 +1,76 @@
+package ds
+
+import (
+	"github.com/crawlab-team/crawlab-db/generic"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	utils2 "github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"github.com/upper/db/v4"
+	"time"
+)
+
+type SqlService struct {
+	// dependencies
+	modelSvc service.ModelService
+
+	// internals
+	ds  *models.DataSource
+	dc  *models.DataCollection
+	s   db.Session
+	col db.Collection
+	t   time.Time
+}
+
+func (svc *SqlService) Insert(records ...interface{}) (err error) {
+	for _, d := range records {
+		var r entity.Result
+		switch d.(type) {
+		case entity.Result:
+			r = d.(entity.Result)
+		default:
+			continue
+		}
+		_r := r.Flatten()
+		if _, err = svc.col.Insert(_r); err != nil {
+			trace.PrintError(err)
+			continue
+		}
+	}
+	return nil
+}
+
+func (svc *SqlService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) {
+	var docs []entity.Result
+	if err := svc.col.Find(utils2.GetSqlQuery(query)).
+		Offset(opts.Skip).
+		Limit(opts.Limit).All(&docs); err != nil {
+		return nil, trace.TraceError(err)
+	}
+	for i := range docs {
+		d := docs[i].ToJSON()
+		results = append(results, &d)
+	}
+	return results, nil
+}
+
+func (svc *SqlService) Count(query generic.ListQuery) (n int, err error) {
+	nInt64, err := svc.col.Find(utils2.GetSqlQuery(query)).Count()
+	if err != nil {
+		return n, err
+	}
+	return int(nInt64), nil
+}
+
+func (svc *SqlService) Index(fields []string) {
+	// TODO: implement me
+}
+
+func (svc *SqlService) SetTime(t time.Time) {
+	svc.t = t
+}
+
+func (svc *SqlService) GetTime() (t time.Time) {
+	return svc.t
+}

core/ds/sql_options.go

@@ -0,0 +1,6 @@
+package ds
+
+type SqlOptions struct {
+	DefaultHost string
+	DefaultPort string
+}

core/ds/sqlite.go

@@ -0,0 +1,52 @@
+package ds
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	utils2 "github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type SqliteService struct {
+	SqlService
+}
+
+func NewDataSourceSqliteService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
+	// service
+	svc := &SqliteService{}
+
+	// dependency injection
+	svc.modelSvc, err = service.GetService()
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// data source
+	if dsId.IsZero() {
+		svc.ds = &models.DataSource{}
+	} else {
+		svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
+		if err != nil {
+			return nil, trace.TraceError(err)
+		}
+	}
+
+	// data collection
+	svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// session
+	svc.s, err = utils2.GetSqliteSession(svc.ds)
+	if err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// collection
+	svc.col = svc.s.Collection(svc.dc.Name)
+
+	return svc, nil
+}