diff --git a/core/apps/docker.go b/core/apps/docker.go index 33ad5703..50e391b2 100644 --- a/core/apps/docker.go +++ b/core/apps/docker.go @@ -167,16 +167,6 @@ func (app *Docker) startSeaweedFs() { } } -func (app *Docker) importDemo() { - for { - if app.Ready() { - break - } - time.Sleep(1 * time.Second) - } - _ = utils.ImportDemo() -} - func NewDocker(svr ServerApp) *Docker { dck := &Docker{ parent: svr, diff --git a/core/cmd/server_test.go b/core/cmd/server_test.go deleted file mode 100644 index 4d7184c5..00000000 --- a/core/cmd/server_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package cmd - -import ( - "github.com/crawlab-team/crawlab/core/apps" - "os" - "testing" -) - -func TestCmdServer(t *testing.T) { - _ = os.Setenv("CRAWLAB_PPROF", "true") - - // app - svr := apps.GetServerV2() - - // start - apps.Start(svr) -} diff --git a/core/color/service.go b/core/color/service.go deleted file mode 100644 index cf32a535..00000000 --- a/core/color/service.go +++ /dev/null @@ -1,82 +0,0 @@ -package color - -import ( - "encoding/hex" - "encoding/json" - "github.com/crawlab-team/crawlab/core/data" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/trace" - "math/rand" - "strconv" - "strings" -) - -func NewService() (svc interfaces.ColorService, err error) { - var cl []*entity.Color - cm := map[string]*entity.Color{} - - if err := json.Unmarshal([]byte(data.ColorsDataText), &cl); err != nil { - return nil, trace.TraceError(err) - } - - for _, c := range cl { - cm[c.Name] = c - } - - return &Service{ - cl: cl, - cm: cm, - }, nil -} - -type Service struct { - cl []*entity.Color - cm map[string]*entity.Color -} - -func (svc *Service) Inject() (err error) { - return nil -} - -func (svc *Service) GetByName(name string) (res interfaces.Color, err error) { - res, ok := svc.cm[name] - if !ok { - return res, errors.ErrorModelNotFound - } - return res, err -} - -func (svc *Service) GetRandom() (res interfaces.Color, err error) { - if len(svc.cl) == 0 { - hexStr, err := svc.getRandomColorHex() - if err != nil { - return res, err - } - return &entity.Color{Hex: hexStr}, nil - } - - idx := rand.Intn(len(svc.cl)) - return svc.cl[idx], nil -} - -func (svc *Service) getRandomColorHex() (res string, err error) { - n := 6 - arr := make([]string, n) - for i := 0; i < n; i++ { - arr[i], err = svc.getRandomHexChar() - if err != nil { - return res, err - } - } - return strings.Join(arr, ""), nil -} - -func (svc *Service) getRandomHexChar() (res string, err error) { - n := rand.Intn(16) - b := []byte(strconv.Itoa(n)) - h := make([]byte, 1) - hex.Encode(h, b) - return string(h), nil -} diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index 55cb78d6..a295df50 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -220,11 +220,6 @@ func InitRoutes(app *gin.Engine) (err error) { Path: "/:id/logs", HandlerFunc: GetTaskLogs, }, - { - Method: http.MethodGet, - Path: "/:id/data", - HandlerFunc: GetTaskData, - }, }...)) RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models2.TokenV2]([]Action{ { diff --git a/core/controllers/task_v2.go b/core/controllers/task_v2.go index afb5b17e..2b3e16a5 100644 --- a/core/controllers/task_v2.go +++ b/core/controllers/task_v2.go @@ -7,12 +7,10 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/result" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/task/log" "github.com/crawlab-team/crawlab/core/task/scheduler" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" "github.com/crawlab-team/crawlab/db/mongo" "github.com/gin-gonic/gin" "github.com/spf13/viper" @@ -441,62 +439,3 @@ func GetTaskLogs(c *gin.Context) { HandleSuccessWithListData(c, logs, total) } - -func GetTaskData(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // pagination - p, err := GetPagination(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // task - t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // result service - resultSvc, err := result.GetResultService(t.SpiderId) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // query - query := generic.ListQuery{ - generic.ListQueryCondition{ - Key: constants.TaskKey, - Op: generic.OpEqual, - Value: t.Id, - }, - } - - // list - data, err := resultSvc.List(query, &generic.ListOptions{ - Skip: (p.Page - 1) * p.Size, - Limit: p.Size, - Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}}, - }) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // total - total, err := resultSvc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithListData(c, data, total) -} diff --git a/core/ds/cockroachdb.go b/core/ds/cockroachdb.go deleted file mode 100644 index 4af04720..00000000 --- a/core/ds/cockroachdb.go +++ /dev/null @@ -1,70 +0,0 @@ -package ds - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type CockroachdbService struct { - SqlService -} - -func NewDataSourceCockroachdbService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { - // service - svc := &CockroachdbService{} - - // dependency injection - svc.modelSvc, err = service.GetService() - if err != nil { - return nil, trace.TraceError(err) - } - - // data source - if dsId.IsZero() { - svc.ds = &models.DataSource{} - } else { - svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) - if err != nil { - return nil, trace.TraceError(err) - } - } - - // data source defaults - if svc.ds.Host == "" { - svc.ds.Host = constants.DefaultHost - } - if svc.ds.Port == 0 { - svc.ds.Port = constants.DefaultCockroachdbPort - } - - // data source password - pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) - if err == nil { - svc.ds.Password, err = utils.DecryptAES(pwd.Password) - if err != nil { - return nil, err - } - } - - // data collection - svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) - if err != nil { - return nil, trace.TraceError(err) - } - - // session - svc.s, err = utils.GetCockroachdbSession(svc.ds) - if err != nil { - return nil, trace.TraceError(err) - } - - // collection - svc.col = svc.s.Collection(svc.dc.Name) - - return svc, nil -} diff --git a/core/ds/default.go b/core/ds/default.go deleted file mode 100644 index e5143a2c..00000000 --- a/core/ds/default.go +++ /dev/null @@ -1 +0,0 @@ -package ds diff --git a/core/ds/es.go b/core/ds/es.go deleted file mode 100644 index de87ee7c..00000000 --- a/core/ds/es.go +++ /dev/null @@ -1,220 +0,0 @@ -package ds - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - constants2 "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - entity2 "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/crawlab/trace" - "github.com/elastic/go-elasticsearch/v8" - "github.com/elastic/go-elasticsearch/v8/esapi" - "github.com/google/uuid" - "go.mongodb.org/mongo-driver/bson/primitive" - "strings" - "sync" - "time" -) - -type ElasticsearchService struct { - // dependencies - modelSvc service.ModelService - - // internals - dc *models.DataCollection // models.DataCollection - ds *models.DataSource // models.DataSource - c *elasticsearch.Client // elasticsearch.Client - t time.Time -} - -func (svc *ElasticsearchService) Insert(records ...interface{}) (err error) { - // wait group - var wg sync.WaitGroup - wg.Add(len(records)) - - // iterate records - for _, r := range records { - // async operation - go func(r interface{}) { - switch r.(type) { - case entity.Result: - // convert type to entity.Result - d := r.(entity.Result) - - // get document id - id := d.GetValue("id") - var docId string - switch id.(type) { - case string: - docId = id.(string) - } - if docId == "" { - docId = uuid.New().String() // generate new uuid if id is empty - } - - // collection - d[constants2.DataCollectionKey] = svc.dc.Name - - // index request - req := esapi.IndexRequest{ - Index: svc.getIndexName(), - DocumentID: docId, - Body: strings.NewReader(d.String()), - } - - // perform request - res, err := req.Do(context.Background(), svc.c) - if err != nil { - trace.PrintError(err) - wg.Done() - return - } - defer res.Body.Close() - if res.IsError() { - trace.PrintError(errors.New(fmt.Sprintf("[ElasticsearchService] [%s] error inserting record: %v", res.Status(), r))) - } - - // release - wg.Done() - default: - wg.Done() - return - } - }(r) - } - - // wait - wg.Wait() - - return nil -} - -func (svc *ElasticsearchService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { - data, err := svc.getListResponse(query, opts, false) - if err != nil { - return nil, err - } - for _, hit := range data.Hits.Hits { - results = append(results, hit.Source) - } - return results, nil -} - -func (svc *ElasticsearchService) Count(query generic.ListQuery) (n int, err error) { - data, err := svc.getListResponse(query, nil, true) - if err != nil { - return n, err - } - return int(data.Hits.Total.Value), nil -} - -func (svc *ElasticsearchService) getListResponse(query generic.ListQuery, opts *generic.ListOptions, trackTotalHits bool) (data *entity2.ElasticsearchResponseData, err error) { - if opts == nil { - opts = &generic.ListOptions{} - } - query = append(query, generic.ListQueryCondition{ - Key: constants2.DataCollectionKey, - Op: constants2.FilterOpEqual, - Value: svc.dc.Name, - }) - res, err := svc.c.Search( - svc.c.Search.WithContext(context.Background()), - svc.c.Search.WithIndex(svc.getIndexName()), - svc.c.Search.WithBody(utils.GetElasticsearchQueryWithOptions(query, opts)), - svc.c.Search.WithTrackTotalHits(trackTotalHits), - ) - if err != nil { - return nil, trace.TraceError(err) - } - defer res.Body.Close() - if res.IsError() { - err = errors.New(fmt.Sprintf("[ElasticsearchService] [%s] error listing records: response=%s, query=%v opts=%v", res.Status(), res.String(), query, opts)) - trace.PrintError(err) - return nil, err - } - data = &entity2.ElasticsearchResponseData{} - if err := json.NewDecoder(res.Body).Decode(data); err != nil { - return nil, trace.TraceError(err) - } - return data, nil -} - -func (svc *ElasticsearchService) getIndexName() (index string) { - if svc.ds.Database == "" { - return svc.dc.Name - } else { - return svc.ds.Name - } -} - -func NewDataSourceElasticsearchService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { - // service - svc := &ElasticsearchService{} - - // dependency injection - svc.modelSvc, err = service.GetService() - if err != nil { - return nil, err - } - - // data source - if dsId.IsZero() { - svc.ds = &models.DataSource{} - } else { - svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) - if err != nil { - return nil, err - } - } - - // data source defaults - if svc.ds.Host == "" { - svc.ds.Host = constants.DefaultHost - } - if svc.ds.Port == 0 { - svc.ds.Port = constants.DefaultElasticsearchPort - } - - // data source password - pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) - if err == nil { - svc.ds.Password, err = utils.DecryptAES(pwd.Password) - if err != nil { - return nil, err - } - } - - // data collection - svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) - if err != nil { - return nil, err - } - - // client - svc.c, err = utils.GetElasticsearchClient(svc.ds) - if err != nil { - return nil, err - } - - return svc, nil -} - -func (svc *ElasticsearchService) Index(fields []string) { - // TODO: implement me -} - -func (svc *ElasticsearchService) SetTime(t time.Time) { - svc.t = t -} - -func (svc *ElasticsearchService) GetTime() (t time.Time) { - return svc.t -} diff --git a/core/ds/kafka.go b/core/ds/kafka.go deleted file mode 100644 index 0bf54940..00000000 --- a/core/ds/kafka.go +++ /dev/null @@ -1,116 +0,0 @@ -package ds - -import ( - "github.com/cenkalti/backoff/v4" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/crawlab/trace" - "github.com/segmentio/kafka-go" - "go.mongodb.org/mongo-driver/bson/primitive" - "time" -) - -type KafkaService struct { - // dependencies - modelSvc service.ModelService - - // internals - dc *models.DataCollection // models.DataCollection - ds *models.DataSource // models.DataSource - c *kafka.Conn // kafka.Conn - rb backoff.BackOff - t time.Time -} - -func (svc *KafkaService) Insert(records ...interface{}) (err error) { - var messages []kafka.Message - for _, r := range records { - switch r.(type) { - case entity.Result: - d := r.(entity.Result) - messages = append(messages, kafka.Message{ - Topic: svc.ds.Database, - Key: []byte(d.GetTaskId().Hex()), - Value: d.Bytes(), - }) - } - } - _, err = svc.c.WriteMessages(messages...) - if err != nil { - return trace.TraceError(err) - } - return nil -} - -func (svc *KafkaService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { - // N/A - return nil, nil -} - -func (svc *KafkaService) Count(query generic.ListQuery) (n int, err error) { - // N/A - return 0, nil -} - -func NewDataSourceKafkaService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { - // service - svc := &KafkaService{} - - // dependency injection - svc.modelSvc, err = service.GetService() - if err != nil { - return nil, err - } - - // data source - if dsId.IsZero() { - svc.ds = &models.DataSource{} - } else { - svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) - if err != nil { - return nil, err - } - } - - // data source defaults - if svc.ds.Host == "" { - svc.ds.Host = constants.DefaultHost - } - if svc.ds.Port == 0 { - svc.ds.Port = constants.DefaultKafkaPort - } - - // data source password - pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) - if err == nil { - svc.ds.Password, err = utils.DecryptAES(pwd.Password) - if err != nil { - return nil, err - } - } - - // data collection - svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) - if err != nil { - return nil, err - } - - return svc, nil -} - -func (svc *KafkaService) Index(fields []string) { - // TODO: implement me -} - -func (svc *KafkaService) SetTime(t time.Time) { - svc.t = t -} - -func (svc *KafkaService) GetTime() (t time.Time) { - return svc.t -} diff --git a/core/ds/mongo.go b/core/ds/mongo.go deleted file mode 100644 index 29d912fa..00000000 --- a/core/ds/mongo.go +++ /dev/null @@ -1,118 +0,0 @@ -package ds - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - utils2 "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "time" -) - -type MongoService struct { - // dependencies - modelSvc service.ModelService - - // internals - dc *models.DataCollection // models.DataCollection - ds *models.DataSource // models.DataSource - c *mongo2.Client - db *mongo2.Database - col *mongo.Col - t time.Time -} - -func (svc *MongoService) Insert(records ...interface{}) (err error) { - _, err = svc.col.InsertMany(records) - return err -} - -func (svc *MongoService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { - var docs []models.Result - if err := svc.col.Find(utils.GetMongoQuery(query), utils.GetMongoOpts(opts)).All(&docs); err != nil { - return nil, err - } - for i := range docs { - results = append(results, &docs[i]) - } - return results, nil -} - -func (svc *MongoService) Count(query generic.ListQuery) (n int, err error) { - return svc.col.Count(utils.GetMongoQuery(query)) -} - -func NewDataSourceMongoService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { - // service - svc := &MongoService{} - - // dependency injection - svc.modelSvc, err = service.GetService() - if err != nil { - return nil, err - } - - // data source - if dsId.IsZero() { - svc.ds = &models.DataSource{} - } else { - svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) - if err != nil { - return nil, err - } - } - - // data source defaults - if svc.ds.Host == "" { - svc.ds.Host = constants.DefaultHost - } - if svc.ds.Port == 0 { - svc.ds.Port = constants.DefaultMongoPort - } - - // data source password - pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) - if err == nil { - svc.ds.Password, err = utils.DecryptAES(pwd.Password) - if err != nil { - return nil, err - } - } - - // data collection - svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) - if err != nil { - return nil, err - } - - // mongo client - svc.c, err = utils2.GetMongoClient(svc.ds) - if err != nil { - return nil, err - } - - // mongo database - svc.db = mongo.GetMongoDb(svc.ds.Database, mongo.WithDbClient(svc.c)) - - // mongo col - svc.col = mongo.GetMongoColWithDb(svc.dc.Name, svc.db) - - return svc, nil -} - -func (svc *MongoService) Index(fields []string) { - // TODO: implement me -} - -func (svc *MongoService) SetTime(t time.Time) { - svc.t = t -} - -func (svc *MongoService) GetTime() (t time.Time) { - return svc.t -} diff --git a/core/ds/mssql.go b/core/ds/mssql.go deleted file mode 100644 index 516f7a32..00000000 --- a/core/ds/mssql.go +++ /dev/null @@ -1,71 +0,0 @@ -package ds - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - utils2 "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type MssqlService struct { - SqlService -} - -func NewDataSourceMssqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { - // service - svc := &MssqlService{} - - // dependency injection - svc.modelSvc, err = service.GetService() - if err != nil { - return nil, trace.TraceError(err) - } - - // data source - if dsId.IsZero() { - svc.ds = &models.DataSource{} - } else { - svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) - if err != nil { - return nil, trace.TraceError(err) - } - } - - // data source defaults - if svc.ds.Host == "" { - svc.ds.Host = constants.DefaultHost - } - if svc.ds.Port == 0 { - svc.ds.Port = constants.DefaultMssqlPort - } - - // data source password - pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) - if err == nil { - svc.ds.Password, err = utils.DecryptAES(pwd.Password) - if err != nil { - return nil, err - } - } - - // data collection - svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) - if err != nil { - return nil, trace.TraceError(err) - } - - // session - svc.s, err = utils2.GetMssqlSession(svc.ds) - if err != nil { - return nil, trace.TraceError(err) - } - - // collection - svc.col = svc.s.Collection(svc.dc.Name) - - return svc, nil -} diff --git a/core/ds/mssql_test.go b/core/ds/mssql_test.go deleted file mode 100644 index 229197c7..00000000 --- a/core/ds/mssql_test.go +++ /dev/null @@ -1,8 +0,0 @@ -package ds - -import "testing" - -func TestNewDataSourceMssqlService(t *testing.T) { - t.Run("insert", func(t *testing.T) { - }) -} diff --git a/core/ds/mysql.go b/core/ds/mysql.go deleted file mode 100644 index fc2f3bae..00000000 --- a/core/ds/mysql.go +++ /dev/null @@ -1,69 +0,0 @@ -package ds - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - utils2 "github.com/crawlab-team/crawlab/core/utils" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type MysqlService struct { - SqlService -} - -func NewDataSourceMysqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { - // service - svc := &MysqlService{} - - // dependency injection - svc.modelSvc, err = service.GetService() - if err != nil { - return nil, err - } - - // data source - if dsId.IsZero() { - svc.ds = &models.DataSource{} - } else { - svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) - if err != nil { - return nil, err - } - } - - // data source defaults - if svc.ds.Host == "" { - svc.ds.Host = constants.DefaultHost - } - if svc.ds.Port == 0 { - svc.ds.Port = constants.DefaultMysqlPort - } - - // data source password - pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) - if err == nil { - svc.ds.Password, err = utils.DecryptAES(pwd.Password) - if err != nil { - return nil, err - } - } - // data collection - svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) - if err != nil { - return nil, err - } - - // session - svc.s, err = utils2.GetMysqlSession(svc.ds) - if err != nil { - return nil, err - } - - // collection - svc.col = svc.s.Collection(svc.dc.Name) - - return svc, nil -} diff --git a/core/ds/options.go b/core/ds/options.go deleted file mode 100644 index 58cc970e..00000000 --- a/core/ds/options.go +++ /dev/null @@ -1,14 +0,0 @@ -package ds - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "time" -) - -type DataSourceServiceOption func(svc interfaces.DataSourceService) - -func WithMonitorInterval(duration time.Duration) DataSourceServiceOption { - return func(svc interfaces.DataSourceService) { - svc.SetMonitorInterval(duration) - } -} diff --git a/core/ds/postgresql.go b/core/ds/postgresql.go deleted file mode 100644 index 485a3a96..00000000 --- a/core/ds/postgresql.go +++ /dev/null @@ -1,71 +0,0 @@ -package ds - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - utils2 "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type PostgresqlService struct { - SqlService -} - -func NewDataSourcePostgresqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { - // service - svc := &PostgresqlService{} - - // dependency injection - svc.modelSvc, err = service.GetService() - if err != nil { - return nil, trace.TraceError(err) - } - - // data source - if dsId.IsZero() { - svc.ds = &models.DataSource{} - } else { - svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) - if err != nil { - return nil, trace.TraceError(err) - } - } - - // data source defaults - if svc.ds.Host == "" { - svc.ds.Host = constants.DefaultHost - } - if svc.ds.Port == 0 { - svc.ds.Port = constants.DefaultPostgresqlPort - } - - // data source password - pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) - if err == nil { - svc.ds.Password, err = utils.DecryptAES(pwd.Password) - if err != nil { - return nil, err - } - } - - // data collection - svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) - if err != nil { - return nil, trace.TraceError(err) - } - - // session - svc.s, err = utils2.GetPostgresqlSession(svc.ds) - if err != nil { - return nil, trace.TraceError(err) - } - - // collection - svc.col = svc.s.Collection(svc.dc.Name) - - return svc, nil -} diff --git a/core/ds/service.go b/core/ds/service.go deleted file mode 100644 index 9991e4d5..00000000 --- a/core/ds/service.go +++ /dev/null @@ -1,305 +0,0 @@ -package ds - -import ( - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/constants" - constants2 "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/result" - "github.com/crawlab-team/crawlab/core/utils" - utils2 "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "sync" - "time" -) - -type Service struct { - // dependencies - modelSvc service.ModelService - - // internals - timeout time.Duration - monitorInterval time.Duration - stopped bool -} - -func (svc *Service) Init() { - // result service registry - reg := result.GetResultServiceRegistry() - - // register result services - reg.Register(constants.DataSourceTypeMongo, NewDataSourceMongoService) - reg.Register(constants.DataSourceTypeMysql, NewDataSourceMysqlService) - reg.Register(constants.DataSourceTypePostgresql, NewDataSourcePostgresqlService) - reg.Register(constants.DataSourceTypeMssql, NewDataSourceMssqlService) - reg.Register(constants.DataSourceTypeSqlite, NewDataSourceSqliteService) - reg.Register(constants.DataSourceTypeCockroachdb, NewDataSourceCockroachdbService) - reg.Register(constants.DataSourceTypeElasticSearch, NewDataSourceElasticsearchService) - reg.Register(constants.DataSourceTypeKafka, NewDataSourceKafkaService) -} - -func (svc *Service) Start() { - // start monitoring - go svc.Monitor() -} - -func (svc *Service) Wait() { - utils.DefaultWait() -} - -func (svc *Service) Stop() { - svc.stopped = true -} - -func (svc *Service) ChangePassword(id primitive.ObjectID, password string) (err error) { - p, err := svc.modelSvc.GetPasswordById(id) - if err == nil { - // exists, save - encryptedPassword, err := utils.EncryptAES(password) - if err != nil { - return err - } - p.Password = encryptedPassword - if err := delegate.NewModelDelegate(p).Save(); err != nil { - return err - } - return nil - } else if err.Error() == mongo.ErrNoDocuments.Error() { - // not exists, add - encryptedPassword, err := utils.EncryptAES(password) - if err != nil { - return err - } - p = &models.Password{ - Id: id, - Password: encryptedPassword, - } - if err := delegate.NewModelDelegate(p).Add(); err != nil { - return err - } - return nil - } else { - // error - return err - } -} - -func (svc *Service) Monitor() { - for { - // return if stopped - if svc.stopped { - return - } - - // monitor - if err := svc.monitor(); err != nil { - trace.PrintError(err) - } - - // wait - time.Sleep(svc.monitorInterval) - } -} - -func (svc *Service) CheckStatus(id primitive.ObjectID) (err error) { - ds, err := svc.modelSvc.GetDataSourceById(id) - if err != nil { - return err - } - return svc.checkStatus(ds, true) -} - -func (svc *Service) SetTimeout(duration time.Duration) { - svc.timeout = duration -} - -func (svc *Service) SetMonitorInterval(duration time.Duration) { - svc.monitorInterval = duration -} - -func (svc *Service) monitor() (err error) { - // start - tic := time.Now() - log.Debugf("[DataSourceService] start monitoring") - - // data source list - dsList, err := svc.modelSvc.GetDataSourceList(nil, nil) - if err != nil { - return err - } - - // waiting group - wg := sync.WaitGroup{} - wg.Add(len(dsList)) - - // iterate data source list - for _, ds := range dsList { - // async operation - go func(ds models.DataSource) { - // check status and save - _ = svc.checkStatus(&ds, true) - - // release - wg.Done() - }(ds) - } - - // wait - wg.Wait() - - // finish - toc := time.Now() - log.Debugf("[DataSourceService] finished monitoring. elapsed: %d ms", (toc.Sub(tic)).Milliseconds()) - - return nil -} - -func (svc *Service) checkStatus(ds *models.DataSource, save bool) (err error) { - // password - if ds.Password == "" { - pwd, err := svc.modelSvc.GetPasswordById(ds.Id) - if err == nil { - ds.Password, err = utils.DecryptAES(pwd.Password) - if err != nil { - return err - } - } else if err.Error() != mongo.ErrNoDocuments.Error() { - return trace.TraceError(err) - } - } - - // check status - if err := svc._checkStatus(ds); err != nil { - ds.Status = constants2.DataSourceStatusOffline - ds.Error = err.Error() - } else { - ds.Status = constants2.DataSourceStatusOnline - ds.Error = "" - } - - // save - if save { - return svc._save(ds) - } - - return nil -} - -func (svc *Service) _save(ds *models.DataSource) (err error) { - log.Debugf("[DataSourceService] saving data source: name=%s, type=%s, status=%s, error=%s", ds.Name, ds.Type, ds.Status, ds.Error) - return delegate.NewModelDelegate(ds).Save() -} - -func (svc *Service) _checkStatus(ds *models.DataSource) (err error) { - switch ds.Type { - case constants.DataSourceTypeMongo: - _, err := utils2.GetMongoClientWithTimeout(ds, svc.timeout) - if err != nil { - return err - } - case constants.DataSourceTypeMysql: - s, err := utils2.GetMysqlSessionWithTimeout(ds, svc.timeout) - if err != nil { - return err - } - if s != nil { - s.Close() - } - case constants.DataSourceTypePostgresql: - s, err := utils2.GetPostgresqlSessionWithTimeout(ds, svc.timeout) - if err != nil { - return err - } - if s != nil { - s.Close() - } - case constants.DataSourceTypeMssql: - s, err := utils2.GetMssqlSessionWithTimeout(ds, svc.timeout) - if err != nil { - return err - } - if s != nil { - s.Close() - } - case constants.DataSourceTypeSqlite: - s, err := utils2.GetSqliteSessionWithTimeout(ds, svc.timeout) - if err != nil { - return err - } - if s != nil { - s.Close() - } - case constants.DataSourceTypeCockroachdb: - s, err := utils2.GetCockroachdbSessionWithTimeout(ds, svc.timeout) - if err != nil { - return err - } - if s != nil { - s.Close() - } - case constants.DataSourceTypeElasticSearch: - _, err := utils2.GetElasticsearchClientWithTimeout(ds, svc.timeout) - if err != nil { - return err - } - case constants.DataSourceTypeKafka: - c, err := utils2.GetKafkaConnectionWithTimeout(ds, svc.timeout) - if err != nil { - return err - } - if c != nil { - c.Close() - } - default: - log.Warnf("[DataSourceService] invalid data source type: %s", ds.Type) - } - return nil -} - -func NewDataSourceService(opts ...DataSourceServiceOption) (svc2 interfaces.DataSourceService, err error) { - // service - svc := &Service{ - monitorInterval: 15 * time.Second, - timeout: 10 * time.Second, - } - - // apply options - for _, opt := range opts { - opt(svc) - } - - // dependency injection - if err := container.GetContainer().Invoke(func(modelSvc service.ModelService) { - svc.modelSvc = modelSvc - }); err != nil { - return nil, trace.TraceError(err) - } - - // initialize - svc.Init() - - // start - svc.Start() - - return svc, nil -} - -var _dsSvc interfaces.DataSourceService - -func GetDataSourceService() (svc interfaces.DataSourceService, err error) { - if _dsSvc != nil { - return _dsSvc, nil - } - svc, err = NewDataSourceService() - if err != nil { - return nil, err - } - _dsSvc = svc - return svc, nil -} diff --git a/core/ds/sql.go b/core/ds/sql.go deleted file mode 100644 index 1fa7b100..00000000 --- a/core/ds/sql.go +++ /dev/null @@ -1,76 +0,0 @@ -package ds - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - utils2 "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/crawlab/trace" - "github.com/upper/db/v4" - "time" -) - -type SqlService struct { - // dependencies - modelSvc service.ModelService - - // internals - ds *models.DataSource - dc *models.DataCollection - s db.Session - col db.Collection - t time.Time -} - -func (svc *SqlService) Insert(records ...interface{}) (err error) { - for _, d := range records { - var r entity.Result - switch d.(type) { - case entity.Result: - r = d.(entity.Result) - default: - continue - } - _r := r.Flatten() - if _, err = svc.col.Insert(_r); err != nil { - trace.PrintError(err) - continue - } - } - return nil -} - -func (svc *SqlService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { - var docs []entity.Result - if err := svc.col.Find(utils2.GetSqlQuery(query)). - Offset(opts.Skip). - Limit(opts.Limit).All(&docs); err != nil { - return nil, trace.TraceError(err) - } - for i := range docs { - d := docs[i].ToJSON() - results = append(results, &d) - } - return results, nil -} - -func (svc *SqlService) Count(query generic.ListQuery) (n int, err error) { - nInt64, err := svc.col.Find(utils2.GetSqlQuery(query)).Count() - if err != nil { - return n, err - } - return int(nInt64), nil -} - -func (svc *SqlService) Index(fields []string) { - // TODO: implement me -} - -func (svc *SqlService) SetTime(t time.Time) { - svc.t = t -} - -func (svc *SqlService) GetTime() (t time.Time) { - return svc.t -} diff --git a/core/ds/sql_options.go b/core/ds/sql_options.go deleted file mode 100644 index e7d26876..00000000 --- a/core/ds/sql_options.go +++ /dev/null @@ -1,6 +0,0 @@ -package ds - -type SqlOptions struct { - DefaultHost string - DefaultPort string -} diff --git a/core/ds/sqlite.go b/core/ds/sqlite.go deleted file mode 100644 index 9e682cd5..00000000 --- a/core/ds/sqlite.go +++ /dev/null @@ -1,52 +0,0 @@ -package ds - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - utils2 "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type SqliteService struct { - SqlService -} - -func NewDataSourceSqliteService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { - // service - svc := &SqliteService{} - - // dependency injection - svc.modelSvc, err = service.GetService() - if err != nil { - return nil, trace.TraceError(err) - } - - // data source - if dsId.IsZero() { - svc.ds = &models.DataSource{} - } else { - svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) - if err != nil { - return nil, trace.TraceError(err) - } - } - - // data collection - svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) - if err != nil { - return nil, trace.TraceError(err) - } - - // session - svc.s, err = utils2.GetSqliteSession(svc.ds) - if err != nil { - return nil, trace.TraceError(err) - } - - // collection - svc.col = svc.s.Collection(svc.dc.Name) - - return svc, nil -} diff --git a/core/grpc/client/client.go b/core/grpc/client/client.go deleted file mode 100644 index 6767e8d7..00000000 --- a/core/grpc/client/client.go +++ /dev/null @@ -1,420 +0,0 @@ -package client - -import ( - "context" - "encoding/json" - "github.com/apex/log" - "github.com/cenkalti/backoff/v4" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/grpc/middlewares" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/utils" - grpc2 "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "github.com/spf13/viper" - "google.golang.org/grpc" - "google.golang.org/grpc/connectivity" - "io" - "time" -) - -type Client struct { - // dependencies - nodeCfgSvc interfaces.NodeConfigService - - // settings - cfgPath string - address interfaces.Address - timeout time.Duration - subscribeType string - handleMessage bool - - // internals - conn *grpc.ClientConn - stream grpc2.NodeService_SubscribeClient - msgCh chan *grpc2.StreamMessage - err error - - // grpc clients - ModelDelegateClient grpc2.ModelDelegateClient - ModelBaseServiceClient grpc2.ModelBaseServiceClient - NodeClient grpc2.NodeServiceClient - TaskClient grpc2.TaskServiceClient - MessageClient grpc2.MessageServiceClient -} - -func (c *Client) Init() (err error) { - // do nothing - return nil -} - -func (c *Client) Start() (err error) { - // connect - if err := c.connect(); err != nil { - return err - } - - // register rpc services - if err := c.Register(); err != nil { - return err - } - - // subscribe - if err := c.subscribe(); err != nil { - return err - } - - // handle stream message - if c.handleMessage { - go c.handleStreamMessage() - } - - return nil -} - -func (c *Client) Stop() (err error) { - // skip if connection is nil - if c.conn == nil { - return nil - } - - // grpc server address - address := c.address.String() - - // unsubscribe - if err := c.unsubscribe(); err != nil { - return err - } - log.Infof("grpc client unsubscribed from %s", address) - - // close connection - if err := c.conn.Close(); err != nil { - return err - } - log.Infof("grpc client disconnected from %s", address) - - return nil -} - -func (c *Client) Register() (err error) { - // model delegate - c.ModelDelegateClient = grpc2.NewModelDelegateClient(c.conn) - - // model base service - c.ModelBaseServiceClient = grpc2.NewModelBaseServiceClient(c.conn) - - // node - c.NodeClient = grpc2.NewNodeServiceClient(c.conn) - - // task - c.TaskClient = grpc2.NewTaskServiceClient(c.conn) - - // message - c.MessageClient = grpc2.NewMessageServiceClient(c.conn) - - // log - log.Infof("[GrpcClient] grpc client registered client services") - log.Debugf("[GrpcClient] ModelDelegateClient: %v", c.ModelDelegateClient) - log.Debugf("[GrpcClient] ModelBaseServiceClient: %v", c.ModelBaseServiceClient) - log.Debugf("[GrpcClient] NodeClient: %v", c.NodeClient) - log.Debugf("[GrpcClient] TaskClient: %v", c.TaskClient) - log.Debugf("[GrpcClient] MessageClient: %v", c.MessageClient) - - return nil -} - -func (c *Client) GetModelDelegateClient() (res grpc2.ModelDelegateClient) { - return c.ModelDelegateClient -} - -func (c *Client) GetModelBaseServiceClient() (res grpc2.ModelBaseServiceClient) { - return c.ModelBaseServiceClient -} - -func (c *Client) GetNodeClient() grpc2.NodeServiceClient { - return c.NodeClient -} - -func (c *Client) GetTaskClient() grpc2.TaskServiceClient { - return c.TaskClient -} - -func (c *Client) GetMessageClient() grpc2.MessageServiceClient { - return c.MessageClient -} - -func (c *Client) SetAddress(address interfaces.Address) { - c.address = address -} - -func (c *Client) SetTimeout(timeout time.Duration) { - c.timeout = timeout -} - -func (c *Client) SetSubscribeType(value string) { - c.subscribeType = value -} - -func (c *Client) SetHandleMessage(handleMessage bool) { - c.handleMessage = handleMessage -} - -func (c *Client) Context() (ctx context.Context, cancel context.CancelFunc) { - return context.WithTimeout(context.Background(), c.timeout) -} - -func (c *Client) NewRequest(d interface{}) (req *grpc2.Request) { - return &grpc2.Request{ - NodeKey: c.nodeCfgSvc.GetNodeKey(), - Data: c.getRequestData(d), - } -} - -func (c *Client) GetConfigPath() (path string) { - return c.cfgPath -} - -func (c *Client) SetConfigPath(path string) { - c.cfgPath = path -} - -func (c *Client) NewModelBaseServiceRequest(id interfaces.ModelId, params interfaces.GrpcBaseServiceParams) (req *grpc2.Request, err error) { - data, err := json.Marshal(params) - if err != nil { - return nil, trace.TraceError(err) - } - msg := &entity.GrpcBaseServiceMessage{ - ModelId: id, - Data: data, - } - return c.NewRequest(msg), nil -} - -func (c *Client) GetMessageChannel() (msgCh chan *grpc2.StreamMessage) { - return c.msgCh -} - -func (c *Client) Restart() (err error) { - if c.needRestart() { - return c.Start() - } - return nil -} - -func (c *Client) IsStarted() (res bool) { - return c.conn != nil -} - -func (c *Client) IsClosed() (res bool) { - if c.conn != nil { - return c.conn.GetState() == connectivity.Shutdown - } - return false -} - -func (c *Client) Err() (err error) { - return c.err -} - -func (c *Client) GetStream() (stream grpc2.NodeService_SubscribeClient) { - return c.stream -} - -func (c *Client) connect() (err error) { - return backoff.RetryNotify(c._connect, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client connect")) -} - -func (c *Client) _connect() (err error) { - // grpc server address - address := c.address.String() - - // timeout context - ctx, cancel := context.WithTimeout(context.Background(), c.timeout) - defer cancel() - - // connection - // TODO: configure dial options - var opts []grpc.DialOption - opts = append(opts, grpc.WithInsecure()) - opts = append(opts, grpc.WithBlock()) - opts = append(opts, grpc.WithChainUnaryInterceptor(middlewares.GetAuthTokenUnaryChainInterceptor(c.nodeCfgSvc))) - opts = append(opts, grpc.WithChainStreamInterceptor(middlewares.GetAuthTokenStreamChainInterceptor(c.nodeCfgSvc))) - c.conn, err = grpc.DialContext(ctx, address, opts...) - if err != nil { - _ = trace.TraceError(err) - return errors.ErrorGrpcClientFailedToStart - } - log.Infof("[GrpcClient] grpc client connected to %s", address) - - return nil -} - -func (c *Client) subscribe() (err error) { - var op func() error - switch c.subscribeType { - case constants.GrpcSubscribeTypeNode: - op = c._subscribeNode - default: - return errors.ErrorGrpcInvalidType - } - return backoff.RetryNotify(op, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client subscribe")) -} - -func (c *Client) _subscribeNode() (err error) { - req := c.NewRequest(&entity.NodeInfo{ - Key: c.nodeCfgSvc.GetNodeKey(), - IsMaster: false, - }) - c.stream, err = c.GetNodeClient().Subscribe(context.Background(), req) - if err != nil { - return trace.TraceError(err) - } - - // log - log.Infof("[GrpcClient] grpc client subscribed to remote server") - - return nil -} - -func (c *Client) unsubscribe() (err error) { - req := c.NewRequest(&entity.NodeInfo{ - Key: c.nodeCfgSvc.GetNodeKey(), - IsMaster: false, - }) - if _, err = c.GetNodeClient().Unsubscribe(context.Background(), req); err != nil { - return trace.TraceError(err) - } - return nil -} - -func (c *Client) handleStreamMessage() { - log.Infof("[GrpcClient] start handling stream message...") - for { - // resubscribe if stream is set to nil - if c.stream == nil { - if err := backoff.RetryNotify(c.subscribe, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client subscribe")); err != nil { - log.Errorf("subscribe") - return - } - } - - // receive stream message - msg, err := c.stream.Recv() - log.Debugf("[GrpcClient] received message: %v", msg) - if err != nil { - // set error - c.err = err - - // end - if err == io.EOF { - log.Infof("[GrpcClient] received EOF signal, disconnecting") - return - } - - // connection closed - if c.IsClosed() { - return - } - - // error - trace.PrintError(err) - c.stream = nil - time.Sleep(1 * time.Second) - continue - } - - // send stream message to channel - c.msgCh <- msg - - // reset error - c.err = nil - } -} - -func (c *Client) needRestart() bool { - switch c.conn.GetState() { - case connectivity.Shutdown, connectivity.TransientFailure: - return true - case connectivity.Idle, connectivity.Connecting, connectivity.Ready: - return false - default: - return false - } -} - -func (c *Client) getRequestData(d interface{}) (data []byte) { - if d == nil { - return data - } - switch d.(type) { - case []byte: - data = d.([]byte) - default: - var err error - data, err = json.Marshal(d) - if err != nil { - panic(err) - } - } - return data -} - -func NewClient() (res interfaces.GrpcClient, err error) { - // client - client := &Client{ - address: entity.NewAddress(&entity.AddressOptions{ - Host: constants.DefaultGrpcClientRemoteHost, - Port: constants.DefaultGrpcClientRemotePort, - }), - timeout: 10 * time.Second, - msgCh: make(chan *grpc2.StreamMessage), - subscribeType: constants.GrpcSubscribeTypeNode, - handleMessage: true, - } - - if viper.GetString("grpc.address") != "" { - client.address, err = entity.NewAddressFromString(viper.GetString("grpc.address")) - if err != nil { - return nil, trace.TraceError(err) - } - } - - // dependency injection - if err := container.GetContainer().Invoke(func(nodeCfgSvc interfaces.NodeConfigService) { - client.nodeCfgSvc = nodeCfgSvc - }); err != nil { - return nil, err - } - - // init - if err := client.Init(); err != nil { - return nil, err - } - - return client, nil -} - -var _client interfaces.GrpcClient - -func GetClient() (c interfaces.GrpcClient, err error) { - if _client != nil { - return _client, nil - } - _client, err = createClient() - if err != nil { - return nil, err - } - return _client, nil -} - -func createClient() (client2 interfaces.GrpcClient, err error) { - if err := container.GetContainer().Invoke(func(client interfaces.GrpcClient) { - client2 = client - }); err != nil { - return nil, trace.TraceError(err) - } - return client2, nil -} diff --git a/core/grpc/client/options.go b/core/grpc/client/options.go index 8f69e3a9..8936e657 100644 --- a/core/grpc/client/options.go +++ b/core/grpc/client/options.go @@ -2,50 +2,12 @@ package client import ( "github.com/crawlab-team/crawlab/core/interfaces" - "time" ) type Option func(client interfaces.GrpcClient) -func WithConfigPath(path string) Option { - return func(c interfaces.GrpcClient) { - c.SetConfigPath(path) - } -} - func WithAddress(address interfaces.Address) Option { return func(c interfaces.GrpcClient) { c.SetAddress(address) } } - -func WithTimeout(timeout time.Duration) Option { - return func(c interfaces.GrpcClient) { - } -} - -func WithSubscribeType(subscribeType string) Option { - return func(c interfaces.GrpcClient) { - c.SetSubscribeType(subscribeType) - } -} - -func WithHandleMessage(handleMessage bool) Option { - return func(c interfaces.GrpcClient) { - c.SetHandleMessage(handleMessage) - } -} - -type PoolOption func(p interfaces.GrpcClientPool) - -func WithPoolConfigPath(path string) PoolOption { - return func(c interfaces.GrpcClientPool) { - c.SetConfigPath(path) - } -} - -func WithPoolSize(size int) PoolOption { - return func(c interfaces.GrpcClientPool) { - c.SetSize(size) - } -} diff --git a/core/grpc/client/pool.go b/core/grpc/client/pool.go deleted file mode 100644 index 6559610c..00000000 --- a/core/grpc/client/pool.go +++ /dev/null @@ -1,88 +0,0 @@ -package client - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/trace" - "github.com/emirpasic/gods/lists/arraylist" - "math/rand" -) - -type Pool struct { - // settings - size int - cfgPath string - - // internals - clients *arraylist.List -} - -func (p *Pool) GetConfigPath() (path string) { - return p.cfgPath -} - -func (p *Pool) SetConfigPath(path string) { - p.cfgPath = path -} - -func (p *Pool) Init() (err error) { - for i := 0; i < p.size; i++ { - if err := p.NewClient(); err != nil { - return err - } - } - return nil -} - -func (p *Pool) NewClient() (err error) { - c, err := NewClient() - if err != nil { - return trace.TraceError(err) - } - if err := c.Start(); err != nil { - return err - } - p.clients.Add(c) - return nil -} - -func (p *Pool) GetClient() (c interfaces.GrpcClient, err error) { - idx := p.getRandomIndex() - res, ok := p.clients.Get(idx) - if !ok { - return nil, trace.TraceError(errors.ErrorGrpcClientNotExists) - } - c, ok = res.(interfaces.GrpcClient) - if !ok { - return nil, trace.TraceError(errors.ErrorGrpcInvalidType) - } - return c, nil -} - -func (p *Pool) SetSize(size int) { - p.size = size -} - -func (p *Pool) getRandomIndex() (idx int) { - return rand.Intn(p.clients.Size()) -} - -func NewPool(opts ...PoolOption) (p interfaces.GrpcClientPool, err error) { - // pool - p = &Pool{ - size: 1, - clients: arraylist.New(), - } - - // apply options - for _, opt := range opts { - opt(p) - } - - // initialize - if err := p.Init(); err != nil { - return nil, err - } - - return p, nil -} diff --git a/core/grpc/client/utils_proto.go b/core/grpc/client/utils_proto.go deleted file mode 100644 index e4759233..00000000 --- a/core/grpc/client/utils_proto.go +++ /dev/null @@ -1,5 +0,0 @@ -package client - -import grpc2 "github.com/crawlab-team/crawlab/grpc" - -var EmptyRequest = &grpc2.Request{} diff --git a/core/grpc/server/message_server.go b/core/grpc/server/message_server.go deleted file mode 100644 index ea29df09..00000000 --- a/core/grpc/server/message_server.go +++ /dev/null @@ -1,93 +0,0 @@ -package server - -import ( - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "io" -) - -type MessageServer struct { - grpc.UnimplementedMessageServiceServer - - // dependencies - modelSvc service.ModelService - cfgSvc interfaces.NodeConfigService - - // internals - server interfaces.GrpcServer -} - -func (svr MessageServer) Connect(stream grpc.MessageService_ConnectServer) (err error) { - finished := make(chan bool) - for { - msg, err := stream.Recv() - nodeKey := "unknown node key" - if msg != nil { - nodeKey = msg.NodeKey - } - if err == io.EOF { - log.Infof("[MessageServer] received signal EOF from node[%s], now quit", nodeKey) - return nil - } - if err != nil { - log.Errorf("[MessageServer] receiving message error from node[%s]: %v", nodeKey, err) - return err - } - switch msg.Code { - case grpc.StreamMessageCode_CONNECT: - log.Infof("[MessageServer] received connect request from node[%s], key: %s", nodeKey, msg.Key) - svr.server.SetSubscribe(msg.Key, &entity.GrpcSubscribe{ - Stream: stream, - Finished: finished, - }) - case grpc.StreamMessageCode_DISCONNECT: - log.Infof("[MessageServer] received disconnect request from node[%s], key: %s", nodeKey, msg.Key) - svr.server.DeleteSubscribe(msg.Key) - return nil - case grpc.StreamMessageCode_SEND: - log.Debugf("[MessageServer] received send request from node[%s] to %s", nodeKey, msg.To) - sub, err := svr.server.GetSubscribe(msg.To) - if err != nil { - return err - } - svr.redirectMessage(sub, msg) - } - } -} - -func (svr MessageServer) redirectMessage(sub interfaces.GrpcSubscribe, msg *grpc.StreamMessage) { - stream := sub.GetStream() - if stream == nil { - trace.PrintError(errors.ErrorGrpcStreamNotFound) - return - } - log.Debugf("[MessageServer] redirect message: %v", msg) - if err := stream.Send(msg); err != nil { - trace.PrintError(err) - return - } -} - -func NewMessageServer() (res *MessageServer, err error) { - // message server - svr := &MessageServer{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - cfgSvc interfaces.NodeConfigService, - ) { - svr.modelSvc = modelSvc - svr.cfgSvc = cfgSvc - }); err != nil { - return nil, err - } - - return svr, nil -} diff --git a/core/grpc/server/model_base_service_server.go b/core/grpc/server/model_base_service_server.go deleted file mode 100644 index 661c9505..00000000 --- a/core/grpc/server/model_base_service_server.go +++ /dev/null @@ -1,139 +0,0 @@ -package server - -import ( - "context" - "encoding/json" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" -) - -type ModelBaseServiceServer struct { - grpc.UnimplementedModelBaseServiceServer - - // dependencies - modelSvc interfaces.ModelService -} - -func (svr ModelBaseServiceServer) GetById(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { - return svc.GetById(params.Id) - }) -} - -func (svr ModelBaseServiceServer) Get(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { - return svc.Get(utils.NormalizeBsonMObjectId(params.Query), params.FindOptions) - }) -} - -func (svr ModelBaseServiceServer) GetList(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { - list, err := svc.GetList(utils.NormalizeBsonMObjectId(params.Query), params.FindOptions) - if err != nil { - return nil, err - } - data, err := json.Marshal(list) - if err != nil { - return nil, err - } - return data, nil - }) -} - -func (svr ModelBaseServiceServer) DeleteById(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { - err := svc.DeleteById(params.Id, params.User) - return nil, err - }) -} - -func (svr ModelBaseServiceServer) Delete(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { - err := svc.Delete(utils.NormalizeBsonMObjectId(params.Query), params.User) - return nil, err - }) -} - -func (svr ModelBaseServiceServer) DeleteList(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { - err := svc.DeleteList(utils.NormalizeBsonMObjectId(params.Query), params.User) - return nil, err - }) -} - -func (svr ModelBaseServiceServer) ForceDeleteList(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { - err := svc.ForceDeleteList(utils.NormalizeBsonMObjectId(params.Query), params.User) - return nil, err - }) -} - -func (svr ModelBaseServiceServer) UpdateById(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { - err := svc.UpdateById(params.Id, params.Update) - return nil, err - }) -} - -func (svr ModelBaseServiceServer) Update(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { - err := svc.Update(utils.NormalizeBsonMObjectId(params.Query), params.Update, params.Fields, params.User) - return nil, err - }) -} - -func (svr ModelBaseServiceServer) UpdateDoc(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { - err := svc.UpdateDoc(utils.NormalizeBsonMObjectId(params.Query), params.Doc, params.Fields, params.User) - return nil, err - }) -} - -func (svr ModelBaseServiceServer) Insert(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { - err := svc.Insert(params.User, params.Docs...) - return nil, err - }) -} - -func (svr ModelBaseServiceServer) Count(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { - return svc.Count(utils.NormalizeBsonMObjectId(params.Query)) - }) -} - -func (svr ModelBaseServiceServer) handleRequest(req *grpc.Request, handle handleBaseServiceRequest) (res *grpc.Response, err error) { - params, msg, err := NewModelBaseServiceBinder(req).BindWithBaseServiceMessage() - if err != nil { - return HandleError(err) - } - svc := svr.modelSvc.GetBaseService(msg.GetModelId()) - d, err := handle(params, svc) - if err != nil { - return HandleError(err) - } - if d == nil { - return HandleSuccess() - } - return HandleSuccessWithData(d) -} - -type handleBaseServiceRequest func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) - -func NewModelBaseServiceServer() (svr2 *ModelBaseServiceServer, err error) { - svr := &ModelBaseServiceServer{} - - // dependency injection - if err := container.GetContainer().Invoke(func(modelSvc service.ModelService) { - svr.modelSvc = modelSvc - }); err != nil { - return nil, trace.TraceError(err) - } - - return svr, nil -} diff --git a/core/grpc/server/model_delegate_binder.go b/core/grpc/server/model_delegate_binder.go deleted file mode 100644 index 172eae0e..00000000 --- a/core/grpc/server/model_delegate_binder.go +++ /dev/null @@ -1,120 +0,0 @@ -package server - -import ( - "encoding/json" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/grpc" -) - -func NewModelDelegateBinder(req *grpc.Request) (b *ModelDelegateBinder) { - return &ModelDelegateBinder{ - req: req, - msg: &entity.GrpcDelegateMessage{}, - } -} - -type ModelDelegateBinder struct { - req *grpc.Request - msg interfaces.GrpcModelDelegateMessage -} - -func (b *ModelDelegateBinder) Bind() (res interface{}, err error) { - if err := b.bindDelegateMessage(); err != nil { - return nil, err - } - - m := models.NewModelMap() - - switch b.msg.GetModelId() { - case interfaces.ModelIdArtifact: - return b.process(&m.Artifact, interfaces.ModelIdTag) - case interfaces.ModelIdTag: - return b.process(&m.Tag, interfaces.ModelIdTag) - case interfaces.ModelIdNode: - return b.process(&m.Node, interfaces.ModelIdTag) - case interfaces.ModelIdProject: - return b.process(&m.Project, interfaces.ModelIdTag) - case interfaces.ModelIdSpider: - return b.process(&m.Spider, interfaces.ModelIdTag) - case interfaces.ModelIdTask: - return b.process(&m.Task) - case interfaces.ModelIdJob: - return b.process(&m.Job) - case interfaces.ModelIdSchedule: - return b.process(&m.Schedule) - case interfaces.ModelIdUser: - return b.process(&m.User) - case interfaces.ModelIdSetting: - return b.process(&m.Setting) - case interfaces.ModelIdToken: - return b.process(&m.Token) - case interfaces.ModelIdVariable: - return b.process(&m.Variable) - case interfaces.ModelIdTaskQueue: - return b.process(&m.TaskQueueItem) - case interfaces.ModelIdTaskStat: - return b.process(&m.TaskStat) - case interfaces.ModelIdSpiderStat: - return b.process(&m.SpiderStat) - case interfaces.ModelIdDataSource: - return b.process(&m.DataSource) - case interfaces.ModelIdDataCollection: - return b.process(&m.DataCollection) - case interfaces.ModelIdResult: - return b.process(&m.Result) - case interfaces.ModelIdPassword: - return b.process(&m.Password) - case interfaces.ModelIdExtraValue: - return b.process(&m.ExtraValue) - case interfaces.ModelIdGit: - return b.process(&m.Git) - case interfaces.ModelIdRole: - return b.process(&m.Role) - case interfaces.ModelIdUserRole: - return b.process(&m.UserRole) - case interfaces.ModelIdPermission: - return b.process(&m.Permission) - case interfaces.ModelIdRolePermission: - return b.process(&m.RolePermission) - case interfaces.ModelIdEnvironment: - return b.process(&m.Environment) - case interfaces.ModelIdDependencySetting: - return b.process(&m.DependencySetting) - default: - return nil, errors.ErrorModelInvalidModelId - } -} - -func (b *ModelDelegateBinder) MustBind() (res interface{}) { - res, err := b.Bind() - if err != nil { - panic(err) - } - return res -} - -func (b *ModelDelegateBinder) BindWithDelegateMessage() (res interface{}, msg interfaces.GrpcModelDelegateMessage, err error) { - if err := json.Unmarshal(b.req.Data, b.msg); err != nil { - return nil, nil, err - } - res, err = b.Bind() - if err != nil { - return nil, nil, err - } - return res, b.msg, nil -} - -func (b *ModelDelegateBinder) process(d interface{}, fieldIds ...interfaces.ModelId) (res interface{}, err error) { - if err := json.Unmarshal(b.msg.GetData(), d); err != nil { - return nil, err - } - //return models.AssignFields(d, fieldIds...) // TODO: do we need to assign fields? - return d, nil -} - -func (b *ModelDelegateBinder) bindDelegateMessage() (err error) { - return json.Unmarshal(b.req.Data, b.msg) -} diff --git a/core/grpc/server/model_delegate_server.go b/core/grpc/server/model_delegate_server.go deleted file mode 100644 index 4f097d55..00000000 --- a/core/grpc/server/model_delegate_server.go +++ /dev/null @@ -1,67 +0,0 @@ -package server - -import ( - "context" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - grpc "github.com/crawlab-team/crawlab/grpc" -) - -type ModelDelegateServer struct { - grpc.UnimplementedModelDelegateServer -} - -// Do and perform an RPC action of constants.Delegate -func (svr ModelDelegateServer) Do(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - // bind message - obj, msg, err := NewModelDelegateBinder(req).BindWithDelegateMessage() - if err != nil { - return HandleError(err) - } - - // convert to model - doc, ok := obj.(interfaces.Model) - if !ok { - return HandleError(errors.ErrorModelInvalidType) - } - - // model delegate - d := delegate.NewModelDelegate(doc) - - // apply method - switch msg.GetMethod() { - case interfaces.ModelDelegateMethodAdd: - err = d.Add() - case interfaces.ModelDelegateMethodSave: - err = d.Save() - case interfaces.ModelDelegateMethodDelete: - err = d.Delete() - case interfaces.ModelDelegateMethodGetArtifact, interfaces.ModelDelegateMethodRefresh: - err = d.Refresh() - } - if err != nil { - return HandleError(err) - } - - // model - m := d.GetModel() - if msg.GetMethod() == interfaces.ModelDelegateMethodGetArtifact { - m, err = d.GetArtifact() - if err != nil { - return nil, err - } - } - - // json bytes - data, err := d.ToBytes(m) - if err != nil { - return nil, err - } - - return HandleSuccessWithData(data) -} - -func NewModelDelegateServer() (svr *ModelDelegateServer) { - return &ModelDelegateServer{} -} diff --git a/core/grpc/server/node_server.go b/core/grpc/server/node_server.go deleted file mode 100644 index ce1c9929..00000000 --- a/core/grpc/server/node_server.go +++ /dev/null @@ -1,199 +0,0 @@ -package server - -import ( - "context" - "encoding/json" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - nodeconfig "github.com/crawlab-team/crawlab/core/node/config" - "github.com/crawlab-team/crawlab/grpc" - "go.mongodb.org/mongo-driver/mongo" -) - -type NodeServer struct { - grpc.UnimplementedNodeServiceServer - - // dependencies - modelSvc service.ModelService - cfgSvc interfaces.NodeConfigService - - // internals - server interfaces.GrpcServer -} - -// Register from handler/worker to master -func (svr NodeServer) Register(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - // unmarshall data - var nodeInfo entity.NodeInfo - if req.Data != nil { - if err := json.Unmarshal(req.Data, &nodeInfo); err != nil { - return HandleError(err) - } - - if nodeInfo.IsMaster { - // error: cannot register master node - return HandleError(errors.ErrorGrpcNotAllowed) - } - } - - // node key - var nodeKey string - if req.NodeKey != "" { - nodeKey = req.NodeKey - } else { - nodeKey = nodeInfo.Key - } - if nodeKey == "" { - return HandleError(errors.ErrorModelMissingRequiredData) - } - - // find in db - node, err := svr.modelSvc.GetNodeByKey(nodeKey, nil) - if err == nil { - if node.IsMaster { - // error: cannot register master node - return HandleError(errors.ErrorGrpcNotAllowed) - } else { - // register existing - node.Status = constants.NodeStatusRegistered - node.Active = true - nodeD := delegate.NewModelNodeDelegate(node) - if err := nodeD.Save(); err != nil { - return HandleError(err) - } - var ok bool - node, ok = nodeD.GetModel().(*models.Node) - if !ok { - return HandleError(errors.ErrorGrpcInvalidType) - } - log.Infof("[NodeServer] updated worker[%s] in db. id: %s", nodeKey, nodeD.GetModel().GetId().Hex()) - } - } else if err == mongo.ErrNoDocuments { - // register new - node = &models.Node{ - Key: nodeKey, - Name: nodeInfo.Name, - Ip: nodeInfo.Ip, - Hostname: nodeInfo.Hostname, - Description: nodeInfo.Description, - MaxRunners: nodeInfo.MaxRunners, - Status: constants.NodeStatusRegistered, - Active: true, - Enabled: true, - } - if node.Name == "" { - node.Name = nodeKey - } - nodeD := delegate.NewModelDelegate(node) - if err := nodeD.Add(); err != nil { - return HandleError(err) - } - var ok bool - node, ok = nodeD.GetModel().(*models.Node) - if !ok { - return HandleError(errors.ErrorGrpcInvalidType) - } - log.Infof("[NodeServer] added worker[%s] in db. id: %s", nodeKey, nodeD.GetModel().GetId().Hex()) - } else { - // error - return HandleError(err) - } - - log.Infof("[NodeServer] master registered worker[%s]", req.GetNodeKey()) - - return HandleSuccessWithData(node) -} - -// SendHeartbeat from worker to master -func (svr NodeServer) SendHeartbeat(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - // find in db - node, err := svr.modelSvc.GetNodeByKey(req.NodeKey, nil) - if err != nil { - if err == mongo.ErrNoDocuments { - return HandleError(errors.ErrorNodeNotExists) - } - return HandleError(err) - } - - // validate status - if node.Status == constants.NodeStatusUnregistered { - return HandleError(errors.ErrorNodeUnregistered) - } - - // update status - nodeD := delegate.NewModelNodeDelegate(node) - if err := nodeD.UpdateStatusOnline(); err != nil { - return HandleError(err) - } - - return HandleSuccessWithData(node) -} - -// Ping from worker to master -func (svr NodeServer) Ping(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return HandleSuccess() -} - -func (svr NodeServer) Subscribe(request *grpc.Request, stream grpc.NodeService_SubscribeServer) (err error) { - log.Infof("[NodeServer] master received subscribe request from node[%s]", request.NodeKey) - - // finished channel - finished := make(chan bool) - - // set subscribe - svr.server.SetSubscribe("node:"+request.NodeKey, &entity.GrpcSubscribe{ - Stream: stream, - Finished: finished, - }) - ctx := stream.Context() - - log.Infof("[NodeServer] master subscribed node[%s]", request.NodeKey) - - // Keep this scope alive because once this scope exits - the stream is closed - for { - select { - case <-finished: - log.Infof("[NodeServer] closing stream for node[%s]", request.NodeKey) - return nil - case <-ctx.Done(): - log.Infof("[NodeServer] node[%s] has disconnected", request.NodeKey) - return nil - } - } -} - -func (svr NodeServer) Unsubscribe(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - sub, err := svr.server.GetSubscribe("node:" + req.NodeKey) - if err != nil { - return nil, errors.ErrorGrpcSubscribeNotExists - } - select { - case sub.GetFinished() <- true: - log.Infof("unsubscribed node[%s]", req.NodeKey) - default: - // Default case is to avoid blocking in case client has already unsubscribed - } - svr.server.DeleteSubscribe(req.NodeKey) - return &grpc.Response{ - Code: grpc.ResponseCode_OK, - Message: "unsubscribed successfully", - }, nil -} - -func NewNodeServer() (res *NodeServer, err error) { - // node server - svr := &NodeServer{} - svr.modelSvc, err = service.GetService() - if err != nil { - return nil, err - } - svr.cfgSvc = nodeconfig.GetNodeConfigService() - - return svr, nil -} diff --git a/core/i18n/service.go b/core/i18n/service.go deleted file mode 100644 index ac717475..00000000 --- a/core/i18n/service.go +++ /dev/null @@ -1,43 +0,0 @@ -package i18n - -import "github.com/crawlab-team/crawlab/core/interfaces" - -var translations []interfaces.Translation - -var _svc interfaces.I18nService - -type Service struct { -} - -func (svc *Service) AddTranslations(t []interfaces.Translation) { - translations = append(translations, t...) -} - -func (svc *Service) GetTranslations() (t []interfaces.Translation) { - return translations -} - -func GetI18nService(cfgPath string) (svc2 interfaces.I18nService, err error) { - if _svc != nil { - return _svc, nil - } - - _svc, err = NewI18nService() - if err != nil { - return nil, err - } - - return _svc, nil -} - -func ProvideGetI18nService(cfgPath string) func() (svc interfaces.I18nService, err error) { - return func() (svc interfaces.I18nService, err error) { - return GetI18nService(cfgPath) - } -} - -func NewI18nService() (svc2 interfaces.I18nService, err error) { - svc := &Service{} - - return svc, nil -} diff --git a/core/middlewares/auth.go b/core/middlewares/auth.go deleted file mode 100644 index 201695cf..00000000 --- a/core/middlewares/auth.go +++ /dev/null @@ -1,50 +0,0 @@ -package middlewares - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/user" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/gin-gonic/gin" - "github.com/spf13/viper" -) - -func AuthorizationMiddleware() gin.HandlerFunc { - userSvc, _ := user.GetUserService() - return func(c *gin.Context) { - // disable auth for test - if viper.GetBool("auth.disabled") { - modelSvc, err := service.GetService() - if err != nil { - utils.HandleErrorInternalServerError(c, err) - return - } - u, err := modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil) - if err != nil { - utils.HandleErrorInternalServerError(c, err) - return - } - c.Set(constants.UserContextKey, u) - c.Next() - return - } - - // token string - tokenStr := c.GetHeader("Authorization") - - // validate token - u, err := userSvc.CheckToken(tokenStr) - if err != nil { - // validation failed, return error response - utils.HandleErrorUnauthorized(c, errors.ErrorHttpUnauthorized) - return - } - - // set user in context - c.Set(constants.UserContextKey, u) - - // validation success - c.Next() - } -} diff --git a/core/middlewares/auth_v2.go b/core/middlewares/auth_v2.go index 61b8954b..d69d4a95 100644 --- a/core/middlewares/auth_v2.go +++ b/core/middlewares/auth_v2.go @@ -3,11 +3,13 @@ package middlewares import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/user" "github.com/crawlab-team/crawlab/core/utils" "github.com/gin-gonic/gin" "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" ) func AuthorizationMiddlewareV2() gin.HandlerFunc { @@ -15,12 +17,7 @@ func AuthorizationMiddlewareV2() gin.HandlerFunc { return func(c *gin.Context) { // disable auth for test if viper.GetBool("auth.disabled") { - modelSvc, err := service.GetService() - if err != nil { - utils.HandleErrorInternalServerError(c, err) - return - } - u, err := modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil) + u, err := service.NewModelServiceV2[models.UserV2]().GetOne(bson.M{"username": constants.DefaultAdminUsername}, nil) if err != nil { utils.HandleErrorInternalServerError(c, err) return diff --git a/core/middlewares/filer_auth.go b/core/middlewares/filer_auth.go deleted file mode 100644 index 1535f2b8..00000000 --- a/core/middlewares/filer_auth.go +++ /dev/null @@ -1,34 +0,0 @@ -package middlewares - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/gin-gonic/gin" - "github.com/spf13/viper" -) - -func FilerAuthorizationMiddleware() gin.HandlerFunc { - return func(c *gin.Context) { - // auth key - authKey := c.GetHeader("Authorization") - - // server auth key - svrAuthKey := viper.GetString("fs.filer.authKey") - - // skip to next if no server auth key is provided - if svrAuthKey == "" { - c.Next() - return - } - - // validate - if authKey != svrAuthKey { - // validation failed, return error response - utils.HandleErrorUnauthorized(c, errors.ErrorHttpUnauthorized) - return - } - - // validation success - c.Next() - } -} diff --git a/core/models/client/binder_basic.go b/core/models/client/binder_basic.go deleted file mode 100644 index 4d4b4716..00000000 --- a/core/models/client/binder_basic.go +++ /dev/null @@ -1,92 +0,0 @@ -package client - -import ( - "encoding/json" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" -) - -func NewBasicBinder(id interfaces.ModelId, res *grpc.Response) (b interfaces.GrpcModelBinder) { - return &BasicBinder{ - id: id, - res: res, - } -} - -type BasicBinder struct { - id interfaces.ModelId - res *grpc.Response -} - -func (b *BasicBinder) Bind() (res interfaces.Model, err error) { - m := models.NewModelMap() - - switch b.id { - case interfaces.ModelIdArtifact: - return b.Process(&m.Artifact) - case interfaces.ModelIdTag: - return b.Process(&m.Tag) - case interfaces.ModelIdNode: - return b.Process(&m.Node) - case interfaces.ModelIdProject: - return b.Process(&m.Project) - case interfaces.ModelIdSpider: - return b.Process(&m.Spider) - case interfaces.ModelIdTask: - return b.Process(&m.Task) - case interfaces.ModelIdJob: - return b.Process(&m.Job) - case interfaces.ModelIdSchedule: - return b.Process(&m.Schedule) - case interfaces.ModelIdUser: - return b.Process(&m.User) - case interfaces.ModelIdSetting: - return b.Process(&m.Setting) - case interfaces.ModelIdToken: - return b.Process(&m.Token) - case interfaces.ModelIdVariable: - return b.Process(&m.Variable) - case interfaces.ModelIdTaskQueue: - return b.Process(&m.TaskQueueItem) - case interfaces.ModelIdTaskStat: - return b.Process(&m.TaskStat) - case interfaces.ModelIdSpiderStat: - return b.Process(&m.SpiderStat) - case interfaces.ModelIdDataSource: - return b.Process(&m.DataSource) - case interfaces.ModelIdDataCollection: - return b.Process(&m.DataCollection) - case interfaces.ModelIdResult: - return b.Process(&m.Result) - case interfaces.ModelIdPassword: - return b.Process(&m.Password) - case interfaces.ModelIdExtraValue: - return b.Process(&m.ExtraValue) - case interfaces.ModelIdGit: - return b.Process(&m.Git) - case interfaces.ModelIdRole: - return b.Process(&m.Role) - case interfaces.ModelIdUserRole: - return b.Process(&m.UserRole) - case interfaces.ModelIdPermission: - return b.Process(&m.Permission) - case interfaces.ModelIdRolePermission: - return b.Process(&m.RolePermission) - case interfaces.ModelIdEnvironment: - return b.Process(&m.Environment) - case interfaces.ModelIdDependencySetting: - return b.Process(&m.DependencySetting) - default: - return nil, errors.ErrorModelInvalidModelId - } -} - -func (b *BasicBinder) Process(d interfaces.Model) (res interfaces.Model, err error) { - if err := json.Unmarshal(b.res.Data, d); err != nil { - return nil, trace.TraceError(err) - } - return d, nil -} diff --git a/core/models/client/binder_list.go b/core/models/client/binder_list.go deleted file mode 100644 index fe578804..00000000 --- a/core/models/client/binder_list.go +++ /dev/null @@ -1,100 +0,0 @@ -package client - -import ( - "encoding/json" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" -) - -func NewListBinder(id interfaces.ModelId, res *grpc.Response) (b interfaces.GrpcModelListBinder) { - return &ListBinder{ - id: id, - res: res, - } -} - -type ListBinder struct { - id interfaces.ModelId - res *grpc.Response -} - -func (b *ListBinder) Bind() (l interfaces.List, err error) { - m := models.NewModelListMap() - - switch b.id { - case interfaces.ModelIdArtifact: - return b.Process(&m.Artifacts) - case interfaces.ModelIdTag: - return b.Process(&m.Tags) - case interfaces.ModelIdNode: - return b.Process(&m.Nodes) - case interfaces.ModelIdProject: - return b.Process(&m.Projects) - case interfaces.ModelIdSpider: - return b.Process(&m.Spiders) - case interfaces.ModelIdTask: - return b.Process(&m.Tasks) - case interfaces.ModelIdJob: - return b.Process(&m.Jobs) - case interfaces.ModelIdSchedule: - return b.Process(&m.Schedules) - case interfaces.ModelIdUser: - return b.Process(&m.Users) - case interfaces.ModelIdSetting: - return b.Process(&m.Settings) - case interfaces.ModelIdToken: - return b.Process(&m.Tokens) - case interfaces.ModelIdVariable: - return b.Process(&m.Variables) - case interfaces.ModelIdTaskQueue: - return b.Process(&m.TaskQueueItems) - case interfaces.ModelIdTaskStat: - return b.Process(&m.TaskStats) - case interfaces.ModelIdSpiderStat: - return b.Process(&m.SpiderStats) - case interfaces.ModelIdDataSource: - return b.Process(&m.DataSources) - case interfaces.ModelIdDataCollection: - return b.Process(&m.DataCollections) - case interfaces.ModelIdResult: - return b.Process(&m.Results) - case interfaces.ModelIdPassword: - return b.Process(&m.Passwords) - case interfaces.ModelIdExtraValue: - return b.Process(&m.ExtraValues) - case interfaces.ModelIdGit: - return b.Process(&m.Gits) - case interfaces.ModelIdRole: - return b.Process(&m.Roles) - case interfaces.ModelIdUserRole: - return b.Process(&m.UserRoles) - case interfaces.ModelIdPermission: - return b.Process(&m.PermissionList) - case interfaces.ModelIdRolePermission: - return b.Process(&m.RolePermissionList) - case interfaces.ModelIdEnvironment: - return b.Process(&m.Environments) - case interfaces.ModelIdDependencySetting: - return b.Process(&m.DependencySettings) - default: - return l, errors.ErrorModelInvalidModelId - } -} - -func (b *ListBinder) MustBind() (res interface{}) { - res, err := b.Bind() - if err != nil { - panic(err) - } - return res -} - -func (b *ListBinder) Process(d interface{}) (l interfaces.List, err error) { - if err := json.Unmarshal(b.res.Data, d); err != nil { - return l, trace.TraceError(err) - } - return d.(interfaces.List), nil -} diff --git a/core/models/client/model_base_service.go b/core/models/client/model_base_service.go deleted file mode 100644 index 6ee6b9dd..00000000 --- a/core/models/client/model_base_service.go +++ /dev/null @@ -1,251 +0,0 @@ -package client - -import ( - "encoding/json" - "github.com/apex/log" - "github.com/cenkalti/backoff/v4" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - errors2 "github.com/pkg/errors" - "github.com/spf13/viper" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "time" -) - -type BaseServiceDelegate struct { - // settings - cfgPath string - - // internals - id interfaces.ModelId - c interfaces.GrpcClient -} - -func (d *BaseServiceDelegate) GetModelId() (id interfaces.ModelId) { - return d.id -} - -func (d *BaseServiceDelegate) SetModelId(id interfaces.ModelId) { - d.id = id -} - -func (d *BaseServiceDelegate) GetConfigPath() (path string) { - return d.cfgPath -} - -func (d *BaseServiceDelegate) SetConfigPath(path string) { - d.cfgPath = path -} - -func (d *BaseServiceDelegate) GetById(id primitive.ObjectID) (doc interfaces.Model, err error) { - log.Debugf("[BaseServiceDelegate] get by id[%s]", id.Hex()) - ctx, cancel := d.c.Context() - defer cancel() - req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id}) - c := d.getModelBaseServiceClient() - if c == nil { - return nil, trace.TraceError(errors.ErrorModelNilPointer) - } - log.Debugf("[BaseServiceDelegate] get by id[%s] req: %v", id.Hex(), req) - res, err := c.GetById(ctx, req) - if err != nil { - return nil, trace.TraceError(err) - } - log.Debugf("[BaseServiceDelegate] get by id[%s] res: %v", id.Hex(), res) - return NewBasicBinder(d.id, res).Bind() -} - -func (d *BaseServiceDelegate) Get(query bson.M, opts *mongo.FindOptions) (doc interfaces.Model, err error) { - ctx, cancel := d.c.Context() - defer cancel() - req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, FindOptions: opts}) - res, err := d.getModelBaseServiceClient().Get(ctx, req) - if err != nil { - return nil, err - } - return NewBasicBinder(d.id, res).Bind() -} - -func (d *BaseServiceDelegate) GetList(query bson.M, opts *mongo.FindOptions) (l interfaces.List, err error) { - ctx, cancel := d.c.Context() - defer cancel() - req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, FindOptions: opts}) - res, err := d.getModelBaseServiceClient().GetList(ctx, req) - if err != nil { - return l, err - } - return NewListBinder(d.id, res).Bind() -} - -func (d *BaseServiceDelegate) DeleteById(id primitive.ObjectID, args ...interface{}) (err error) { - u := utils.GetUserFromArgs(args...) - ctx, cancel := d.c.Context() - defer cancel() - req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id, User: u}) - _, err = d.getModelBaseServiceClient().DeleteById(ctx, req) - if err != nil { - return err - } - return nil -} - -func (d *BaseServiceDelegate) Delete(query bson.M, args ...interface{}) (err error) { - u := utils.GetUserFromArgs(args...) - ctx, cancel := d.c.Context() - defer cancel() - req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u}) - _, err = d.getModelBaseServiceClient().Delete(ctx, req) - if err != nil { - return err - } - return nil -} - -func (d *BaseServiceDelegate) DeleteList(query bson.M, args ...interface{}) (err error) { - u := utils.GetUserFromArgs(args...) - ctx, cancel := d.c.Context() - defer cancel() - req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u}) - _, err = d.getModelBaseServiceClient().DeleteList(ctx, req) - if err != nil { - return err - } - return nil -} - -func (d *BaseServiceDelegate) ForceDeleteList(query bson.M, args ...interface{}) (err error) { - u := utils.GetUserFromArgs(args...) - ctx, cancel := d.c.Context() - defer cancel() - req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u}) - _, err = d.getModelBaseServiceClient().ForceDeleteList(ctx, req) - if err != nil { - return err - } - return nil -} - -func (d *BaseServiceDelegate) UpdateById(id primitive.ObjectID, update bson.M, args ...interface{}) (err error) { - u := utils.GetUserFromArgs(args...) - ctx, cancel := d.c.Context() - defer cancel() - req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id, Update: update, User: u}) - _, err = d.getModelBaseServiceClient().UpdateById(ctx, req) - if err != nil { - return err - } - return nil -} - -func (d *BaseServiceDelegate) Update(query bson.M, update bson.M, fields []string, args ...interface{}) (err error) { - u := utils.GetUserFromArgs(args...) - ctx, cancel := d.c.Context() - defer cancel() - req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, Update: update, Fields: fields, User: u}) - _, err = d.getModelBaseServiceClient().Update(ctx, req) - if err != nil { - return err - } - return nil -} - -func (d *BaseServiceDelegate) UpdateDoc(query bson.M, doc interfaces.Model, fields []string, args ...interface{}) (err error) { - u := utils.GetUserFromArgs(args...) - ctx, cancel := d.c.Context() - defer cancel() - req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, Doc: doc, Fields: fields, User: u}) - _, err = d.getModelBaseServiceClient().UpdateDoc(ctx, req) - if err != nil { - return err - } - return nil -} - -func (d *BaseServiceDelegate) Insert(u interfaces.User, docs ...interface{}) (err error) { - ctx, cancel := d.c.Context() - defer cancel() - req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Docs: docs, User: u}) - _, err = d.getModelBaseServiceClient().Insert(ctx, req) - if err != nil { - return err - } - return nil -} - -func (d *BaseServiceDelegate) Count(query bson.M) (total int, err error) { - ctx, cancel := d.c.Context() - defer cancel() - req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query}) - res, err := d.getModelBaseServiceClient().Count(ctx, req) - if err != nil { - return total, err - } - if err := json.Unmarshal(res.Data, &total); err != nil { - return total, err - } - return total, nil -} - -func (d *BaseServiceDelegate) newRequest(params interfaces.GrpcBaseServiceParams) (req *grpc.Request, err error) { - return d.c.NewModelBaseServiceRequest(d.id, params) -} - -func (d *BaseServiceDelegate) mustNewRequest(params *entity.GrpcBaseServiceParams) (req *grpc.Request) { - req, err := d.newRequest(params) - if err != nil { - panic(err) - } - return req -} - -func (d *BaseServiceDelegate) getModelBaseServiceClient() (c grpc.ModelBaseServiceClient) { - if err := backoff.Retry(func() (err error) { - c = d.c.GetModelBaseServiceClient() - if c == nil { - err = errors2.New("unable to get model base service client") - log.Debugf("[BaseServiceDelegate] err: %v", err) - return err - } - return nil - }, backoff.NewConstantBackOff(1*time.Second)); err != nil { - trace.PrintError(err) - } - return c -} - -func NewBaseServiceDelegate(opts ...ModelBaseServiceDelegateOption) (svc2 interfaces.GrpcClientModelBaseService, err error) { - // base service - svc := &BaseServiceDelegate{} - - // apply options - for _, opt := range opts { - opt(svc) - } - - // config path - if viper.GetString("config.path") != "" { - svc.cfgPath = viper.GetString("config.path") - } - - // dependency injection - if err := container.GetContainer().Invoke(func(client interfaces.GrpcClient) { - svc.c = client - }); err != nil { - return nil, err - } - - return svc, nil -} - -func ProvideBaseServiceDelegate(id interfaces.ModelId) func() (svc interfaces.GrpcClientModelBaseService, err error) { - return func() (svc interfaces.GrpcClientModelBaseService, err error) { - return NewBaseServiceDelegate(WithBaseServiceModelId(id)) - } -} diff --git a/core/models/client/model_delegate.go b/core/models/client/model_delegate.go deleted file mode 100644 index d59524ca..00000000 --- a/core/models/client/model_delegate.go +++ /dev/null @@ -1,329 +0,0 @@ -package client - -import ( - "encoding/json" - config2 "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/grpc/client" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/utils" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "github.com/spf13/viper" -) - -func NewModelDelegate(doc interfaces.Model, opts ...ModelDelegateOption) interfaces.GrpcClientModelDelegate { - switch doc.(type) { - case *models.Artifact: - return newModelDelegate(interfaces.ModelIdArtifact, doc, opts...) - case *models.Tag: - return newModelDelegate(interfaces.ModelIdTag, doc, opts...) - case *models.Node: - return newModelDelegate(interfaces.ModelIdNode, doc, opts...) - case *models.Project: - return newModelDelegate(interfaces.ModelIdProject, doc, opts...) - case *models.Spider: - return newModelDelegate(interfaces.ModelIdSpider, doc, opts...) - case *models.Task: - return newModelDelegate(interfaces.ModelIdTask, doc, opts...) - case *models.Job: - return newModelDelegate(interfaces.ModelIdJob, doc, opts...) - case *models.Schedule: - return newModelDelegate(interfaces.ModelIdSchedule, doc, opts...) - case *models.User: - return newModelDelegate(interfaces.ModelIdUser, doc, opts...) - case *models.Setting: - return newModelDelegate(interfaces.ModelIdSetting, doc, opts...) - case *models.Token: - return newModelDelegate(interfaces.ModelIdToken, doc, opts...) - case *models.Variable: - return newModelDelegate(interfaces.ModelIdVariable, doc, opts...) - case *models.TaskQueueItem: - return newModelDelegate(interfaces.ModelIdTaskQueue, doc, opts...) - case *models.TaskStat: - return newModelDelegate(interfaces.ModelIdTaskStat, doc, opts...) - case *models.SpiderStat: - return newModelDelegate(interfaces.ModelIdSpiderStat, doc, opts...) - case *models.DataSource: - return newModelDelegate(interfaces.ModelIdDataSource, doc, opts...) - case *models.DataCollection: - return newModelDelegate(interfaces.ModelIdDataCollection, doc, opts...) - case *models.Result: - return newModelDelegate(interfaces.ModelIdResult, doc, opts...) - case *models.Password: - return newModelDelegate(interfaces.ModelIdPassword, doc, opts...) - case *models.ExtraValue: - return newModelDelegate(interfaces.ModelIdExtraValue, doc, opts...) - case *models.Git: - return newModelDelegate(interfaces.ModelIdGit, doc, opts...) - case *models.UserRole: - return newModelDelegate(interfaces.ModelIdUserRole, doc, opts...) - case *models.Permission: - return newModelDelegate(interfaces.ModelIdPermission, doc, opts...) - case *models.RolePermission: - return newModelDelegate(interfaces.ModelIdRolePermission, doc, opts...) - case *models.Environment: - return newModelDelegate(interfaces.ModelIdEnvironment, doc, opts...) - case *models.DependencySetting: - return newModelDelegate(interfaces.ModelIdDependencySetting, doc, opts...) - default: - _ = trace.TraceError(errors.ErrorModelInvalidType) - return nil - } -} - -func newModelDelegate(id interfaces.ModelId, doc interfaces.Model, opts ...ModelDelegateOption) interfaces.GrpcClientModelDelegate { - var err error - - // collection name - colName := models.GetModelColName(id) - - // model delegate - d := &ModelDelegate{ - id: id, - colName: colName, - doc: doc, - cfgPath: config2.GetConfigPath(), - a: &models.Artifact{ - Col: colName, - }, - } - - // config path - if viper.GetString("config.path") != "" { - d.cfgPath = viper.GetString("config.path") - } - - // apply options - for _, opt := range opts { - opt(d) - } - - // grpc client - d.c, err = client.GetClient() - if err != nil { - trace.PrintError(errors.ErrorModelInvalidType) - return nil - } - if !d.c.IsStarted() { - if err := d.c.Start(); err != nil { - trace.PrintError(err) - return nil - } - } else if d.c.IsClosed() { - if err := d.c.Restart(); err != nil { - trace.PrintError(err) - return nil - } - } - - return d -} - -type ModelDelegate struct { - // settings - cfgPath string - - // internals - id interfaces.ModelId - colName string - c interfaces.GrpcClient - doc interfaces.Model - a interfaces.ModelArtifact -} - -func (d *ModelDelegate) Add() (err error) { - return d.do(interfaces.ModelDelegateMethodAdd) -} - -func (d *ModelDelegate) Save() (err error) { - return d.do(interfaces.ModelDelegateMethodSave) -} - -func (d *ModelDelegate) Delete() (err error) { - return d.do(interfaces.ModelDelegateMethodDelete) -} - -func (d *ModelDelegate) GetArtifact() (res interfaces.ModelArtifact, err error) { - if err := d.do(interfaces.ModelDelegateMethodGetArtifact); err != nil { - return nil, err - } - return d.a, nil -} - -func (d *ModelDelegate) GetModel() (res interfaces.Model) { - return d.doc -} - -func (d *ModelDelegate) Refresh() (err error) { - return d.refresh() -} - -func (d *ModelDelegate) GetConfigPath() (path string) { - return d.cfgPath -} - -func (d *ModelDelegate) SetConfigPath(path string) { - d.cfgPath = path -} - -func (d *ModelDelegate) Close() (err error) { - return d.c.Stop() -} - -func (d *ModelDelegate) ToBytes(m interface{}) (bytes []byte, err error) { - if m != nil { - return utils.JsonToBytes(m) - } - return json.Marshal(d.doc) -} - -func (d *ModelDelegate) do(method interfaces.ModelDelegateMethod) (err error) { - switch method { - case interfaces.ModelDelegateMethodAdd: - err = d.add() - case interfaces.ModelDelegateMethodSave: - err = d.save() - case interfaces.ModelDelegateMethodDelete: - err = d.delete() - case interfaces.ModelDelegateMethodGetArtifact, interfaces.ModelDelegateMethodRefresh: - return d.refresh() - default: - return trace.TraceError(errors.ErrorModelInvalidType) - } - - if err != nil { - return err - } - - return nil -} - -func (d *ModelDelegate) add() (err error) { - ctx, cancel := d.c.Context() - defer cancel() - method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodAdd) - res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{ - ModelId: d.id, - Method: method, - Data: d.mustGetData(), - })) - if err != nil { - return trace.TraceError(err) - } - if err := d.deserialize(res, method); err != nil { - return err - } - return d.refreshArtifact() -} - -func (d *ModelDelegate) save() (err error) { - ctx, cancel := d.c.Context() - defer cancel() - method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodSave) - res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{ - ModelId: d.id, - Method: method, - Data: d.mustGetData(), - })) - if err != nil { - return trace.TraceError(err) - } - if err := d.deserialize(res, method); err != nil { - return err - } - return d.refreshArtifact() -} - -func (d *ModelDelegate) delete() (err error) { - ctx, cancel := d.c.Context() - defer cancel() - method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodDelete) - res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{ - ModelId: d.id, - Method: method, - Data: d.mustGetData(), - })) - if err != nil { - return trace.TraceError(err) - } - if err := d.deserialize(res, method); err != nil { - return err - } - return d.refreshArtifact() -} - -func (d *ModelDelegate) refresh() (err error) { - ctx, cancel := d.c.Context() - defer cancel() - method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodRefresh) - res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{ - ModelId: d.id, - Method: method, - Data: d.mustGetData(), - })) - if err != nil { - return trace.TraceError(err) - } - if err := d.deserialize(res, method); err != nil { - return err - } - return nil -} - -func (d *ModelDelegate) refreshArtifact() (err error) { - _, err = d.getArtifact() - return err -} - -func (d *ModelDelegate) getArtifact() (res2 interfaces.ModelArtifact, err error) { - ctx, cancel := d.c.Context() - defer cancel() - method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodGetArtifact) - res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{ - ModelId: d.id, - Method: method, - Data: d.mustGetData(), - })) - if err != nil { - return nil, err - } - if err := d.deserialize(res, method); err != nil { - return nil, err - } - return d.a, nil -} - -func (d *ModelDelegate) mustGetData() (data []byte) { - data, err := d.getData() - if err != nil { - panic(err) - } - return data -} - -func (d *ModelDelegate) getData() (data []byte, err error) { - return json.Marshal(d.doc) -} - -func (d *ModelDelegate) deserialize(res *grpc.Response, method interfaces.ModelDelegateMethod) (err error) { - if method == interfaces.ModelDelegateMethodGetArtifact { - res, err := NewBasicBinder(interfaces.ModelIdArtifact, res).Bind() - if err != nil { - return err - } - a, ok := res.(interfaces.ModelArtifact) - if !ok { - return trace.TraceError(errors.ErrorModelInvalidType) - } - d.a = a - } else { - d.doc, err = NewBasicBinder(d.id, res).Bind() - if err != nil { - return err - } - } - return nil -} diff --git a/core/models/client/model_environment_service.go b/core/models/client/model_environment_service.go deleted file mode 100644 index 6de3fbd9..00000000 --- a/core/models/client/model_environment_service.go +++ /dev/null @@ -1,70 +0,0 @@ -package client - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type EnvironmentServiceDelegate struct { - interfaces.GrpcClientModelBaseService -} - -func (svc *EnvironmentServiceDelegate) GetEnvironmentById(id primitive.ObjectID) (e interfaces.Environment, err error) { - res, err := svc.GetById(id) - if err != nil { - return nil, err - } - s, ok := res.(interfaces.Environment) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return s, nil -} - -func (svc *EnvironmentServiceDelegate) GetEnvironment(query bson.M, opts *mongo.FindOptions) (e interfaces.Environment, err error) { - res, err := svc.Get(query, opts) - if err != nil { - return nil, err - } - s, ok := res.(interfaces.Environment) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return s, nil -} - -func (svc *EnvironmentServiceDelegate) GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Environment, err error) { - list, err := svc.GetList(query, opts) - if err != nil { - return nil, err - } - for _, item := range list.GetModels() { - s, ok := item.(interfaces.Environment) - if !ok { - return nil, errors.ErrorModelInvalidType - } - res = append(res, s) - } - return res, nil -} - -func NewEnvironmentServiceDelegate() (svc2 interfaces.GrpcClientModelEnvironmentService, err error) { - var opts []ModelBaseServiceDelegateOption - - // apply options - opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdEnvironment)) - - // base service - baseSvc, err := NewBaseServiceDelegate(opts...) - if err != nil { - return nil, err - } - - // service - svc := &EnvironmentServiceDelegate{baseSvc} - - return svc, nil -} diff --git a/core/models/client/model_node_delegate.go b/core/models/client/model_node_delegate.go deleted file mode 100644 index 4d7d8eb4..00000000 --- a/core/models/client/model_node_delegate.go +++ /dev/null @@ -1,37 +0,0 @@ -package client - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "time" -) - -type ModelNodeDelegate struct { - n interfaces.Node - interfaces.GrpcClientModelDelegate -} - -func (d *ModelNodeDelegate) UpdateStatus(active bool, activeTs *time.Time, status string) (err error) { - d.n.SetActive(active) - if activeTs != nil { - d.n.SetActiveTs(*activeTs) - } - d.n.SetStatus(status) - return d.Save() -} - -func (d *ModelNodeDelegate) UpdateStatusOnline() (err error) { - now := time.Now() - return d.UpdateStatus(true, &now, constants.NodeStatusOnline) -} - -func (d *ModelNodeDelegate) UpdateStatusOffline() (err error) { - return d.UpdateStatus(false, nil, constants.NodeStatusOffline) -} - -func NewModelNodeDelegate(n interfaces.Node) interfaces.ModelNodeDelegate { - return &ModelNodeDelegate{ - n: n, - GrpcClientModelDelegate: NewModelDelegate(n), - } -} diff --git a/core/models/client/model_node_service.go b/core/models/client/model_node_service.go deleted file mode 100644 index 2518b1ec..00000000 --- a/core/models/client/model_node_service.go +++ /dev/null @@ -1,74 +0,0 @@ -package client - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type NodeServiceDelegate struct { - interfaces.GrpcClientModelBaseService -} - -func (svc *NodeServiceDelegate) GetNodeById(id primitive.ObjectID) (n interfaces.Node, err error) { - res, err := svc.GetById(id) - if err != nil { - return nil, err - } - s, ok := res.(interfaces.Node) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return s, nil -} - -func (svc *NodeServiceDelegate) GetNode(query bson.M, opts *mongo.FindOptions) (n interfaces.Node, err error) { - res, err := svc.Get(query, opts) - if err != nil { - return nil, err - } - s, ok := res.(interfaces.Node) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return s, nil -} - -func (svc *NodeServiceDelegate) GetNodeByKey(key string) (n interfaces.Node, err error) { - return svc.GetNode(bson.M{"key": key}, nil) -} - -func (svc *NodeServiceDelegate) GetNodeList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Node, err error) { - list, err := svc.GetList(query, opts) - if err != nil { - return nil, err - } - for _, item := range list.GetModels() { - s, ok := item.(interfaces.Node) - if !ok { - return nil, errors.ErrorModelInvalidType - } - res = append(res, s) - } - return res, nil -} - -func NewNodeServiceDelegate() (svc2 interfaces.GrpcClientModelNodeService, err error) { - var opts []ModelBaseServiceDelegateOption - - // apply options - opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdNode)) - - // base service - baseSvc, err := NewBaseServiceDelegate(opts...) - if err != nil { - return nil, err - } - - // service - svc := &NodeServiceDelegate{baseSvc} - - return svc, nil -} diff --git a/core/models/client/model_service.go b/core/models/client/model_service.go deleted file mode 100644 index fde84d56..00000000 --- a/core/models/client/model_service.go +++ /dev/null @@ -1,48 +0,0 @@ -package client - -import ( - config2 "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" -) - -type ServiceDelegate struct { - // settings - cfgPath string - - // internals - c interfaces.GrpcClient -} - -func (d *ServiceDelegate) GetConfigPath() string { - return d.cfgPath -} - -func (d *ServiceDelegate) SetConfigPath(path string) { - d.cfgPath = path -} - -func (d *ServiceDelegate) NewBaseServiceDelegate(id interfaces.ModelId) (svc interfaces.GrpcClientModelBaseService, err error) { - var opts []ModelBaseServiceDelegateOption - opts = append(opts, WithBaseServiceModelId(id)) - if d.cfgPath != "" { - opts = append(opts, WithBaseServiceConfigPath(d.cfgPath)) - } - return NewBaseServiceDelegate(opts...) -} - -func NewServiceDelegate() (svc2 interfaces.GrpcClientModelService, err error) { - // service delegate - svc := &ServiceDelegate{ - cfgPath: config2.GetConfigPath(), - } - - // dependency injection - if err := container.GetContainer().Invoke(func(client interfaces.GrpcClient) { - svc.c = client - }); err != nil { - return nil, err - } - - return svc, nil -} diff --git a/core/models/client/model_spider_service.go b/core/models/client/model_spider_service.go deleted file mode 100644 index 164baee9..00000000 --- a/core/models/client/model_spider_service.go +++ /dev/null @@ -1,70 +0,0 @@ -package client - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type SpiderServiceDelegate struct { - interfaces.GrpcClientModelBaseService -} - -func (svc *SpiderServiceDelegate) GetSpiderById(id primitive.ObjectID) (s interfaces.Spider, err error) { - res, err := svc.GetById(id) - if err != nil { - return nil, err - } - s, ok := res.(interfaces.Spider) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return s, nil -} - -func (svc *SpiderServiceDelegate) GetSpider(query bson.M, opts *mongo.FindOptions) (s interfaces.Spider, err error) { - res, err := svc.Get(query, opts) - if err != nil { - return nil, err - } - s, ok := res.(interfaces.Spider) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return s, nil -} - -func (svc *SpiderServiceDelegate) GetSpiderList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Spider, err error) { - list, err := svc.GetList(query, opts) - if err != nil { - return nil, err - } - for _, item := range list.GetModels() { - s, ok := item.(interfaces.Spider) - if !ok { - return nil, errors.ErrorModelInvalidType - } - res = append(res, s) - } - return res, nil -} - -func NewSpiderServiceDelegate() (svc2 interfaces.GrpcClientModelSpiderService, err error) { - var opts []ModelBaseServiceDelegateOption - - // apply options - opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdSpider)) - - // base service - baseSvc, err := NewBaseServiceDelegate(opts...) - if err != nil { - return nil, err - } - - // service - svc := &SpiderServiceDelegate{baseSvc} - - return svc, nil -} diff --git a/core/models/client/model_task_service.go b/core/models/client/model_task_service.go deleted file mode 100644 index 16a9612b..00000000 --- a/core/models/client/model_task_service.go +++ /dev/null @@ -1,70 +0,0 @@ -package client - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type TaskServiceDelegate struct { - interfaces.GrpcClientModelBaseService -} - -func (svc *TaskServiceDelegate) GetTaskById(id primitive.ObjectID) (t interfaces.Task, err error) { - res, err := svc.GetById(id) - if err != nil { - return nil, err - } - s, ok := res.(interfaces.Task) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return s, nil -} - -func (svc *TaskServiceDelegate) GetTask(query bson.M, opts *mongo.FindOptions) (t interfaces.Task, err error) { - res, err := svc.Get(query, opts) - if err != nil { - return nil, err - } - s, ok := res.(interfaces.Task) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return s, nil -} - -func (svc *TaskServiceDelegate) GetTaskList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Task, err error) { - list, err := svc.GetList(query, opts) - if err != nil { - return nil, err - } - for _, item := range list.GetModels() { - s, ok := item.(interfaces.Task) - if !ok { - return nil, errors.ErrorModelInvalidType - } - res = append(res, s) - } - return res, nil -} - -func NewTaskServiceDelegate() (svc2 interfaces.GrpcClientModelTaskService, err error) { - var opts []ModelBaseServiceDelegateOption - - // apply options - opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdTask)) - - // base service - baseSvc, err := NewBaseServiceDelegate(opts...) - if err != nil { - return nil, err - } - - // service - svc := &TaskServiceDelegate{baseSvc} - - return svc, nil -} diff --git a/core/models/client/model_task_stat_service.go b/core/models/client/model_task_stat_service.go deleted file mode 100644 index 1714772b..00000000 --- a/core/models/client/model_task_stat_service.go +++ /dev/null @@ -1,70 +0,0 @@ -package client - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type TaskStatServiceDelegate struct { - interfaces.GrpcClientModelBaseService -} - -func (svc *TaskStatServiceDelegate) GetTaskStatById(id primitive.ObjectID) (t interfaces.TaskStat, err error) { - res, err := svc.GetById(id) - if err != nil { - return nil, err - } - s, ok := res.(interfaces.TaskStat) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return s, nil -} - -func (svc *TaskStatServiceDelegate) GetTaskStat(query bson.M, opts *mongo.FindOptions) (t interfaces.TaskStat, err error) { - res, err := svc.Get(query, opts) - if err != nil { - return nil, err - } - s, ok := res.(interfaces.TaskStat) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return s, nil -} - -func (svc *TaskStatServiceDelegate) GetTaskStatList(query bson.M, opts *mongo.FindOptions) (res []interfaces.TaskStat, err error) { - list, err := svc.GetList(query, opts) - if err != nil { - return nil, err - } - for _, item := range list.GetModels() { - s, ok := item.(interfaces.TaskStat) - if !ok { - return nil, errors.ErrorModelInvalidType - } - res = append(res, s) - } - return res, nil -} - -func NewTaskStatServiceDelegate() (svc2 interfaces.GrpcClientModelTaskStatService, err error) { - var opts []ModelBaseServiceDelegateOption - - // apply options - opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdTaskStat)) - - // base service - baseSvc, err := NewBaseServiceDelegate(opts...) - if err != nil { - return nil, err - } - - // service - svc := &TaskStatServiceDelegate{baseSvc} - - return svc, nil -} diff --git a/core/models/client/options.go b/core/models/client/options.go deleted file mode 100644 index e0966802..00000000 --- a/core/models/client/options.go +++ /dev/null @@ -1,33 +0,0 @@ -package client - -import "github.com/crawlab-team/crawlab/core/interfaces" - -type ModelDelegateOption func(delegate interfaces.GrpcClientModelDelegate) - -func WithDelegateConfigPath(path string) ModelDelegateOption { - return func(d interfaces.GrpcClientModelDelegate) { - d.SetConfigPath(path) - } -} - -type ModelServiceDelegateOption func(delegate interfaces.GrpcClientModelService) - -func WithServiceConfigPath(path string) ModelServiceDelegateOption { - return func(d interfaces.GrpcClientModelService) { - d.SetConfigPath(path) - } -} - -type ModelBaseServiceDelegateOption func(delegate interfaces.GrpcClientModelBaseService) - -func WithBaseServiceModelId(id interfaces.ModelId) ModelBaseServiceDelegateOption { - return func(d interfaces.GrpcClientModelBaseService) { - d.SetModelId(id) - } -} - -func WithBaseServiceConfigPath(path string) ModelBaseServiceDelegateOption { - return func(d interfaces.GrpcClientModelBaseService) { - d.SetConfigPath(path) - } -} diff --git a/core/models/config_spider/common.go b/core/models/config_spider/common.go deleted file mode 100644 index 8ad3ad15..00000000 --- a/core/models/config_spider/common.go +++ /dev/null @@ -1,24 +0,0 @@ -package config_spider - -import "github.com/crawlab-team/crawlab/core/entity" - -func GetAllFields(data entity.ConfigSpiderData) []entity.Field { - var fields []entity.Field - for _, stage := range data.Stages { - fields = append(fields, stage.Fields...) - } - return fields -} - -func GetStartStageName(data entity.ConfigSpiderData) string { - // 如果 start_stage 设置了且在 stages 里，则返回 - if data.StartStage != "" { - return data.StartStage - } - - // 否则返回第一个 stage - for _, stage := range data.Stages { - return stage.Name - } - return "" -} diff --git a/core/models/config_spider/scrapy.go b/core/models/config_spider/scrapy.go deleted file mode 100644 index 8dab8461..00000000 --- a/core/models/config_spider/scrapy.go +++ /dev/null @@ -1,263 +0,0 @@ -package config_spider - -//import ( -// "errors" -// "fmt" -// "github.com/crawlab-team/crawlab/core/constants" -// "github.com/crawlab-team/crawlab/core/entity" -// "github.com/crawlab-team/crawlab/core/models" -// "github.com/crawlab-team/crawlab/core/utils" -// "path/filepath" -//) -// -//type ScrapyGenerator struct { -// Spider model.Spider -// ConfigData entity.ConfigSpiderData -//} -// -//// 生成爬虫文件 -//func (g ScrapyGenerator) Generate() error { -// // 生成 items.py -// if err := g.ProcessItems(); err != nil { -// return err -// } -// -// // 生成 spider.py -// if err := g.ProcessSpider(); err != nil { -// return err -// } -// return nil -//} -// -//// 生成 items.py -//func (g ScrapyGenerator) ProcessItems() error { -// // 待处理文件名 -// src := g.Spider.Src -// filePath := filepath.Join(src, "config_spider", "items.py") -// -// // 获取所有字段 -// fields := g.GetAllFields() -// -// // 字段名列表（包含默认字段名） -// fieldNames := []string{ -// "_id", -// "task_id", -// "ts", -// } -// -// // 加入字段 -// for _, field := range fields { -// fieldNames = append(fieldNames, field.Name) -// } -// -// // 将字段名转化为python代码 -// str := "" -// for _, fieldName := range fieldNames { -// line := g.PadCode(fmt.Sprintf("%s = scrapy.Field()", fieldName), 1) -// str += line -// } -// -// // 将占位符替换为代码 -// if err := utils.SetFileVariable(filePath, constants.AnchorItems, str); err != nil { -// return err -// } -// -// return nil -//} -// -//// 生成 spider.py -//func (g ScrapyGenerator) ProcessSpider() error { -// // 待处理文件名 -// src := g.Spider.Src -// filePath := filepath.Join(src, "config_spider", "spiders", "spider.py") -// -// // 替换 start_stage -// if err := utils.SetFileVariable(filePath, constants.AnchorStartStage, "parse_"+GetStartStageName(g.ConfigData)); err != nil { -// return err -// } -// -// // 替换 start_url -// if err := utils.SetFileVariable(filePath, constants.AnchorStartUrl, g.ConfigData.StartUrl); err != nil { -// return err -// } -// -// // 替换 parsers -// strParser := "" -// for _, stage := range g.ConfigData.Stages { -// stageName := stage.Name -// stageStr := g.GetParserString(stageName, stage) -// strParser += stageStr -// } -// if err := utils.SetFileVariable(filePath, constants.AnchorParsers, strParser); err != nil { -// return err -// } -// -// return nil -//} -// -//func (g ScrapyGenerator) GetParserString(stageName string, stage entity.Stage) string { -// // 构造函数定义行 -// strDef := g.PadCode(fmt.Sprintf("def parse_%s(self, response):", stageName), 1) -// -// strParse := "" -// if stage.IsList { -// // 列表逻辑 -// strParse = g.GetListParserString(stageName, stage) -// } else { -// // 非列表逻辑 -// strParse = g.GetNonListParserString(stageName, stage) -// } -// -// // 构造 -// str := fmt.Sprintf(`%s%s`, strDef, strParse) -// -// return str -//} -// -//func (g ScrapyGenerator) PadCode(str string, num int) string { -// res := "" -// for i := 0; i < num; i++ { -// res += " " -// } -// res += str -// res += "\n" -// return res -//} -// -//func (g ScrapyGenerator) GetNonListParserString(stageName string, stage entity.Stage) string { -// str := "" -// -// // 获取或构造item -// str += g.PadCode("item = Item() if response.meta.get('item') is None else response.meta.get('item')", 2) -// -// // 遍历字段列表 -// for _, f := range stage.Fields { -// line := fmt.Sprintf(`item['%s'] = response.%s.extract_first()`, f.Name, g.GetExtractStringFromField(f)) -// line = g.PadCode(line, 2) -// str += line -// } -// -// // next stage 字段 -// if f, err := g.GetNextStageField(stage); err == nil { -// // 如果找到 next stage 字段，进行下一个回调 -// str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url="get_real_url(response, item['%s'])", callback=self.parse_%s, meta={'item': item})`, f.Name, f.NextStage), 2) -// } else { -// // 如果没找到 next stage 字段，返回 item -// str += g.PadCode(fmt.Sprintf(`yield item`), 2) -// } -// -// // 加入末尾换行 -// str += g.PadCode("", 0) -// -// return str -//} -// -//func (g ScrapyGenerator) GetListParserString(stageName string, stage entity.Stage) string { -// str := "" -// -// // 获取前一个 stage 的 item -// str += g.PadCode(`prev_item = response.meta.get('item')`, 2) -// -// // for 循环遍历列表 -// str += g.PadCode(fmt.Sprintf(`for elem in response.%s:`, g.GetListString(stage)), 2) -// -// // 构造item -// str += g.PadCode(`item = Item()`, 3) -// -// // 遍历字段列表 -// for _, f := range stage.Fields { -// line := fmt.Sprintf(`item['%s'] = elem.%s.extract_first()`, f.Name, g.GetExtractStringFromField(f)) -// line = g.PadCode(line, 3) -// str += line -// } -// -// // 把前一个 stage 的 item 值赋给当前 item -// str += g.PadCode(`if prev_item is not None:`, 3) -// str += g.PadCode(`for key, value in prev_item.items():`, 4) -// str += g.PadCode(`item[key] = value`, 5) -// -// // next stage 字段 -// if f, err := g.GetNextStageField(stage); err == nil { -// // 如果 url 为空，则不进入下一个 stage -// str += g.PadCode(fmt.Sprintf(`if not item['%s']:`, f.Name), 3) -// str += g.PadCode(`continue`, 4) -// -// // 如果找到 next stage 字段，进行下一个回调 -// str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, item['%s']), callback=self.parse_%s, meta={'item': item})`, f.Name, f.NextStage), 3) -// } else { -// // 如果没找到 next stage 字段，返回 item -// str += g.PadCode(fmt.Sprintf(`yield item`), 3) -// } -// -// // 分页 -// if stage.PageCss != "" || stage.PageXpath != "" { -// str += g.PadCode(fmt.Sprintf(`next_url = response.%s.extract_first()`, g.GetExtractStringFromStage(stage)), 2) -// str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, next_url), callback=self.parse_%s, meta={'item': prev_item})`, stageName), 2) -// } -// -// // 加入末尾换行 -// str += g.PadCode("", 0) -// -// return str -//} -// -//// 获取所有字段 -//func (g ScrapyGenerator) GetAllFields() []entity.Field { -// return GetAllFields(g.ConfigData) -//} -// -//// 获取包含 next stage 的字段 -//func (g ScrapyGenerator) GetNextStageField(stage entity.Stage) (entity.Field, error) { -// for _, field := range stage.Fields { -// if field.NextStage != "" { -// return field, nil -// } -// } -// return entity.Field{}, errors.New("cannot find next stage field") -//} -// -//func (g ScrapyGenerator) GetExtractStringFromField(f entity.Field) string { -// if f.Css != "" { -// // 如果为CSS -// if f.Attr == "" { -// // 文本 -// return fmt.Sprintf(`css('%s::text')`, f.Css) -// } else { -// // 属性 -// return fmt.Sprintf(`css('%s::attr("%s")')`, f.Css, f.Attr) -// } -// } else { -// // 如果为XPath -// if f.Attr == "" { -// // 文本 -// return fmt.Sprintf(`xpath('string(%s)')`, f.Xpath) -// } else { -// // 属性 -// return fmt.Sprintf(`xpath('%s/@%s')`, f.Xpath, f.Attr) -// } -// } -//} -// -//func (g ScrapyGenerator) GetExtractStringFromStage(stage entity.Stage) string { -// // 分页元素属性，默认为 href -// pageAttr := "href" -// if stage.PageAttr != "" { -// pageAttr = stage.PageAttr -// } -// -// if stage.PageCss != "" { -// // 如果为CSS -// return fmt.Sprintf(`css('%s::attr("%s")')`, stage.PageCss, pageAttr) -// } else { -// // 如果为XPath -// return fmt.Sprintf(`xpath('%s/@%s')`, stage.PageXpath, pageAttr) -// } -//} -// -//func (g ScrapyGenerator) GetListString(stage entity.Stage) string { -// if stage.ListCss != "" { -// return fmt.Sprintf(`css('%s')`, stage.ListCss) -// } else { -// return fmt.Sprintf(`xpath('%s')`, stage.ListXpath) -// } -//} diff --git a/core/models/delegate/base_test.go b/core/models/delegate/base_test.go deleted file mode 100644 index fd84e128..00000000 --- a/core/models/delegate/base_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package delegate_test - -import ( - "context" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "testing" - "time" -) - -func SetupTest(t *testing.T) { - CleanupTest() - t.Cleanup(CleanupTest) -} - -func CleanupTest() { - db := mongo.GetMongoDb("") - names, _ := db.ListCollectionNames(context.Background(), bson.M{}) - for _, n := range names { - _, _ = db.Collection(n).DeleteMany(context.Background(), bson.M{}) - } - - // avoid caching - time.Sleep(200 * time.Millisecond) -} diff --git a/core/models/delegate/model.go b/core/models/delegate/model.go deleted file mode 100644 index 3e9d8139..00000000 --- a/core/models/delegate/model.go +++ /dev/null @@ -1,362 +0,0 @@ -package delegate - -import ( - "encoding/json" - errors2 "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/event" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/errors" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "reflect" - "time" -) - -func NewModelDelegate(doc interfaces.Model, args ...interface{}) interfaces.ModelDelegate { - switch doc.(type) { - case *models.Artifact: - return newModelDelegate(interfaces.ModelIdArtifact, doc, args...) - case *models.Tag: - return newModelDelegate(interfaces.ModelIdTag, doc, args...) - case *models.Node: - return newModelDelegate(interfaces.ModelIdNode, doc, args...) - case *models.Project: - return newModelDelegate(interfaces.ModelIdProject, doc, args...) - case *models.Spider: - return newModelDelegate(interfaces.ModelIdSpider, doc, args...) - case *models.Task: - return newModelDelegate(interfaces.ModelIdTask, doc, args...) - case *models.Job: - return newModelDelegate(interfaces.ModelIdJob, doc, args...) - case *models.Schedule: - return newModelDelegate(interfaces.ModelIdSchedule, doc, args...) - case *models.User: - return newModelDelegate(interfaces.ModelIdUser, doc, args...) - case *models.Setting: - return newModelDelegate(interfaces.ModelIdSetting, doc, args...) - case *models.Token: - return newModelDelegate(interfaces.ModelIdToken, doc, args...) - case *models.Variable: - return newModelDelegate(interfaces.ModelIdVariable, doc, args...) - case *models.TaskQueueItem: - return newModelDelegate(interfaces.ModelIdTaskQueue, doc, args...) - case *models.TaskStat: - return newModelDelegate(interfaces.ModelIdTaskStat, doc, args...) - case *models.SpiderStat: - return newModelDelegate(interfaces.ModelIdSpiderStat, doc, args...) - case *models.DataSource: - return newModelDelegate(interfaces.ModelIdDataSource, doc, args...) - case *models.DataCollection: - return newModelDelegate(interfaces.ModelIdDataCollection, doc, args...) - case *models.Result: - return newModelDelegate(interfaces.ModelIdResult, doc, args...) - case *models.Password: - return newModelDelegate(interfaces.ModelIdPassword, doc, args...) - case *models.ExtraValue: - return newModelDelegate(interfaces.ModelIdExtraValue, doc, args...) - case *models.Git: - return newModelDelegate(interfaces.ModelIdGit, doc, args...) - case *models.Role: - return newModelDelegate(interfaces.ModelIdRole, doc, args...) - case *models.UserRole: - return newModelDelegate(interfaces.ModelIdUserRole, doc, args...) - case *models.Permission: - return newModelDelegate(interfaces.ModelIdPermission, doc, args...) - case *models.RolePermission: - return newModelDelegate(interfaces.ModelIdRolePermission, doc, args...) - case *models.Environment: - return newModelDelegate(interfaces.ModelIdEnvironment, doc, args...) - case *models.DependencySetting: - return newModelDelegate(interfaces.ModelIdDependencySetting, doc, args...) - default: - _ = trace.TraceError(errors2.ErrorModelInvalidType) - return nil - } -} - -func newModelDelegate(id interfaces.ModelId, doc interfaces.Model, args ...interface{}) interfaces.ModelDelegate { - // user - u := utils.GetUserFromArgs(args...) - - // collection name - colName := models.GetModelColName(id) - - // model delegate - d := &ModelDelegate{ - id: id, - colName: colName, - doc: doc, - a: &models.Artifact{ - Col: colName, - }, - u: u, - } - - return d -} - -type ModelDelegate struct { - id interfaces.ModelId - colName string - doc interfaces.Model // doc to delegate - cd bson.M // current doc - od bson.M // original doc - a interfaces.ModelArtifact // artifact - u interfaces.User // user -} - -// Add model -func (d *ModelDelegate) Add() (err error) { - return d.do(interfaces.ModelDelegateMethodAdd) -} - -// Save model -func (d *ModelDelegate) Save() (err error) { - return d.do(interfaces.ModelDelegateMethodSave) -} - -// Delete model -func (d *ModelDelegate) Delete() (err error) { - return d.do(interfaces.ModelDelegateMethodDelete) -} - -// GetArtifact refresh artifact and return it -func (d *ModelDelegate) GetArtifact() (res interfaces.ModelArtifact, err error) { - if err := d.do(interfaces.ModelDelegateMethodGetArtifact); err != nil { - return nil, err - } - return d.a, nil -} - -// Refresh model -func (d *ModelDelegate) Refresh() (err error) { - return d.refresh() -} - -// GetModel return model -func (d *ModelDelegate) GetModel() (res interfaces.Model) { - return d.doc -} - -func (d *ModelDelegate) ToBytes(m interface{}) (bytes []byte, err error) { - if m != nil { - return utils.JsonToBytes(m) - } - return json.Marshal(d.doc) -} - -// do action given the model delegate method -func (d *ModelDelegate) do(method interfaces.ModelDelegateMethod) (err error) { - switch method { - case interfaces.ModelDelegateMethodAdd: - err = d.add() - case interfaces.ModelDelegateMethodSave: - err = d.save() - case interfaces.ModelDelegateMethodDelete: - err = d.delete() - case interfaces.ModelDelegateMethodGetArtifact, interfaces.ModelDelegateMethodRefresh: - err = d.refresh() - default: - return trace.TraceError(errors2.ErrorModelInvalidType) - } - - if err != nil { - return err - } - - // trigger event - eventName := GetEventName(d, method) - go event.SendEvent(eventName, d.doc) - - return nil -} - -// add model -func (d *ModelDelegate) add() (err error) { - if d.doc == nil { - return trace.TraceError(errors.ErrMissingValue) - } - if d.doc.GetId().IsZero() { - d.doc.SetId(primitive.NewObjectID()) - } - col := mongo.GetMongoCol(d.colName) - if _, err = col.Insert(d.doc); err != nil { - return trace.TraceError(err) - } - if err := d.upsertArtifact(); err != nil { - return trace.TraceError(err) - } - return d.refresh() -} - -// save model -func (d *ModelDelegate) save() (err error) { - // validate - if d.doc == nil || d.doc.GetId().IsZero() { - return trace.TraceError(errors.ErrMissingValue) - } - - // collection - col := mongo.GetMongoCol(d.colName) - - // current doc - docData, err := bson.Marshal(d.doc) - if err != nil { - trace.PrintError(err) - } else { - if err := bson.Unmarshal(docData, &d.cd); err != nil { - trace.PrintError(err) - } - } - - // original doc - if err := col.FindId(d.doc.GetId()).One(&d.od); err != nil { - trace.PrintError(err) - } - - // replace - if err := col.ReplaceId(d.doc.GetId(), d.doc); err != nil { - return trace.TraceError(err) - } - - // upsert artifact - if err := d.upsertArtifact(); err != nil { - return trace.TraceError(err) - } - - return d.refresh() -} - -// delete model -func (d *ModelDelegate) delete() (err error) { - if d.doc.GetId().IsZero() { - return trace.TraceError(errors2.ErrorModelMissingId) - } - col := mongo.GetMongoCol(d.colName) - if err := col.FindId(d.doc.GetId()).One(d.doc); err != nil { - return trace.TraceError(err) - } - if err := col.DeleteId(d.doc.GetId()); err != nil { - return trace.TraceError(err) - } - return d.deleteArtifact() -} - -// refresh model and artifact -func (d *ModelDelegate) refresh() (err error) { - if d.doc.GetId().IsZero() { - return trace.TraceError(errors2.ErrorModelMissingId) - } - col := mongo.GetMongoCol(d.colName) - fr := col.FindId(d.doc.GetId()) - if err := fr.One(d.doc); err != nil { - return trace.TraceError(err) - } - return d.refreshArtifact() -} - -// refresh artifact -func (d *ModelDelegate) refreshArtifact() (err error) { - if d.doc.GetId().IsZero() { - return trace.TraceError(errors2.ErrorModelMissingId) - } - col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) - if err := col.FindId(d.doc.GetId()).One(d.a); err != nil { - return trace.TraceError(err) - } - return nil -} - -// upsertArtifact -func (d *ModelDelegate) upsertArtifact() (err error) { - // skip - if d._skip() { - return nil - } - - // validate id - if d.doc.GetId().IsZero() { - return trace.TraceError(errors.ErrMissingValue) - } - - // mongo collection - col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) - - // assign id to artifact - d.a.SetId(d.doc.GetId()) - - // attempt to find artifact - if err := col.FindId(d.doc.GetId()).One(d.a); err != nil { - if err == mongo2.ErrNoDocuments { - // new artifact - d.a.GetSys().SetCreateTs(time.Now()) - d.a.GetSys().SetUpdateTs(time.Now()) - if d.u != nil && !reflect.ValueOf(d.u).IsZero() { - d.a.GetSys().SetCreateUid(d.u.GetId()) - d.a.GetSys().SetUpdateUid(d.u.GetId()) - } - _, err = col.Insert(d.a) - if err != nil { - return trace.TraceError(err) - } - return nil - } else { - // error - return trace.TraceError(err) - } - } - - // existing artifact - d.a.GetSys().SetUpdateTs(time.Now()) - if d.u != nil { - d.a.GetSys().SetUpdateUid(d.u.GetId()) - } - - // save new artifact - return col.ReplaceId(d.a.GetId(), d.a) -} - -// deleteArtifact -func (d *ModelDelegate) deleteArtifact() (err error) { - // skip - if d._skip() { - return nil - } - - if d.doc.GetId().IsZero() { - return trace.TraceError(errors.ErrMissingValue) - } - col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) - d.a.SetId(d.doc.GetId()) - d.a.SetObj(d.doc) - d.a.SetDel(true) - d.a.GetSys().SetDeleteTs(time.Now()) - if d.u != nil { - d.a.GetSys().SetDeleteUid(d.u.GetId()) - } - return col.ReplaceId(d.doc.GetId(), d.a) -} - -func (d *ModelDelegate) hasChange() (ok bool) { - return !utils.BsonMEqual(d.cd, d.od) -} - -func (d *ModelDelegate) _skip() (ok bool) { - switch d.id { - case - interfaces.ModelIdArtifact, - interfaces.ModelIdTaskQueue, - interfaces.ModelIdTaskStat, - interfaces.ModelIdSpiderStat, - interfaces.ModelIdResult, - interfaces.ModelIdPassword: - return true - default: - return false - } -} diff --git a/core/models/delegate/model_node.go b/core/models/delegate/model_node.go deleted file mode 100644 index 99bdd11d..00000000 --- a/core/models/delegate/model_node.go +++ /dev/null @@ -1,37 +0,0 @@ -package delegate - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "time" -) - -type ModelNodeDelegate struct { - n interfaces.Node - interfaces.ModelDelegate -} - -func (d *ModelNodeDelegate) UpdateStatus(active bool, activeTs *time.Time, status string) (err error) { - d.n.SetActive(active) - if activeTs != nil { - d.n.SetActiveTs(*activeTs) - } - d.n.SetStatus(status) - return d.Save() -} - -func (d *ModelNodeDelegate) UpdateStatusOnline() (err error) { - now := time.Now() - return d.UpdateStatus(true, &now, constants.NodeStatusOnline) -} - -func (d *ModelNodeDelegate) UpdateStatusOffline() (err error) { - return d.UpdateStatus(false, nil, constants.NodeStatusOffline) -} - -func NewModelNodeDelegate(n interfaces.Node) interfaces.ModelNodeDelegate { - return &ModelNodeDelegate{ - n: n, - ModelDelegate: NewModelDelegate(n), - } -} diff --git a/core/models/delegate/model_node_test.go b/core/models/delegate/model_node_test.go deleted file mode 100644 index 2e0f847e..00000000 --- a/core/models/delegate/model_node_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package delegate_test - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/stretchr/testify/require" - "testing" -) - -func TestNode_Add(t *testing.T) { - SetupTest(t) - - n := &models2.Node{} - - err := delegate.NewModelDelegate(n).Add() - require.Nil(t, err) - require.NotNil(t, n.Id) - - // validate artifact - a, err := delegate.NewModelDelegate(n).GetArtifact() - require.Nil(t, err) - require.Equal(t, n.Id, a.GetId()) - require.NotNil(t, a.GetSys().GetCreateTs()) - require.NotNil(t, a.GetSys().GetUpdateTs()) -} - -func TestNode_Save(t *testing.T) { - SetupTest(t) - - n := &models2.Node{} - - err := delegate.NewModelDelegate(n).Add() - - name := "test_node" - n.Name = name - err = delegate.NewModelDelegate(n).Save() - require.Nil(t, err) - - err = mongo.GetMongoCol(interfaces.ModelColNameNode).FindId(n.Id).One(&n) - require.Nil(t, err) - require.Equal(t, name, n.Name) -} - -func TestNode_Delete(t *testing.T) { - SetupTest(t) - - n := &models2.Node{ - Name: "test_node", - } - - err := delegate.NewModelDelegate(n).Add() - require.Nil(t, err) - - err = delegate.NewModelDelegate(n).Delete() - require.Nil(t, err) - - var a models2.Artifact - col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) - err = col.FindId(n.Id).One(&a) - require.Nil(t, err) - require.NotNil(t, a.Obj) - require.True(t, a.Del) -} diff --git a/core/models/delegate/model_role_test.go b/core/models/delegate/model_role_test.go deleted file mode 100644 index dcab9bf5..00000000 --- a/core/models/delegate/model_role_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package delegate_test - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/stretchr/testify/require" - "testing" -) - -func TestRole_Add(t *testing.T) { - SetupTest(t) - - p := &models2.Role{} - - err := delegate.NewModelDelegate(p).Add() - require.Nil(t, err) - require.NotNil(t, p.Id) - - a, err := delegate.NewModelDelegate(p).GetArtifact() - require.Nil(t, err) - require.Equal(t, p.Id, a.GetId()) - require.NotNil(t, a.GetSys().GetCreateTs()) - require.NotNil(t, a.GetSys().GetUpdateTs()) -} - -func TestRole_Save(t *testing.T) { - SetupTest(t) - - p := &models2.Role{} - - err := delegate.NewModelDelegate(p).Add() - require.Nil(t, err) - - name := "test_role" - p.Name = name - err = delegate.NewModelDelegate(p).Save() - require.Nil(t, err) - - err = mongo.GetMongoCol(interfaces.ModelColNameRole).FindId(p.Id).One(&p) - require.Nil(t, err) - require.Equal(t, name, p.Name) -} - -func TestRole_Delete(t *testing.T) { - SetupTest(t) - - p := &models2.Role{ - Name: "test_role", - } - - err := delegate.NewModelDelegate(p).Add() - require.Nil(t, err) - - err = delegate.NewModelDelegate(p).Delete() - require.Nil(t, err) - - var a models2.Artifact - col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) - err = col.FindId(p.Id).One(&a) - require.Nil(t, err) - require.NotNil(t, a.Obj) - require.True(t, a.Del) -} diff --git a/core/models/delegate/model_test.go b/core/models/delegate/model_test.go deleted file mode 100644 index 93dc9140..00000000 --- a/core/models/delegate/model_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package delegate_test - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/stretchr/testify/require" - "testing" -) - -func TestProject_Add(t *testing.T) { - SetupTest(t) - - p := &models2.Project{} - - err := delegate.NewModelDelegate(p).Add() - require.Nil(t, err) - require.NotNil(t, p.Id) - - a, err := delegate.NewModelDelegate(p).GetArtifact() - require.Nil(t, err) - require.Equal(t, p.Id, a.GetId()) - require.NotNil(t, a.GetSys().GetCreateTs()) - require.NotNil(t, a.GetSys().GetUpdateTs()) -} - -func TestProject_Save(t *testing.T) { - SetupTest(t) - - p := &models2.Project{} - - err := delegate.NewModelDelegate(p).Add() - require.Nil(t, err) - - name := "test_project" - p.Name = name - err = delegate.NewModelDelegate(p).Save() - require.Nil(t, err) - - err = mongo.GetMongoCol(interfaces.ModelColNameProject).FindId(p.Id).One(&p) - require.Nil(t, err) - require.Equal(t, name, p.Name) -} - -func TestProject_Delete(t *testing.T) { - SetupTest(t) - - p := &models2.Project{ - Name: "test_project", - } - - err := delegate.NewModelDelegate(p).Add() - require.Nil(t, err) - - err = delegate.NewModelDelegate(p).Delete() - require.Nil(t, err) - - var a models2.Artifact - col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) - err = col.FindId(p.Id).One(&a) - require.Nil(t, err) - require.NotNil(t, a.Obj) - require.True(t, a.Del) -} diff --git a/core/models/delegate/utils_event.go b/core/models/delegate/utils_event.go deleted file mode 100644 index d5cd3f89..00000000 --- a/core/models/delegate/utils_event.go +++ /dev/null @@ -1,20 +0,0 @@ -package delegate - -import ( - "fmt" - "github.com/crawlab-team/crawlab/core/interfaces" -) - -func GetEventName(d *ModelDelegate, method interfaces.ModelDelegateMethod) (eventName string) { - return getEventName(d, method) -} - -func getEventName(d *ModelDelegate, method interfaces.ModelDelegateMethod) (eventName string) { - if method == interfaces.ModelDelegateMethodSave { - hasChange := d.hasChange() - if hasChange { - method = interfaces.ModelDelegateMethodChange - } - } - return fmt.Sprintf("model:%s:%s", d.colName, method) -} diff --git a/core/models/models/artifact.go b/core/models/models/artifact.go deleted file mode 100644 index 55e07af8..00000000 --- a/core/models/models/artifact.go +++ /dev/null @@ -1,56 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Artifact struct { - Id primitive.ObjectID `bson:"_id" json:"_id"` - Col string `bson:"_col" json:"_col"` - Del bool `bson:"_del" json:"_del"` - TagIds []primitive.ObjectID `bson:"_tid" json:"_tid"` - Sys *ArtifactSys `bson:"_sys" json:"_sys"` - Obj interface{} `bson:"_obj" json:"_obj"` -} - -func (a *Artifact) GetId() (id primitive.ObjectID) { - return a.Id -} - -func (a *Artifact) SetId(id primitive.ObjectID) { - a.Id = id -} - -func (a *Artifact) GetSys() (sys interfaces.ModelArtifactSys) { - if a.Sys == nil { - a.Sys = &ArtifactSys{} - } - return a.Sys -} - -func (a *Artifact) GetTagIds() (ids []primitive.ObjectID) { - return a.TagIds -} - -func (a *Artifact) SetTagIds(ids []primitive.ObjectID) { - a.TagIds = ids -} - -func (a *Artifact) SetObj(obj interfaces.Model) { - a.Obj = obj -} - -func (a *Artifact) SetDel(del bool) { - a.Del = del -} - -type ArtifactList []Artifact - -func (l *ArtifactList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/artifact_sys.go b/core/models/models/artifact_sys.go deleted file mode 100644 index 539d8fb0..00000000 --- a/core/models/models/artifact_sys.go +++ /dev/null @@ -1,63 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" - "time" -) - -type ArtifactSys struct { - CreateTs time.Time `json:"create_ts" bson:"create_ts"` - CreateUid primitive.ObjectID `json:"create_uid" bson:"create_uid"` - UpdateTs time.Time `json:"update_ts" bson:"update_ts"` - UpdateUid primitive.ObjectID `json:"update_uid" bson:"update_uid"` - DeleteTs time.Time `json:"delete_ts" bson:"delete_ts"` - DeleteUid primitive.ObjectID `json:"delete_uid" bson:"delete_uid"` -} - -func (sys *ArtifactSys) GetCreateTs() time.Time { - return sys.CreateTs -} - -func (sys *ArtifactSys) SetCreateTs(ts time.Time) { - sys.CreateTs = ts -} - -func (sys *ArtifactSys) GetUpdateTs() time.Time { - return sys.UpdateTs -} - -func (sys *ArtifactSys) SetUpdateTs(ts time.Time) { - sys.UpdateTs = ts -} - -func (sys *ArtifactSys) GetDeleteTs() time.Time { - return sys.DeleteTs -} - -func (sys *ArtifactSys) SetDeleteTs(ts time.Time) { - sys.DeleteTs = ts -} - -func (sys *ArtifactSys) GetCreateUid() primitive.ObjectID { - return sys.CreateUid -} - -func (sys *ArtifactSys) SetCreateUid(id primitive.ObjectID) { - sys.CreateUid = id -} - -func (sys *ArtifactSys) GetUpdateUid() primitive.ObjectID { - return sys.UpdateUid -} - -func (sys *ArtifactSys) SetUpdateUid(id primitive.ObjectID) { - sys.UpdateUid = id -} - -func (sys *ArtifactSys) GetDeleteUid() primitive.ObjectID { - return sys.DeleteUid -} - -func (sys *ArtifactSys) SetDeleteUid(id primitive.ObjectID) { - sys.DeleteUid = id -} diff --git a/core/models/models/base.go b/core/models/models/base.go deleted file mode 100644 index a17bb98f..00000000 --- a/core/models/models/base.go +++ /dev/null @@ -1,13 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type BaseModel struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` -} - -func (d *BaseModel) GetId() (id primitive.ObjectID) { - return d.Id -} diff --git a/core/models/models/data_collection.go b/core/models/models/data_collection.go deleted file mode 100644 index dfc6fb23..00000000 --- a/core/models/models/data_collection.go +++ /dev/null @@ -1,36 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type DataCollection struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Name string `json:"name" bson:"name"` - Fields []entity.DataField `json:"fields" bson:"fields"` - Dedup struct { - Enabled bool `json:"enabled" bson:"enabled"` - Keys []string `json:"keys" bson:"keys"` - Type string `json:"type" bson:"type"` - } `json:"dedup" bson:"dedup"` -} - -func (dc *DataCollection) GetId() (id primitive.ObjectID) { - return dc.Id -} - -func (dc *DataCollection) SetId(id primitive.ObjectID) { - dc.Id = id -} - -type DataCollectionList []DataCollection - -func (l *DataCollectionList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/data_source.go b/core/models/models/data_source.go deleted file mode 100644 index 5693d9eb..00000000 --- a/core/models/models/data_source.go +++ /dev/null @@ -1,42 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type DataSource struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Name string `json:"name" bson:"name"` - Type string `json:"type" bson:"type"` - Description string `json:"description" bson:"description"` - Host string `json:"host" bson:"host"` - Port int `json:"port" bson:"port"` - Url string `json:"url" bson:"url"` - Hosts []string `json:"hosts" bson:"hosts"` - Database string `json:"database" bson:"database"` - Username string `json:"username" bson:"username"` - Password string `json:"password,omitempty" bson:"-"` - ConnectType string `json:"connect_type" bson:"connect_type"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"` -} - -func (ds *DataSource) GetId() (id primitive.ObjectID) { - return ds.Id -} - -func (ds *DataSource) SetId(id primitive.ObjectID) { - ds.Id = id -} - -type DataSourceList []DataSource - -func (l *DataSourceList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/dependency_setting.go b/core/models/models/dependency_setting.go deleted file mode 100644 index d6232ede..00000000 --- a/core/models/models/dependency_setting.go +++ /dev/null @@ -1,36 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" - "time" -) - -type DependencySetting struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - Cmd string `json:"cmd" bson:"cmd"` - Proxy string `json:"proxy" bson:"proxy"` - LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"` -} - -func (j *DependencySetting) GetId() (id primitive.ObjectID) { - return j.Id -} - -func (j *DependencySetting) SetId(id primitive.ObjectID) { - j.Id = id -} - -type DependencySettingList []DependencySetting - -func (l *DependencySettingList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/environment.go b/core/models/models/environment.go deleted file mode 100644 index 1598f130..00000000 --- a/core/models/models/environment.go +++ /dev/null @@ -1,46 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Environment struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Key string `json:"key" bson:"key"` - Value string `json:"value" bson:"value"` -} - -func (e *Environment) GetId() (id primitive.ObjectID) { - return e.Id -} - -func (e *Environment) SetId(id primitive.ObjectID) { - e.Id = id -} - -func (e *Environment) GetKey() (key string) { - return e.Key -} - -func (e *Environment) SetKey(key string) { - e.Key = key -} - -func (e *Environment) GetValue() (value string) { - return e.Value -} - -func (e *Environment) SetValue(value string) { - e.Value = value -} - -type EnvironmentList []Environment - -func (l *EnvironmentList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/extra_value.go b/core/models/models/extra_value.go deleted file mode 100644 index 4d65bf5f..00000000 --- a/core/models/models/extra_value.go +++ /dev/null @@ -1,64 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type ExtraValue struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - ObjectId primitive.ObjectID `json:"oid" bson:"oid"` - Model string `json:"model" bson:"m"` - Type string `json:"type" bson:"t"` - Value interface{} `json:"value" bson:"v"` -} - -func (ev *ExtraValue) GetId() (id primitive.ObjectID) { - return ev.Id -} - -func (ev *ExtraValue) SetId(id primitive.ObjectID) { - ev.Id = id -} - -func (ev *ExtraValue) GetValue() (v interface{}) { - return ev.Value -} - -func (ev *ExtraValue) SetValue(v interface{}) { - ev.Value = v -} - -func (ev *ExtraValue) GetObjectId() (oid primitive.ObjectID) { - return ev.ObjectId -} - -func (ev *ExtraValue) SetObjectId(oid primitive.ObjectID) { - ev.ObjectId = oid -} - -func (ev *ExtraValue) GetModel() (m string) { - return ev.Model -} - -func (ev *ExtraValue) SetModel(m string) { - ev.Model = m -} - -func (ev *ExtraValue) GetType() (t string) { - return ev.Type -} - -func (ev *ExtraValue) SetType(t string) { - ev.Type = t -} - -type ExtraValueList []ExtraValue - -func (l *ExtraValueList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/git.go b/core/models/models/git.go deleted file mode 100644 index 1a9a0a58..00000000 --- a/core/models/models/git.go +++ /dev/null @@ -1,82 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Git struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Url string `json:"url" bson:"url"` - AuthType string `json:"auth_type" bson:"auth_type"` - Username string `json:"username" bson:"username"` - Password string `json:"password" bson:"password"` - CurrentBranch string `json:"current_branch" bson:"current_branch"` - AutoPull bool `json:"auto_pull" bson:"auto_pull"` -} - -func (g *Git) GetId() (id primitive.ObjectID) { - return g.Id -} - -func (g *Git) SetId(id primitive.ObjectID) { - g.Id = id -} - -func (g *Git) GetUrl() (url string) { - return g.Url -} - -func (g *Git) SetUrl(url string) { - g.Url = url -} - -func (g *Git) GetAuthType() (authType string) { - return g.AuthType -} - -func (g *Git) SetAuthType(authType string) { - g.AuthType = authType -} - -func (g *Git) GetUsername() (username string) { - return g.Username -} - -func (g *Git) SetUsername(username string) { - g.Username = username -} - -func (g *Git) GetPassword() (password string) { - return g.Password -} - -func (g *Git) SetPassword(password string) { - g.Password = password -} - -func (g *Git) GetCurrentBranch() (currentBranch string) { - return g.CurrentBranch -} - -func (g *Git) SetCurrentBranch(currentBranch string) { - g.CurrentBranch = currentBranch -} - -func (g *Git) GetAutoPull() (autoPull bool) { - return g.AutoPull -} - -func (g *Git) SetAutoPull(autoPull bool) { - g.AutoPull = autoPull -} - -type GitList []Git - -func (l *GitList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/job.go b/core/models/models/job.go deleted file mode 100644 index 66c17720..00000000 --- a/core/models/models/job.go +++ /dev/null @@ -1,29 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Job struct { - Id primitive.ObjectID `bson:"_id" json:"_id"` - TaskId primitive.ObjectID `bson:"task_id" json:"task_id"` -} - -func (j *Job) GetId() (id primitive.ObjectID) { - return j.Id -} - -func (j *Job) SetId(id primitive.ObjectID) { - j.Id = id -} - -type JobList []Job - -func (l *JobList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/node.go b/core/models/models/node.go deleted file mode 100644 index c9698106..00000000 --- a/core/models/models/node.go +++ /dev/null @@ -1,119 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" - "time" -) - -type Node struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Ip string `json:"ip" bson:"ip"` - Port string `json:"port" bson:"port"` - Mac string `json:"mac" bson:"mac"` - Hostname string `json:"hostname" bson:"hostname"` - Description string `json:"description" bson:"description"` - IsMaster bool `json:"is_master" bson:"is_master"` - Status string `json:"status" bson:"status"` - Enabled bool `json:"enabled" bson:"enabled"` - Active bool `json:"active" bson:"active"` - ActiveTs time.Time `json:"active_ts" bson:"active_ts"` - AvailableRunners int `json:"available_runners" bson:"available_runners"` - MaxRunners int `json:"max_runners" bson:"max_runners"` -} - -func (n *Node) GetId() (id primitive.ObjectID) { - return n.Id -} - -func (n *Node) SetId(id primitive.ObjectID) { - n.Id = id -} - -func (n *Node) GetName() (name string) { - return n.Name -} - -func (n *Node) SetName(name string) { - n.Name = name -} - -func (n *Node) GetDescription() (description string) { - return n.Description -} - -func (n *Node) SetDescription(description string) { - n.Description = description -} - -func (n *Node) GetKey() (key string) { - return n.Key -} - -func (n *Node) GetIsMaster() (ok bool) { - return n.IsMaster -} - -func (n *Node) GetActive() (active bool) { - return n.Active -} - -func (n *Node) SetActive(active bool) { - n.Active = active -} - -func (n *Node) SetActiveTs(activeTs time.Time) { - n.ActiveTs = activeTs -} - -func (n *Node) GetStatus() (status string) { - return n.Status -} - -func (n *Node) SetStatus(status string) { - n.Status = status -} - -func (n *Node) GetEnabled() (enabled bool) { - return n.Enabled -} - -func (n *Node) SetEnabled(enabled bool) { - n.Enabled = enabled -} - -func (n *Node) GetAvailableRunners() (runners int) { - return n.AvailableRunners -} - -func (n *Node) SetAvailableRunners(runners int) { - n.AvailableRunners = runners -} - -func (n *Node) GetMaxRunners() (runners int) { - return n.MaxRunners -} - -func (n *Node) SetMaxRunners(runners int) { - n.MaxRunners = runners -} - -func (n *Node) IncrementAvailableRunners() { - n.AvailableRunners++ -} - -func (n *Node) DecrementAvailableRunners() { - n.AvailableRunners-- -} - -type NodeList []Node - -func (l *NodeList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/password.go b/core/models/models/password.go deleted file mode 100644 index 4be613ef..00000000 --- a/core/models/models/password.go +++ /dev/null @@ -1,29 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Password struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Password string `json:"password" bson:"p"` -} - -func (p *Password) GetId() (id primitive.ObjectID) { - return p.Id -} - -func (p *Password) SetId(id primitive.ObjectID) { - p.Id = id -} - -type PasswordList []Password - -func (l *PasswordList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/permission.go b/core/models/models/permission.go deleted file mode 100644 index 51c42d37..00000000 --- a/core/models/models/permission.go +++ /dev/null @@ -1,91 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Permission struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Type string `json:"type" bson:"type"` - Target []string `json:"target" bson:"target"` - Allow []string `json:"allow" bson:"allow"` - Deny []string `json:"deny" bson:"deny"` -} - -func (p *Permission) GetId() (id primitive.ObjectID) { - return p.Id -} - -func (p *Permission) SetId(id primitive.ObjectID) { - p.Id = id -} - -func (p *Permission) GetKey() (key string) { - return p.Key -} - -func (p *Permission) SetKey(key string) { - p.Key = key -} - -func (p *Permission) GetName() (name string) { - return p.Name -} - -func (p *Permission) SetName(name string) { - p.Name = name -} - -func (p *Permission) GetDescription() (description string) { - return p.Description -} - -func (p *Permission) SetDescription(description string) { - p.Description = description -} - -func (p *Permission) GetType() (t string) { - return p.Type -} - -func (p *Permission) SetType(t string) { - p.Type = t -} - -func (p *Permission) GetTarget() (target []string) { - return p.Target -} - -func (p *Permission) SetTarget(target []string) { - p.Target = target -} - -func (p *Permission) GetAllow() (include []string) { - return p.Allow -} - -func (p *Permission) SetAllow(include []string) { - p.Allow = include -} - -func (p *Permission) GetDeny() (exclude []string) { - return p.Deny -} - -func (p *Permission) SetDeny(exclude []string) { - p.Deny = exclude -} - -type PermissionList []Permission - -func (l *PermissionList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/project.go b/core/models/models/project.go deleted file mode 100644 index 59b01212..00000000 --- a/core/models/models/project.go +++ /dev/null @@ -1,47 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Project struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Spiders int `json:"spiders" bson:"-"` -} - -func (p *Project) GetId() (id primitive.ObjectID) { - return p.Id -} - -func (p *Project) SetId(id primitive.ObjectID) { - p.Id = id -} - -func (p *Project) GetName() (name string) { - return p.Name -} - -func (p *Project) SetName(name string) { - p.Name = name -} - -func (p *Project) GetDescription() (description string) { - return p.Description -} - -func (p *Project) SetDescription(description string) { - p.Description = description -} - -type ProjectList []Project - -func (l *ProjectList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/result.go b/core/models/models/result.go deleted file mode 100644 index 9c2957b0..00000000 --- a/core/models/models/result.go +++ /dev/null @@ -1,60 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Result bson.M - -func (r *Result) GetId() (id primitive.ObjectID) { - res, ok := r.Value()["_id"] - if ok { - id, ok = res.(primitive.ObjectID) - if ok { - return id - } - } - return id -} - -func (r *Result) SetId(id primitive.ObjectID) { - (*r)["_id"] = id -} - -func (r *Result) Value() map[string]interface{} { - return *r -} - -func (r *Result) SetValue(key string, value interface{}) { - (*r)[key] = value -} - -func (r *Result) GetValue(key string) (value interface{}) { - return (*r)[key] -} - -func (r *Result) GetTaskId() (id primitive.ObjectID) { - res := r.GetValue(constants.TaskKey) - if res == nil { - return id - } - id, _ = res.(primitive.ObjectID) - return id -} - -func (r *Result) SetTaskId(id primitive.ObjectID) { - r.SetValue(constants.TaskKey, id) -} - -type ResultList []Result - -func (l *ResultList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/role.go b/core/models/models/role.go deleted file mode 100644 index 3a73d8f6..00000000 --- a/core/models/models/role.go +++ /dev/null @@ -1,55 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Role struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` -} - -func (r *Role) GetId() (id primitive.ObjectID) { - return r.Id -} - -func (r *Role) SetId(id primitive.ObjectID) { - r.Id = id -} - -func (r *Role) GetKey() (key string) { - return r.Key -} - -func (r *Role) SetKey(key string) { - r.Key = key -} - -func (r *Role) GetName() (name string) { - return r.Name -} - -func (r *Role) SetName(name string) { - r.Name = name -} - -func (r *Role) GetDescription() (description string) { - return r.Description -} - -func (r *Role) SetDescription(description string) { - r.Description = description -} - -type RoleList []Role - -func (l *RoleList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/role_permission.go b/core/models/models/role_permission.go deleted file mode 100644 index d603c143..00000000 --- a/core/models/models/role_permission.go +++ /dev/null @@ -1,30 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type RolePermission struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"` -} - -func (ur *RolePermission) GetId() (id primitive.ObjectID) { - return ur.Id -} - -func (ur *RolePermission) SetId(id primitive.ObjectID) { - ur.Id = id -} - -type RolePermissionList []RolePermission - -func (l *RolePermissionList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/schedule.go b/core/models/models/schedule.go deleted file mode 100644 index 96c455b3..00000000 --- a/core/models/models/schedule.go +++ /dev/null @@ -1,113 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/robfig/cron/v3" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Schedule struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` - Cron string `json:"cron" bson:"cron"` - EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` - Cmd string `json:"cmd" bson:"cmd"` - Param string `json:"param" bson:"param"` - Mode string `json:"mode" bson:"mode"` - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` - Priority int `json:"priority" bson:"priority"` - Enabled bool `json:"enabled" bson:"enabled"` - UserId primitive.ObjectID `json:"user_id" bson:"user_id"` -} - -func (s *Schedule) GetId() (id primitive.ObjectID) { - return s.Id -} - -func (s *Schedule) SetId(id primitive.ObjectID) { - s.Id = id -} - -func (s *Schedule) GetEnabled() (enabled bool) { - return s.Enabled -} - -func (s *Schedule) SetEnabled(enabled bool) { - s.Enabled = enabled -} - -func (s *Schedule) GetEntryId() (id cron.EntryID) { - return s.EntryId -} - -func (s *Schedule) SetEntryId(id cron.EntryID) { - s.EntryId = id -} - -func (s *Schedule) GetCron() (c string) { - return s.Cron -} - -func (s *Schedule) SetCron(c string) { - s.Cron = c -} - -func (s *Schedule) GetSpiderId() (id primitive.ObjectID) { - return s.SpiderId -} - -func (s *Schedule) SetSpiderId(id primitive.ObjectID) { - s.SpiderId = id -} - -func (s *Schedule) GetMode() (mode string) { - return s.Mode -} - -func (s *Schedule) SetMode(mode string) { - s.Mode = mode -} - -func (s *Schedule) GetNodeIds() (ids []primitive.ObjectID) { - return s.NodeIds -} - -func (s *Schedule) SetNodeIds(ids []primitive.ObjectID) { - s.NodeIds = ids -} - -func (s *Schedule) GetCmd() (cmd string) { - return s.Cmd -} - -func (s *Schedule) SetCmd(cmd string) { - s.Cmd = cmd -} - -func (s *Schedule) GetParam() (param string) { - return s.Param -} - -func (s *Schedule) SetParam(param string) { - s.Param = param -} - -func (s *Schedule) GetPriority() (p int) { - return s.Priority -} - -func (s *Schedule) SetPriority(p int) { - s.Priority = p -} - -type ScheduleList []Schedule - -func (l *ScheduleList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/setting.go b/core/models/models/setting.go deleted file mode 100644 index b0fc9d88..00000000 --- a/core/models/models/setting.go +++ /dev/null @@ -1,31 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Setting struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Key string `json:"key" bson:"key"` - Value bson.M `json:"value" bson:"value"` -} - -func (s *Setting) GetId() (id primitive.ObjectID) { - return s.Id -} - -func (s *Setting) SetId(id primitive.ObjectID) { - s.Id = id -} - -type SettingList []Setting - -func (l *SettingList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/spider.go b/core/models/models/spider.go deleted file mode 100644 index 49b89418..00000000 --- a/core/models/models/spider.go +++ /dev/null @@ -1,137 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Env struct { - Name string `json:"name" bson:"name"` - Value string `json:"value" bson:"value"` -} - -type Spider struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` // spider id - Name string `json:"name" bson:"name"` // spider name - Type string `json:"type" bson:"type"` // spider type - ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id - ColName string `json:"col_name,omitempty" bson:"-"` // data collection name - DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id - DataSource *DataSource `json:"data_source,omitempty" bson:"-"` // data source - Description string `json:"description" bson:"description"` // description - ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id - Mode string `json:"mode" bson:"mode"` // default Task.Mode - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds - Stat *SpiderStat `json:"stat,omitempty" bson:"-"` - - // execution - Cmd string `json:"cmd" bson:"cmd"` // execute command - Param string `json:"param" bson:"param"` // default task param - Priority int `json:"priority" bson:"priority"` - AutoInstall bool `json:"auto_install" bson:"auto_install"` - - // settings - IncrementalSync bool `json:"incremental_sync" bson:"incremental_sync"` // whether to incrementally sync files -} - -func (s *Spider) GetId() (id primitive.ObjectID) { - return s.Id -} - -func (s *Spider) SetId(id primitive.ObjectID) { - s.Id = id -} - -func (s *Spider) GetName() (name string) { - return s.Name -} - -func (s *Spider) SetName(name string) { - s.Name = name -} - -func (s *Spider) GetDescription() (description string) { - return s.Description -} - -func (s *Spider) SetDescription(description string) { - s.Description = description -} - -func (s *Spider) GetType() (ty string) { - return s.Type -} - -func (s *Spider) GetMode() (mode string) { - return s.Mode -} - -func (s *Spider) SetMode(mode string) { - s.Mode = mode -} - -func (s *Spider) GetNodeIds() (ids []primitive.ObjectID) { - return s.NodeIds -} - -func (s *Spider) SetNodeIds(ids []primitive.ObjectID) { - s.NodeIds = ids -} - -func (s *Spider) GetCmd() (cmd string) { - return s.Cmd -} - -func (s *Spider) SetCmd(cmd string) { - s.Cmd = cmd -} - -func (s *Spider) GetParam() (param string) { - return s.Param -} - -func (s *Spider) SetParam(param string) { - s.Param = param -} - -func (s *Spider) GetPriority() (p int) { - return s.Priority -} - -func (s *Spider) SetPriority(p int) { - s.Priority = p -} - -func (s *Spider) GetColId() (id primitive.ObjectID) { - return s.ColId -} - -func (s *Spider) SetColId(id primitive.ObjectID) { - s.ColId = id -} - -func (s *Spider) GetIncrementalSync() (incrementalSync bool) { - return s.IncrementalSync -} - -func (s *Spider) SetIncrementalSync(incrementalSync bool) { - s.IncrementalSync = incrementalSync -} - -func (s *Spider) GetAutoInstall() (autoInstall bool) { - return s.AutoInstall -} - -func (s *Spider) SetAutoInstall(autoInstall bool) { - s.AutoInstall = autoInstall -} - -type SpiderList []Spider - -func (l *SpiderList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/spider_stat.go b/core/models/models/spider_stat.go deleted file mode 100644 index 09047b90..00000000 --- a/core/models/models/spider_stat.go +++ /dev/null @@ -1,38 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type SpiderStat struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"` - LastTask *Task `json:"last_task,omitempty" bson:"-"` - Tasks int `json:"tasks" bson:"tasks"` - Results int `json:"results" bson:"results"` - WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second - RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second - TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second - AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second - AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second - AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second -} - -func (s *SpiderStat) GetId() (id primitive.ObjectID) { - return s.Id -} - -func (s *SpiderStat) SetId(id primitive.ObjectID) { - s.Id = id -} - -type SpiderStatList []SpiderStat - -func (l *SpiderStatList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/tag.go b/core/models/models/tag.go deleted file mode 100644 index 86708240..00000000 --- a/core/models/models/tag.go +++ /dev/null @@ -1,44 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Tag struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Name string `json:"name" bson:"name"` - Color string `json:"color" bson:"color"` - Description string `json:"description" bson:"description"` - Col string `json:"col" bson:"col"` -} - -func (t *Tag) GetId() (id primitive.ObjectID) { - return t.Id -} - -func (t *Tag) SetId(id primitive.ObjectID) { - t.Id = id -} - -func (t *Tag) GetName() (res string) { - return t.Name -} - -func (t *Tag) GetColor() (res string) { - return t.Color -} - -func (t *Tag) SetCol(col string) { - t.Col = col -} - -type TagList []Tag - -func (l *TagList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/task.go b/core/models/models/task.go deleted file mode 100644 index 6dcc926a..00000000 --- a/core/models/models/task.go +++ /dev/null @@ -1,118 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" - "time" -) - -type Task struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` - Status string `json:"status" bson:"status"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - Cmd string `json:"cmd" bson:"cmd"` - Param string `json:"param" bson:"param"` - Error string `json:"error" bson:"error"` - Pid int `json:"pid" bson:"pid"` - ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` // Schedule.Id - Type string `json:"type" bson:"type"` - Mode string `json:"mode" bson:"mode"` // running mode of Task - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // list of Node.Id - ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"` // parent Task.Id if it'Spider a sub-task - Priority int `json:"priority" bson:"priority"` - Stat *TaskStat `json:"stat,omitempty" bson:"-"` - HasSub bool `json:"has_sub" json:"has_sub"` // whether to have sub-tasks - SubTasks []Task `json:"sub_tasks,omitempty" bson:"-"` - Spider *Spider `json:"spider,omitempty" bson:"-"` - UserId primitive.ObjectID `json:"-" bson:"-"` - CreateTs time.Time `json:"create_ts" bson:"create_ts"` -} - -func (t *Task) GetId() (id primitive.ObjectID) { - return t.Id -} - -func (t *Task) SetId(id primitive.ObjectID) { - t.Id = id -} - -func (t *Task) GetNodeId() (id primitive.ObjectID) { - return t.NodeId -} - -func (t *Task) SetNodeId(id primitive.ObjectID) { - t.NodeId = id -} - -func (t *Task) GetNodeIds() (ids []primitive.ObjectID) { - return t.NodeIds -} - -func (t *Task) GetStatus() (status string) { - return t.Status -} - -func (t *Task) SetStatus(status string) { - t.Status = status -} - -func (t *Task) GetError() (error string) { - return t.Error -} - -func (t *Task) SetError(error string) { - t.Error = error -} - -func (t *Task) GetPid() (pid int) { - return t.Pid -} - -func (t *Task) SetPid(pid int) { - t.Pid = pid -} - -func (t *Task) GetSpiderId() (id primitive.ObjectID) { - return t.SpiderId -} - -func (t *Task) GetType() (ty string) { - return t.Type -} - -func (t *Task) GetCmd() (cmd string) { - return t.Cmd -} - -func (t *Task) GetParam() (param string) { - return t.Param -} - -func (t *Task) GetPriority() (p int) { - return t.Priority -} - -func (t *Task) GetUserId() (id primitive.ObjectID) { - return t.UserId -} - -func (t *Task) SetUserId(id primitive.ObjectID) { - t.UserId = id -} - -type TaskList []Task - -func (l *TaskList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} - -type TaskDailyItem struct { - Date string `json:"date" bson:"_id"` - TaskCount int `json:"task_count" bson:"task_count"` - AvgRuntimeDuration float64 `json:"avg_runtime_duration" bson:"avg_runtime_duration"` -} diff --git a/core/models/models/task_queue_item.go b/core/models/models/task_queue_item.go deleted file mode 100644 index f2f08d49..00000000 --- a/core/models/models/task_queue_item.go +++ /dev/null @@ -1,30 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type TaskQueueItem struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Priority int `json:"p" bson:"p"` - NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"` -} - -func (t *TaskQueueItem) GetId() (id primitive.ObjectID) { - return t.Id -} - -func (t *TaskQueueItem) SetId(id primitive.ObjectID) { - t.Id = id -} - -type TaskQueueItemList []TaskQueueItem - -func (l *TaskQueueItemList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/task_stat.go b/core/models/models/task_stat.go deleted file mode 100644 index d8f50905..00000000 --- a/core/models/models/task_stat.go +++ /dev/null @@ -1,101 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" - "time" -) - -type TaskStat struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"` - StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` - EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` - WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond - RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond - TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond - ResultCount int64 `json:"result_count" bson:"result_count"` - ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"` -} - -func (s *TaskStat) GetId() (id primitive.ObjectID) { - return s.Id -} - -func (s *TaskStat) SetId(id primitive.ObjectID) { - s.Id = id -} - -func (s *TaskStat) GetCreateTs() (ts time.Time) { - return s.CreateTs -} - -func (s *TaskStat) SetCreateTs(ts time.Time) { - s.CreateTs = ts -} - -func (s *TaskStat) GetStartTs() (ts time.Time) { - return s.StartTs -} - -func (s *TaskStat) SetStartTs(ts time.Time) { - s.StartTs = ts -} - -func (s *TaskStat) GetEndTs() (ts time.Time) { - return s.EndTs -} - -func (s *TaskStat) SetEndTs(ts time.Time) { - s.EndTs = ts -} - -func (s *TaskStat) GetWaitDuration() (d int64) { - return s.WaitDuration -} - -func (s *TaskStat) SetWaitDuration(d int64) { - s.WaitDuration = d -} - -func (s *TaskStat) GetRuntimeDuration() (d int64) { - return s.RuntimeDuration -} - -func (s *TaskStat) SetRuntimeDuration(d int64) { - s.RuntimeDuration = d -} - -func (s *TaskStat) GetTotalDuration() (d int64) { - return s.WaitDuration + s.RuntimeDuration -} - -func (s *TaskStat) SetTotalDuration(d int64) { - s.TotalDuration = d -} - -func (s *TaskStat) GetResultCount() (c int64) { - return s.ResultCount -} - -func (s *TaskStat) SetResultCount(c int64) { - s.ResultCount = c -} - -func (s *TaskStat) GetErrorLogCount() (c int64) { - return s.ErrorLogCount -} - -func (s *TaskStat) SetErrorLogCount(c int64) { - s.ErrorLogCount = c -} - -type TaskStatList []TaskStat - -func (l *TaskStatList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/token.go b/core/models/models/token.go deleted file mode 100644 index 7a957e3e..00000000 --- a/core/models/models/token.go +++ /dev/null @@ -1,30 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Token struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Name string `json:"name" bson:"name"` - Token string `json:"token" bson:"token"` -} - -func (t *Token) GetId() (id primitive.ObjectID) { - return t.Id -} - -func (t *Token) SetId(id primitive.ObjectID) { - t.Id = id -} - -type TokenList []Token - -func (l *TokenList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/user.go b/core/models/models/user.go deleted file mode 100644 index c03bf388..00000000 --- a/core/models/models/user.go +++ /dev/null @@ -1,59 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type User struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Username string `json:"username" bson:"username"` - Password string `json:"password,omitempty" bson:"-"` - Role string `json:"role" bson:"role"` - Email string `json:"email" bson:"email"` - //Setting UserSetting `json:"setting" bson:"setting"` -} - -func (u *User) GetId() (id primitive.ObjectID) { - return u.Id -} - -func (u *User) SetId(id primitive.ObjectID) { - u.Id = id -} - -func (u *User) GetUsername() (name string) { - return u.Username -} - -func (u *User) GetPassword() (p string) { - return u.Password -} - -func (u *User) GetRole() (r string) { - return u.Role -} - -func (u *User) GetEmail() (email string) { - return u.Email -} - -//type UserSetting struct { -// NotificationTrigger string `json:"notification_trigger" bson:"notification_trigger"` -// DingTalkRobotWebhook string `json:"ding_talk_robot_webhook" bson:"ding_talk_robot_webhook"` -// WechatRobotWebhook string `json:"wechat_robot_webhook" bson:"wechat_robot_webhook"` -// EnabledNotifications []string `json:"enabled_notifications" bson:"enabled_notifications"` -// ErrorRegexPattern string `json:"error_regex_pattern" bson:"error_regex_pattern"` -// MaxErrorLog int `json:"max_error_log" bson:"max_error_log"` -// LogExpireDuration int64 `json:"log_expire_duration" bson:"log_expire_duration"` -//} - -type UserList []User - -func (l *UserList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/user_role.go b/core/models/models/user_role.go deleted file mode 100644 index a706bec6..00000000 --- a/core/models/models/user_role.go +++ /dev/null @@ -1,30 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type UserRole struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - UserId primitive.ObjectID `json:"user_id" bson:"user_id"` -} - -func (ur *UserRole) GetId() (id primitive.ObjectID) { - return ur.Id -} - -func (ur *UserRole) SetId(id primitive.ObjectID) { - ur.Id = id -} - -type UserRoleList []UserRole - -func (l *UserRoleList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/models/utils_binder_legacy.go b/core/models/models/utils_binder_legacy.go deleted file mode 100644 index 0c46d791..00000000 --- a/core/models/models/utils_binder_legacy.go +++ /dev/null @@ -1,96 +0,0 @@ -package models - -//func AssignFields(d interface{}, fieldIds ...interfaces.ModelId) (res interface{}, err error) { -// return assignFields(d, fieldIds...) -//} -// -//func AssignListFields(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) { -// return assignListFields(list, fieldIds...) -//} -// -//func AssignListFieldsAsPtr(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) { -// return assignListFieldsAsPtr(list, fieldIds...) -//} -// -//func assignFields(d interface{}, fieldIds ...interfaces.ModelId) (res interface{}, err error) { -// doc, ok := d.(interfaces.Model) -// if !ok { -// return nil, errors.ErrorModelInvalidType -// } -// if len(fieldIds) == 0 { -// return doc, nil -// } -// for _, fid := range fieldIds { -// switch fid { -// case interfaces.ModelIdTag: -// // convert interface -// d, ok := doc.(interfaces.ModelWithTags) -// if !ok { -// return nil, errors.ErrorModelInvalidType -// } -// -// // attempt to get artifact -// a, err := doc.GetArtifact() -// if err != nil { -// return nil, err -// } -// -// // skip if no artifact found -// if a == nil { -// return d, nil -// } -// -// // assign tags -// tags, err := a.GetTags() -// if err != nil { -// return nil, err -// } -// d.SetTags(tags) -// -// return d, nil -// } -// } -// return doc, nil -//} -// -//func _assignListFields(asPtr bool, list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) { -// vList := reflect.ValueOf(list) -// if vList.Kind() != reflect.Array && -// vList.Kind() != reflect.Slice { -// return res, errors.ErrorModelInvalidType -// } -// for i := 0; i < vList.Len(); i++ { -// vItem := vList.Index(i) -// var item interface{} -// if vItem.CanAddr() { -// item = vItem.Addr().Interface() -// } else { -// item = vItem.Interface() -// } -// doc, ok := item.(interfaces.Model) -// if !ok { -// return res, errors.ErrorModelInvalidType -// } -// ptr, err := assignFields(doc, fieldIds...) -// if err != nil { -// return res, err -// } -// v := reflect.ValueOf(ptr) -// if !asPtr { -// // non-pointer item -// res.Add(v.Elem().Interface()) -// } else { -// // pointer item -// res.Add(v.Interface()) -// } -// } -// return res, nil -//} -// -//func assignListFields(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) { -// return _assignListFields(false, list, fieldIds...) -//} -// -//func assignListFieldsAsPtr(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) { -// return _assignListFields(true, list, fieldIds...) -//} diff --git a/core/models/models/utils_col.go b/core/models/models/utils_col.go deleted file mode 100644 index b5971796..00000000 --- a/core/models/models/utils_col.go +++ /dev/null @@ -1,10 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/utils/binders" -) - -func GetModelColName(id interfaces.ModelId) (colName string) { - return binders.NewColNameBinder(id).MustBindString() -} diff --git a/core/models/models/utils_model_map.go b/core/models/models/utils_model_map.go deleted file mode 100644 index f03ce11e..00000000 --- a/core/models/models/utils_model_map.go +++ /dev/null @@ -1,95 +0,0 @@ -package models - -type ModelMap struct { - Artifact Artifact - Tag Tag - Node Node - Project Project - Spider Spider - Task Task - Job Job - Schedule Schedule - User User - Setting Setting - Token Token - Variable Variable - TaskQueueItem TaskQueueItem - TaskStat TaskStat - SpiderStat SpiderStat - DataSource DataSource - DataCollection DataCollection - Result Result - Password Password - ExtraValue ExtraValue - Git Git - Role Role - UserRole UserRole - Permission Permission - RolePermission RolePermission - Environment Environment - DependencySetting DependencySetting -} - -type ModelListMap struct { - Artifacts ArtifactList - Tags TagList - Nodes NodeList - Projects ProjectList - Spiders SpiderList - Tasks TaskList - Jobs JobList - Schedules ScheduleList - Users UserList - Settings SettingList - Tokens TokenList - Variables VariableList - TaskQueueItems TaskQueueItemList - TaskStats TaskStatList - SpiderStats SpiderStatList - DataSources DataSourceList - DataCollections DataCollectionList - Results ResultList - Passwords PasswordList - ExtraValues ExtraValueList - Gits GitList - Roles RoleList - UserRoles UserRoleList - PermissionList PermissionList - RolePermissionList RolePermissionList - Environments EnvironmentList - DependencySettings DependencySettingList -} - -func NewModelMap() (m *ModelMap) { - return &ModelMap{} -} - -func NewModelListMap() (m *ModelListMap) { - return &ModelListMap{ - Artifacts: ArtifactList{}, - Tags: TagList{}, - Nodes: NodeList{}, - Projects: ProjectList{}, - Spiders: SpiderList{}, - Tasks: TaskList{}, - Jobs: JobList{}, - Schedules: ScheduleList{}, - Users: UserList{}, - Settings: SettingList{}, - Tokens: TokenList{}, - Variables: VariableList{}, - TaskQueueItems: TaskQueueItemList{}, - TaskStats: TaskStatList{}, - SpiderStats: SpiderStatList{}, - DataSources: DataSourceList{}, - DataCollections: DataCollectionList{}, - Results: ResultList{}, - Passwords: PasswordList{}, - ExtraValues: ExtraValueList{}, - Gits: GitList{}, - Roles: RoleList{}, - PermissionList: PermissionList{}, - RolePermissionList: RolePermissionList{}, - Environments: EnvironmentList{}, - } -} diff --git a/core/models/models/utils_tag.go b/core/models/models/utils_tag.go deleted file mode 100644 index c5c070c3..00000000 --- a/core/models/models/utils_tag.go +++ /dev/null @@ -1,34 +0,0 @@ -package models - -import ( - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/trace" -) - -func convertInterfacesToTags(tags []interfaces.Tag) (res []Tag) { - if tags == nil { - return nil - } - for _, t := range tags { - tag, ok := t.(*Tag) - if !ok { - log.Warnf("%v: cannot convert tag", trace.TraceError(errors.ErrorModelInvalidType)) - return nil - } - if tag == nil { - log.Warnf("%v: cannot convert tag", trace.TraceError(errors.ErrorModelInvalidType)) - return nil - } - res = append(res, *tag) - } - return res -} - -func convertTagsToInterfaces(tags []Tag) (res []interfaces.Tag) { - for _, t := range tags { - res = append(res, &t) - } - return res -} diff --git a/core/models/models/variable.go b/core/models/models/variable.go deleted file mode 100644 index 08e0d15d..00000000 --- a/core/models/models/variable.go +++ /dev/null @@ -1,31 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Variable struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Key string `json:"key" bson:"key"` - Value string `json:"value" bson:"value"` - Remark string `json:"remark" bson:"remark"` -} - -func (v *Variable) GetId() (id primitive.ObjectID) { - return v.Id -} - -func (v *Variable) SetId(id primitive.ObjectID) { - v.Id = id -} - -type VariableList []Variable - -func (l *VariableList) GetModels() (res []interfaces.Model) { - for i := range *l { - d := (*l)[i] - res = append(res, &d) - } - return res -} diff --git a/core/models/service/artifact_service.go b/core/models/service/artifact_service.go deleted file mode 100644 index 8155942a..00000000 --- a/core/models/service/artifact_service.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeArtifact(d interface{}, err error) (res *models2.Artifact, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Artifact) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetArtifactById(id primitive.ObjectID) (res *models2.Artifact, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdArtifact).GetById(id) - return convertTypeArtifact(d, err) -} - -func (svc *Service) GetArtifact(query bson.M, opts *mongo.FindOptions) (res *models2.Artifact, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdArtifact).Get(query, opts) - return convertTypeArtifact(d, err) -} - -func (svc *Service) GetArtifactList(query bson.M, opts *mongo.FindOptions) (res []models2.Artifact, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdArtifact).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.Artifact) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/base_service.go b/core/models/service/base_service.go deleted file mode 100644 index 20d2362a..00000000 --- a/core/models/service/base_service.go +++ /dev/null @@ -1,420 +0,0 @@ -package service - -import ( - "encoding/json" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "reflect" - "strings" - "sync" - "time" -) - -type BaseService struct { - id interfaces.ModelId - col *mongo.Col -} - -func (svc *BaseService) GetModelId() (id interfaces.ModelId) { - return svc.id -} - -func (svc *BaseService) SetModelId(id interfaces.ModelId) { - svc.id = id -} - -func (svc *BaseService) GetCol() (col *mongo.Col) { - return svc.col -} - -func (svc *BaseService) SetCol(col *mongo.Col) { - svc.col = col -} - -func (svc *BaseService) GetById(id primitive.ObjectID) (res interfaces.Model, err error) { - // find result - fr := svc.findId(id) - - // bind - return NewBasicBinder(svc.id, fr).Bind() -} - -func (svc *BaseService) Get(query bson.M, opts *mongo.FindOptions) (res interfaces.Model, err error) { - // find result - fr := svc.find(query, opts) - - // bind - return NewBasicBinder(svc.id, fr).Bind() -} - -func (svc *BaseService) GetList(query bson.M, opts *mongo.FindOptions) (l interfaces.List, err error) { - // find result - tic := time.Now() - log.Debugf("baseService.GetMany -> svc.find:start") - log.Debugf("baseService.GetMany -> svc.id: %v", svc.id) - log.Debugf("baseService.GetMany -> svc.col.GetName(): %v", svc.col.GetName()) - log.Debugf("baseService.GetMany -> query: %v", query) - log.Debugf("baseService.GetMany -> opts: %v", opts) - fr := svc.find(query, opts) - log.Debugf("baseService.GetMany -> svc.find:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds()) - - // bind - return NewListBinder(svc.id, fr).Bind() -} - -func (svc *BaseService) DeleteById(id primitive.ObjectID, args ...interface{}) (err error) { - return svc.deleteId(id, args...) -} - -func (svc *BaseService) Delete(query bson.M, args ...interface{}) (err error) { - return svc.delete(query) -} - -func (svc *BaseService) DeleteList(query bson.M, args ...interface{}) (err error) { - return svc.deleteList(query) -} - -func (svc *BaseService) ForceDeleteList(query bson.M, args ...interface{}) (err error) { - return svc.forceDeleteList(query) -} - -func (svc *BaseService) UpdateById(id primitive.ObjectID, update bson.M, args ...interface{}) (err error) { - return svc.updateId(id, update) -} - -func (svc *BaseService) Update(query bson.M, update bson.M, fields []string, args ...interface{}) (err error) { - return svc.update(query, update, fields) -} - -func (svc *BaseService) UpdateDoc(query bson.M, doc interfaces.Model, fields []string, args ...interface{}) (err error) { - return svc.update(query, doc, fields) -} - -func (svc *BaseService) Insert(u interfaces.User, docs ...interface{}) (err error) { - log.Debugf("baseService.Insert -> svc.col.GetName(): %v", svc.col.GetName()) - log.Debugf("baseService.Insert -> docs: %v", docs) - return svc.insert(u, docs...) -} - -func (svc *BaseService) Count(query bson.M) (total int, err error) { - return svc.count(query) -} - -func (svc *BaseService) findId(id primitive.ObjectID) (fr *mongo.FindResult) { - if svc.col == nil { - return mongo.NewFindResultWithError(constants.ErrMissingCol) - } - return svc.col.FindId(id) -} - -func (svc *BaseService) find(query bson.M, opts *mongo.FindOptions) (fr *mongo.FindResult) { - if svc.col == nil { - return mongo.NewFindResultWithError(constants.ErrMissingCol) - } - return svc.col.Find(query, opts) -} - -func (svc *BaseService) deleteId(id primitive.ObjectID, args ...interface{}) (err error) { - if svc.col == nil { - return trace.TraceError(constants.ErrMissingCol) - } - fr := svc.findId(id) - doc, err := NewBasicBinder(svc.id, fr).Bind() - if err != nil { - return err - } - return delegate.NewModelDelegate(doc, svc._getUserFromArgs(args...)).Delete() -} - -func (svc *BaseService) delete(query bson.M, args ...interface{}) (err error) { - if svc.col == nil { - return trace.TraceError(constants.ErrMissingCol) - } - var doc models2.BaseModel - if err := svc.find(query, nil).One(&doc); err != nil { - return err - } - return svc.deleteId(doc.Id, svc._getUserFromArgs(args...)) -} - -func (svc *BaseService) deleteList(query bson.M, args ...interface{}) (err error) { - if svc.col == nil { - return trace.TraceError(constants.ErrMissingCol) - } - fr := svc.find(query, nil) - list, err := NewListBinder(svc.id, fr).Bind() - if err != nil { - return err - } - for _, doc := range list.GetModels() { - if err := delegate.NewModelDelegate(doc, svc._getUserFromArgs(args...)).Delete(); err != nil { - return err - } - } - return nil -} - -func (svc *BaseService) forceDeleteList(query bson.M, args ...interface{}) (err error) { - return svc.col.Delete(query) -} - -func (svc *BaseService) count(query bson.M) (total int, err error) { - if svc.col == nil { - return total, trace.TraceError(constants.ErrMissingCol) - } - return svc.col.Count(query) -} - -func (svc *BaseService) update(query bson.M, update interface{}, fields []string, args ...interface{}) (err error) { - update, err = svc._getUpdateBsonM(update, fields) - if err != nil { - return err - } - return svc._update(query, update, svc._getUserFromArgs(args...)) -} - -func (svc *BaseService) updateId(id primitive.ObjectID, update interface{}, args ...interface{}) (err error) { - update, err = svc._getUpdateBsonM(update, nil) - if err != nil { - return err - } - return svc._updateById(id, update, svc._getUserFromArgs(args...)) -} - -func (svc *BaseService) insert(u interfaces.User, docs ...interface{}) (err error) { - // validate col - if svc.col == nil { - return trace.TraceError(constants.ErrMissingCol) - } - - // iterate docs - for i, doc := range docs { - switch doc.(type) { - case map[string]interface{}: - // doc type: map[string]interface{}, need to handle _id - d := doc.(map[string]interface{}) - vId, ok := d["_id"] - if !ok { - // _id not exists - d["_id"] = primitive.NewObjectID() - } else { - // _id exists - switch vId.(type) { - case string: - // _id type: string - sId, ok := vId.(string) - if ok { - d["_id"], err = primitive.ObjectIDFromHex(sId) - if err != nil { - return trace.TraceError(err) - } - } - case primitive.ObjectID: - // _id type: primitive.ObjectID - // do nothing - default: - return trace.TraceError(errors.ErrorModelInvalidType) - } - } - } - docs[i] = doc - } - - // perform insert - ids, err := svc.col.InsertMany(docs) - if err != nil { - return err - } - - // upsert artifacts - query := bson.M{ - "_id": bson.M{ - "$in": ids, - }, - } - fr := svc.col.Find(query, nil) - list, err := NewListBinder(svc.id, fr).Bind() - for _, doc := range list.GetModels() { - // upsert artifact when performing model delegate save - if err := delegate.NewModelDelegate(doc, u).Save(); err != nil { - return err - } - } - - return nil -} - -func (svc *BaseService) _update(query bson.M, update interface{}, args ...interface{}) (err error) { - // ids of query - var ids []primitive.ObjectID - list, err := NewListBinder(svc.id, svc.find(query, nil)).Bind() - if err != nil { - return err - } - for _, doc := range list.GetModels() { - ids = append(ids, doc.GetId()) - } - - // update model objects - if err := svc.col.Update(query, update); err != nil { - return err - } - - // update artifacts - u := svc._getUserFromArgs(args...) - return mongo.GetMongoCol(interfaces.ModelColNameArtifact).Update(query, svc._getUpdateArtifactUpdate(u)) -} - -func (svc *BaseService) _updateById(id primitive.ObjectID, update interface{}, args ...interface{}) (err error) { - // update model object - if err := svc.col.UpdateId(id, update); err != nil { - return err - } - - // update artifact - u := svc._getUserFromArgs(args...) - return mongo.GetMongoCol(interfaces.ModelColNameArtifact).UpdateId(id, svc._getUpdateArtifactUpdate(u)) -} - -func (svc *BaseService) _getUpdateBsonM(update interface{}, fields []string) (res bson.M, err error) { - switch update.(type) { - case interfaces.Model: - // convert to bson.M - var updateBsonM bson.M - bytes, err := json.Marshal(&update) - if err != nil { - return nil, err - } - if err := json.Unmarshal(bytes, &updateBsonM); err != nil { - return nil, err - } - return svc._getUpdateBsonM(updateBsonM, fields) - - case bson.M: - // convert to bson.M - updateBsonM := update.(bson.M) - - // filter fields if not nil - if fields != nil { - // fields map - fieldsMap := map[string]bool{} - for _, f := range fields { - fieldsMap[f] = true - } - - // remove unselected fields - for k := range updateBsonM { - if _, ok := fieldsMap[k]; !ok { - delete(updateBsonM, k) - } - } - } - - // normalize update bson.M - if !svc._containsDollar(updateBsonM) { - if _, ok := updateBsonM["$set"]; !ok { - updateBsonM = bson.M{ - "$set": updateBsonM, - } - } - } - - return updateBsonM, nil - } - - v := reflect.ValueOf(update) - switch v.Kind() { - case reflect.Struct: - if v.CanAddr() { - update = v.Addr().Interface() - return svc._getUpdateBsonM(update, fields) - } - return nil, errors.ErrorModelInvalidType - default: - return nil, errors.ErrorModelInvalidType - } -} - -func (svc *BaseService) _getUpdateArtifactUpdate(u interfaces.User) (res bson.M) { - var uid primitive.ObjectID - if u != nil { - uid = u.GetId() - } - return bson.M{ - "$set": bson.M{ - "_sys.update_ts": time.Now(), - "_sys.update_uid": uid, - }, - } -} - -func (svc *BaseService) _getUserFromArgs(args ...interface{}) (u interfaces.User) { - return utils.GetUserFromArgs(args...) -} - -func (svc *BaseService) _containsDollar(updateBsonM bson.M) (ok bool) { - for k := range updateBsonM { - if strings.HasPrefix(k, "$") { - return true - } - } - return false -} - -func NewBaseService(id interfaces.ModelId, opts ...BaseServiceOption) (svc2 interfaces.ModelBaseService) { - // service - svc := &BaseService{ - id: id, - } - - // apply options - for _, opt := range opts { - opt(svc) - } - - // get collection name if not set - if svc.GetCol() == nil { - colName := models2.GetModelColName(id) - svc.SetCol(mongo.GetMongoCol(colName)) - } - - return svc -} - -var store = sync.Map{} - -func GetBaseService(id interfaces.ModelId) (svc interfaces.ModelBaseService) { - res, ok := store.Load(id) - if ok { - svc, ok = res.(interfaces.ModelBaseService) - if ok { - return svc - } - } - svc = NewBaseService(id) - store.Store(id, svc) - return svc -} - -func GetBaseServiceByColName(id interfaces.ModelId, colName string) (svc interfaces.ModelBaseService) { - res, ok := store.Load(colName) - if ok { - svc, ok = res.(interfaces.ModelBaseService) - if ok { - return svc - } - } - col := mongo.GetMongoCol(colName) - svc = NewBaseService(id, WithBaseServiceCol(col)) - store.Store(colName, svc) - return svc -} diff --git a/core/models/service/binder_basic.go b/core/models/service/binder_basic.go deleted file mode 100644 index df0fb643..00000000 --- a/core/models/service/binder_basic.go +++ /dev/null @@ -1,92 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" -) - -func NewBasicBinder(id interfaces.ModelId, fr *mongo.FindResult) (b interfaces.ModelBinder) { - return &BasicBinder{ - id: id, - fr: fr, - m: models.NewModelMap(), - } -} - -type BasicBinder struct { - id interfaces.ModelId - fr *mongo.FindResult - m *models.ModelMap -} - -func (b *BasicBinder) Bind() (res interfaces.Model, err error) { - m := b.m - - switch b.id { - case interfaces.ModelIdArtifact: - return b.Process(&m.Artifact) - case interfaces.ModelIdTag: - return b.Process(&m.Tag) - case interfaces.ModelIdNode: - return b.Process(&m.Node) - case interfaces.ModelIdProject: - return b.Process(&m.Project) - case interfaces.ModelIdSpider: - return b.Process(&m.Spider) - case interfaces.ModelIdTask: - return b.Process(&m.Task) - case interfaces.ModelIdJob: - return b.Process(&m.Job) - case interfaces.ModelIdSchedule: - return b.Process(&m.Schedule) - case interfaces.ModelIdUser: - return b.Process(&m.User) - case interfaces.ModelIdSetting: - return b.Process(&m.Setting) - case interfaces.ModelIdToken: - return b.Process(&m.Token) - case interfaces.ModelIdVariable: - return b.Process(&m.Variable) - case interfaces.ModelIdTaskQueue: - return b.Process(&m.TaskQueueItem) - case interfaces.ModelIdTaskStat: - return b.Process(&m.TaskStat) - case interfaces.ModelIdSpiderStat: - return b.Process(&m.SpiderStat) - case interfaces.ModelIdDataSource: - return b.Process(&m.DataSource) - case interfaces.ModelIdDataCollection: - return b.Process(&m.DataCollection) - case interfaces.ModelIdResult: - return b.Process(&m.Result) - case interfaces.ModelIdPassword: - return b.Process(&m.Password) - case interfaces.ModelIdExtraValue: - return b.Process(&m.ExtraValue) - case interfaces.ModelIdGit: - return b.Process(&m.Git) - case interfaces.ModelIdRole: - return b.Process(&m.Role) - case interfaces.ModelIdUserRole: - return b.Process(&m.UserRole) - case interfaces.ModelIdPermission: - return b.Process(&m.Permission) - case interfaces.ModelIdRolePermission: - return b.Process(&m.RolePermission) - case interfaces.ModelIdEnvironment: - return b.Process(&m.Environment) - case interfaces.ModelIdDependencySetting: - return b.Process(&m.DependencySetting) - default: - return nil, errors.ErrorModelInvalidModelId - } -} - -func (b *BasicBinder) Process(d interfaces.Model) (res interfaces.Model, err error) { - if err := b.fr.One(d); err != nil { - return nil, err - } - return d, nil -} diff --git a/core/models/service/binder_list.go b/core/models/service/binder_list.go deleted file mode 100644 index a7a8f50d..00000000 --- a/core/models/service/binder_list.go +++ /dev/null @@ -1,93 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" -) - -func NewListBinder(id interfaces.ModelId, fr *mongo.FindResult) (b interfaces.ModelListBinder) { - return &ListBinder{ - id: id, - m: models.NewModelListMap(), - fr: fr, - b: NewBasicBinder(id, fr), - } -} - -type ListBinder struct { - id interfaces.ModelId - m *models.ModelListMap - fr *mongo.FindResult - b interfaces.ModelBinder -} - -func (b *ListBinder) Bind() (l interfaces.List, err error) { - m := b.m - - switch b.id { - case interfaces.ModelIdArtifact: - return b.Process(&m.Artifacts) - case interfaces.ModelIdTag: - return b.Process(&m.Tags) - case interfaces.ModelIdNode: - return b.Process(&m.Nodes) - case interfaces.ModelIdProject: - return b.Process(&m.Projects) - case interfaces.ModelIdSpider: - return b.Process(&m.Spiders) - case interfaces.ModelIdTask: - return b.Process(&m.Tasks) - case interfaces.ModelIdSchedule: - return b.Process(&m.Schedules) - case interfaces.ModelIdUser: - return b.Process(&m.Users) - case interfaces.ModelIdSetting: - return b.Process(&m.Settings) - case interfaces.ModelIdToken: - return b.Process(&m.Tokens) - case interfaces.ModelIdVariable: - return b.Process(&m.Variables) - case interfaces.ModelIdTaskQueue: - return b.Process(&m.TaskQueueItems) - case interfaces.ModelIdTaskStat: - return b.Process(&m.TaskStats) - case interfaces.ModelIdSpiderStat: - return b.Process(&m.SpiderStats) - case interfaces.ModelIdDataSource: - return b.Process(&m.DataSources) - case interfaces.ModelIdDataCollection: - return b.Process(&m.DataCollections) - case interfaces.ModelIdResult: - return b.Process(&m.Results) - case interfaces.ModelIdPassword: - return b.Process(&m.Passwords) - case interfaces.ModelIdExtraValue: - return b.Process(&m.ExtraValues) - case interfaces.ModelIdGit: - return b.Process(&m.Gits) - case interfaces.ModelIdRole: - return b.Process(&m.Roles) - case interfaces.ModelIdUserRole: - return b.Process(&m.UserRoles) - case interfaces.ModelIdPermission: - return b.Process(&m.PermissionList) - case interfaces.ModelIdRolePermission: - return b.Process(&m.RolePermissionList) - case interfaces.ModelIdEnvironment: - return b.Process(&m.Environments) - case interfaces.ModelIdDependencySetting: - return b.Process(&m.DependencySettings) - default: - return l, errors.ErrorModelInvalidModelId - } -} - -func (b *ListBinder) Process(d interface{}) (l interfaces.List, err error) { - if err := b.fr.All(d); err != nil { - return l, trace.TraceError(err) - } - return d.(interfaces.List), nil -} diff --git a/core/models/service/data_collection_service.go b/core/models/service/data_collection_service.go deleted file mode 100644 index 6b2fd284..00000000 --- a/core/models/service/data_collection_service.go +++ /dev/null @@ -1,45 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeDataCollection(d interface{}, err error) (res *models2.DataCollection, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.DataCollection) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetDataCollectionById(id primitive.ObjectID) (res *models2.DataCollection, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdDataCollection).GetById(id) - return convertTypeDataCollection(d, err) -} - -func (svc *Service) GetDataCollection(query bson.M, opts *mongo.FindOptions) (res *models2.DataCollection, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdDataCollection).Get(query, opts) - return convertTypeDataCollection(d, err) -} - -func (svc *Service) GetDataCollectionList(query bson.M, opts *mongo.FindOptions) (res []models2.DataCollection, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdDataCollection).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.DataCollection) - res = append(res, *d) - } - return res, nil -} - -func (svc *Service) GetDataCollectionByName(name string, opts *mongo.FindOptions) (res *models2.DataCollection, err error) { - query := bson.M{"name": name} - return svc.GetDataCollection(query, opts) -} diff --git a/core/models/service/data_source_service.go b/core/models/service/data_source_service.go deleted file mode 100644 index 44ecb387..00000000 --- a/core/models/service/data_source_service.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeDataSource(d interface{}, err error) (res *models2.DataSource, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.DataSource) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetDataSourceById(id primitive.ObjectID) (res *models2.DataSource, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdDataSource).GetById(id) - return convertTypeDataSource(d, err) -} - -func (svc *Service) GetDataSource(query bson.M, opts *mongo.FindOptions) (res *models2.DataSource, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdDataSource).Get(query, opts) - return convertTypeDataSource(d, err) -} - -func (svc *Service) GetDataSourceList(query bson.M, opts *mongo.FindOptions) (res []models2.DataSource, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdDataSource).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.DataSource) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/dependency_setting_service.go b/core/models/service/dependency_setting_service.go deleted file mode 100644 index 41600285..00000000 --- a/core/models/service/dependency_setting_service.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeDependencySetting(d interface{}, err error) (res *models2.DependencySetting, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.DependencySetting) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetDependencySettingById(id primitive.ObjectID) (res *models2.DependencySetting, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdDependencySetting).GetById(id) - return convertTypeDependencySetting(d, err) -} - -func (svc *Service) GetDependencySetting(query bson.M, opts *mongo.FindOptions) (res *models2.DependencySetting, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdDependencySetting).Get(query, opts) - return convertTypeDependencySetting(d, err) -} - -func (svc *Service) GetDependencySettingList(query bson.M, opts *mongo.FindOptions) (res []models2.DependencySetting, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdDependencySetting).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.DependencySetting) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/environment_service.go b/core/models/service/environment_service.go deleted file mode 100644 index 5b82aa39..00000000 --- a/core/models/service/environment_service.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeEnvironment(d interface{}, err error) (res *models2.Environment, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Environment) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetEnvironmentById(id primitive.ObjectID) (res *models2.Environment, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdEnvironment).GetById(id) - return convertTypeEnvironment(d, err) -} - -func (svc *Service) GetEnvironment(query bson.M, opts *mongo.FindOptions) (res *models2.Environment, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdEnvironment).Get(query, opts) - return convertTypeEnvironment(d, err) -} - -func (svc *Service) GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []models2.Environment, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdEnvironment).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.Environment) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/extra_value_service.go b/core/models/service/extra_value_service.go deleted file mode 100644 index 5fa1bb6e..00000000 --- a/core/models/service/extra_value_service.go +++ /dev/null @@ -1,45 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeExtraValue(d interface{}, err error) (res *models.ExtraValue, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models.ExtraValue) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetExtraValueById(id primitive.ObjectID) (res *models.ExtraValue, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdExtraValue).GetById(id) - return convertTypeExtraValue(d, err) -} - -func (svc *Service) GetExtraValue(query bson.M, opts *mongo.FindOptions) (res *models.ExtraValue, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdExtraValue).Get(query, opts) - return convertTypeExtraValue(d, err) -} - -func (svc *Service) GetExtraValueList(query bson.M, opts *mongo.FindOptions) (res []models.ExtraValue, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdExtraValue).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models.ExtraValue) - res = append(res, *d) - } - return res, nil -} - -func (svc *Service) GetExtraValueByObjectIdModel(oid primitive.ObjectID, m string, opts *mongo.FindOptions) (res *models.ExtraValue, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdExtraValue).Get(bson.M{"oid": oid, "m": m}, opts) - return convertTypeExtraValue(d, err) -} diff --git a/core/models/service/git_service.go b/core/models/service/git_service.go deleted file mode 100644 index 41ad0997..00000000 --- a/core/models/service/git_service.go +++ /dev/null @@ -1,43 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeGit(d interface{}, err error) (res *models2.Git, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Git) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetGitById(id primitive.ObjectID) (res *models2.Git, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdGit).GetById(id) - return convertTypeGit(d, err) -} - -func (svc *Service) GetGit(query bson.M, opts *mongo.FindOptions) (res *models2.Git, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdGit).Get(query, opts) - return convertTypeGit(d, err) -} - -func (svc *Service) GetGitList(query bson.M, opts *mongo.FindOptions) (res []models2.Git, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdGit).GetList(query, opts) - if l == nil { - return nil, nil - } - for _, doc := range l.GetModels() { - d := doc.(*models2.Git) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/interface.go b/core/models/service/interface.go deleted file mode 100644 index c03ff391..00000000 --- a/core/models/service/interface.go +++ /dev/null @@ -1,105 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type ModelService interface { - interfaces.ModelService - DropAll() (err error) - GetNodeById(id primitive.ObjectID) (res *models.Node, err error) - GetNode(query bson.M, opts *mongo.FindOptions) (res *models.Node, err error) - GetNodeList(query bson.M, opts *mongo.FindOptions) (res []models.Node, err error) - GetNodeByKey(key string, opts *mongo.FindOptions) (res *models.Node, err error) - GetProjectById(id primitive.ObjectID) (res *models.Project, err error) - GetProject(query bson.M, opts *mongo.FindOptions) (res *models.Project, err error) - GetProjectList(query bson.M, opts *mongo.FindOptions) (res []models.Project, err error) - GetArtifactById(id primitive.ObjectID) (res *models.Artifact, err error) - GetArtifact(query bson.M, opts *mongo.FindOptions) (res *models.Artifact, err error) - GetArtifactList(query bson.M, opts *mongo.FindOptions) (res []models.Artifact, err error) - GetTagById(id primitive.ObjectID) (res *models.Tag, err error) - GetTag(query bson.M, opts *mongo.FindOptions) (res *models.Tag, err error) - GetTagList(query bson.M, opts *mongo.FindOptions) (res []models.Tag, err error) - GetTagIds(colName string, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error) - UpdateTagsById(colName string, id primitive.ObjectID, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error) - UpdateTags(colName string, query bson.M, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error) - GetJobById(id primitive.ObjectID) (res *models.Job, err error) - GetJob(query bson.M, opts *mongo.FindOptions) (res *models.Job, err error) - GetJobList(query bson.M, opts *mongo.FindOptions) (res []models.Job, err error) - GetScheduleById(id primitive.ObjectID) (res *models.Schedule, err error) - GetSchedule(query bson.M, opts *mongo.FindOptions) (res *models.Schedule, err error) - GetScheduleList(query bson.M, opts *mongo.FindOptions) (res []models.Schedule, err error) - GetUserById(id primitive.ObjectID) (res *models.User, err error) - GetUser(query bson.M, opts *mongo.FindOptions) (res *models.User, err error) - GetUserList(query bson.M, opts *mongo.FindOptions) (res []models.User, err error) - GetUserByUsername(username string, opts *mongo.FindOptions) (res *models.User, err error) - GetUserByUsernameWithPassword(username string, opts *mongo.FindOptions) (res *models.User, err error) - GetSettingById(id primitive.ObjectID) (res *models.Setting, err error) - GetSetting(query bson.M, opts *mongo.FindOptions) (res *models.Setting, err error) - GetSettingList(query bson.M, opts *mongo.FindOptions) (res []models.Setting, err error) - GetSettingByKey(key string, opts *mongo.FindOptions) (res *models.Setting, err error) - GetSpiderById(id primitive.ObjectID) (res *models.Spider, err error) - GetSpider(query bson.M, opts *mongo.FindOptions) (res *models.Spider, err error) - GetSpiderList(query bson.M, opts *mongo.FindOptions) (res []models.Spider, err error) - GetTaskById(id primitive.ObjectID) (res *models.Task, err error) - GetTask(query bson.M, opts *mongo.FindOptions) (res *models.Task, err error) - GetTaskList(query bson.M, opts *mongo.FindOptions) (res []models.Task, err error) - GetTokenById(id primitive.ObjectID) (res *models.Token, err error) - GetToken(query bson.M, opts *mongo.FindOptions) (res *models.Token, err error) - GetTokenList(query bson.M, opts *mongo.FindOptions) (res []models.Token, err error) - GetVariableById(id primitive.ObjectID) (res *models.Variable, err error) - GetVariable(query bson.M, opts *mongo.FindOptions) (res *models.Variable, err error) - GetVariableList(query bson.M, opts *mongo.FindOptions) (res []models.Variable, err error) - GetVariableByKey(key string, opts *mongo.FindOptions) (res *models.Variable, err error) - GetTaskQueueItemById(id primitive.ObjectID) (res *models.TaskQueueItem, err error) - GetTaskQueueItem(query bson.M, opts *mongo.FindOptions) (res *models.TaskQueueItem, err error) - GetTaskQueueItemList(query bson.M, opts *mongo.FindOptions) (res []models.TaskQueueItem, err error) - GetTaskStatById(id primitive.ObjectID) (res *models.TaskStat, err error) - GetTaskStat(query bson.M, opts *mongo.FindOptions) (res *models.TaskStat, err error) - GetTaskStatList(query bson.M, opts *mongo.FindOptions) (res []models.TaskStat, err error) - GetSpiderStatById(id primitive.ObjectID) (res *models.SpiderStat, err error) - GetSpiderStat(query bson.M, opts *mongo.FindOptions) (res *models.SpiderStat, err error) - GetSpiderStatList(query bson.M, opts *mongo.FindOptions) (res []models.SpiderStat, err error) - GetDataSourceById(id primitive.ObjectID) (res *models.DataSource, err error) - GetDataSource(query bson.M, opts *mongo.FindOptions) (res *models.DataSource, err error) - GetDataSourceList(query bson.M, opts *mongo.FindOptions) (res []models.DataSource, err error) - GetDataCollectionById(id primitive.ObjectID) (res *models.DataCollection, err error) - GetDataCollection(query bson.M, opts *mongo.FindOptions) (res *models.DataCollection, err error) - GetDataCollectionList(query bson.M, opts *mongo.FindOptions) (res []models.DataCollection, err error) - GetDataCollectionByName(name string, opts *mongo.FindOptions) (res *models.DataCollection, err error) - GetPasswordById(id primitive.ObjectID) (res *models.Password, err error) - GetPassword(query bson.M, opts *mongo.FindOptions) (res *models.Password, err error) - GetPasswordList(query bson.M, opts *mongo.FindOptions) (res []models.Password, err error) - GetExtraValueById(id primitive.ObjectID) (res *models.ExtraValue, err error) - GetExtraValue(query bson.M, opts *mongo.FindOptions) (res *models.ExtraValue, err error) - GetExtraValueList(query bson.M, opts *mongo.FindOptions) (res []models.ExtraValue, err error) - GetExtraValueByObjectIdModel(oid primitive.ObjectID, m string, opts *mongo.FindOptions) (res *models.ExtraValue, err error) - GetGitById(id primitive.ObjectID) (res *models.Git, err error) - GetGit(query bson.M, opts *mongo.FindOptions) (res *models.Git, err error) - GetGitList(query bson.M, opts *mongo.FindOptions) (res []models.Git, err error) - GetRoleById(id primitive.ObjectID) (res *models.Role, err error) - GetRole(query bson.M, opts *mongo.FindOptions) (res *models.Role, err error) - GetRoleList(query bson.M, opts *mongo.FindOptions) (res []models.Role, err error) - GetRoleByName(name string, opts *mongo.FindOptions) (res *models.Role, err error) - GetRoleByKey(key string, opts *mongo.FindOptions) (res *models.Role, err error) - GetUserRoleById(id primitive.ObjectID) (res *models.UserRole, err error) - GetUserRole(query bson.M, opts *mongo.FindOptions) (res *models.UserRole, err error) - GetUserRoleList(query bson.M, opts *mongo.FindOptions) (res []models.UserRole, err error) - GetUserRoleListByUserId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.UserRole, err error) - GetUserRoleListByRoleId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.UserRole, err error) - GetPermissionById(id primitive.ObjectID) (res *models.Permission, err error) - GetPermission(query bson.M, opts *mongo.FindOptions) (res *models.Permission, err error) - GetPermissionList(query bson.M, opts *mongo.FindOptions) (res []models.Permission, err error) - GetPermissionByKey(key string, opts *mongo.FindOptions) (res *models.Permission, err error) - GetRolePermission(query bson.M, opts *mongo.FindOptions) (res *models.RolePermission, err error) - GetRolePermissionList(query bson.M, opts *mongo.FindOptions) (res []models.RolePermission, err error) - GetRolePermissionListByRoleId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.RolePermission, err error) - GetRolePermissionListByPermissionId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.RolePermission, err error) - GetEnvironmentById(id primitive.ObjectID) (res *models.Environment, err error) - GetEnvironment(query bson.M, opts *mongo.FindOptions) (res *models.Environment, err error) - GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []models.Environment, err error) -} diff --git a/core/models/service/job_service.go b/core/models/service/job_service.go deleted file mode 100644 index 2ec74964..00000000 --- a/core/models/service/job_service.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeJob(d interface{}, err error) (res *models2.Job, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Job) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetJobById(id primitive.ObjectID) (res *models2.Job, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdJob).GetById(id) - return convertTypeJob(d, err) -} - -func (svc *Service) GetJob(query bson.M, opts *mongo.FindOptions) (res *models2.Job, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdJob).Get(query, opts) - return convertTypeJob(d, err) -} - -func (svc *Service) GetJobList(query bson.M, opts *mongo.FindOptions) (res []models2.Job, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdJob).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.Job) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/node_service.go b/core/models/service/node_service.go deleted file mode 100644 index f99d6d84..00000000 --- a/core/models/service/node_service.go +++ /dev/null @@ -1,45 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeNode(d interface{}, err error) (res *models2.Node, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Node) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetNodeById(id primitive.ObjectID) (res *models2.Node, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdNode).GetById(id) - return convertTypeNode(d, err) -} - -func (svc *Service) GetNode(query bson.M, opts *mongo.FindOptions) (res *models2.Node, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdNode).Get(query, opts) - return convertTypeNode(d, err) -} - -func (svc *Service) GetNodeList(query bson.M, opts *mongo.FindOptions) (res []models2.Node, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdNode).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.Node) - res = append(res, *d) - } - return res, nil -} - -func (svc *Service) GetNodeByKey(key string, opts *mongo.FindOptions) (res *models2.Node, err error) { - query := bson.M{"key": key} - return svc.GetNode(query, opts) -} diff --git a/core/models/service/options.go b/core/models/service/options.go deleted file mode 100644 index 8677d0c1..00000000 --- a/core/models/service/options.go +++ /dev/null @@ -1,25 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/db/mongo" -) - -type Option func(ModelService) - -type BaseServiceOption func(svc interfaces.ModelBaseService) - -func WithBaseServiceModelId(id interfaces.ModelId) BaseServiceOption { - return func(svc interfaces.ModelBaseService) { - svc.SetModelId(id) - } -} - -func WithBaseServiceCol(col *mongo.Col) BaseServiceOption { - return func(svc interfaces.ModelBaseService) { - _svc, ok := svc.(*BaseService) - if ok { - _svc.SetCol(col) - } - } -} diff --git a/core/models/service/password_service.go b/core/models/service/password_service.go deleted file mode 100644 index 47101522..00000000 --- a/core/models/service/password_service.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypePassword(d interface{}, err error) (res *models2.Password, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Password) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetPasswordById(id primitive.ObjectID) (res *models2.Password, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdPassword).GetById(id) - return convertTypePassword(d, err) -} - -func (svc *Service) GetPassword(query bson.M, opts *mongo.FindOptions) (res *models2.Password, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdPassword).Get(query, opts) - return convertTypePassword(d, err) -} - -func (svc *Service) GetPasswordList(query bson.M, opts *mongo.FindOptions) (res []models2.Password, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdPassword).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.Password) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/permission_service.go b/core/models/service/permission_service.go deleted file mode 100644 index e35d63d6..00000000 --- a/core/models/service/permission_service.go +++ /dev/null @@ -1,48 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypePermission(d interface{}, err error) (res *models2.Permission, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Permission) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetPermissionById(id primitive.ObjectID) (res *models2.Permission, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdPermission).GetById(id) - return convertTypePermission(d, err) -} - -func (svc *Service) GetPermission(query bson.M, opts *mongo.FindOptions) (res *models2.Permission, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdPermission).Get(query, opts) - return convertTypePermission(d, err) -} - -func (svc *Service) GetPermissionList(query bson.M, opts *mongo.FindOptions) (res []models2.Permission, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdPermission).GetList(query, opts) - if err != nil { - return nil, err - } - for _, doc := range l.GetModels() { - d := doc.(*models2.Permission) - res = append(res, *d) - } - return res, nil -} - -func (svc *Service) GetPermissionByKey(key string, opts *mongo.FindOptions) (res *models2.Permission, err error) { - query := bson.M{"key": key} - return svc.GetPermission(query, opts) -} diff --git a/core/models/service/project_service.go b/core/models/service/project_service.go deleted file mode 100644 index 6dd3e10e..00000000 --- a/core/models/service/project_service.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeProject(d interface{}, err error) (res *models2.Project, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Project) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetProjectById(id primitive.ObjectID) (res *models2.Project, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdProject).GetById(id) - return convertTypeProject(d, err) -} - -func (svc *Service) GetProject(query bson.M, opts *mongo.FindOptions) (res *models2.Project, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdProject).Get(query, opts) - return convertTypeProject(d, err) -} - -func (svc *Service) GetProjectList(query bson.M, opts *mongo.FindOptions) (res []models2.Project, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdProject).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.Project) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/role_permission_service.go b/core/models/service/role_permission_service.go deleted file mode 100644 index 5d980d41..00000000 --- a/core/models/service/role_permission_service.go +++ /dev/null @@ -1,51 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeRolePermission(d interface{}, err error) (res *models2.RolePermission, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.RolePermission) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetRolePermissionById(id primitive.ObjectID) (res *models2.RolePermission, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdRolePermission).GetById(id) - return convertTypeRolePermission(d, err) -} - -func (svc *Service) GetRolePermission(query bson.M, opts *mongo.FindOptions) (res *models2.RolePermission, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdRolePermission).Get(query, opts) - return convertTypeRolePermission(d, err) -} - -func (svc *Service) GetRolePermissionList(query bson.M, opts *mongo.FindOptions) (res []models2.RolePermission, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdRolePermission).GetList(query, opts) - if err != nil { - return nil, err - } - for _, doc := range l.GetModels() { - d := doc.(*models2.RolePermission) - res = append(res, *d) - } - return res, nil -} - -func (svc *Service) GetRolePermissionListByRoleId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models2.RolePermission, err error) { - return svc.GetRolePermissionList(bson.M{"role_id": id}, opts) -} - -func (svc *Service) GetRolePermissionListByPermissionId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models2.RolePermission, err error) { - return svc.GetRolePermissionList(bson.M{"permission_id": id}, opts) -} diff --git a/core/models/service/role_service.go b/core/models/service/role_service.go deleted file mode 100644 index c456dbde..00000000 --- a/core/models/service/role_service.go +++ /dev/null @@ -1,53 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeRole(d interface{}, err error) (res *models2.Role, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Role) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetRoleById(id primitive.ObjectID) (res *models2.Role, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdRole).GetById(id) - return convertTypeRole(d, err) -} - -func (svc *Service) GetRole(query bson.M, opts *mongo.FindOptions) (res *models2.Role, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdRole).Get(query, opts) - return convertTypeRole(d, err) -} - -func (svc *Service) GetRoleList(query bson.M, opts *mongo.FindOptions) (res []models2.Role, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdRole).GetList(query, opts) - if err != nil { - return nil, err - } - for _, doc := range l.GetModels() { - d := doc.(*models2.Role) - res = append(res, *d) - } - return res, nil -} - -func (svc *Service) GetRoleByName(name string, opts *mongo.FindOptions) (res *models2.Role, err error) { - query := bson.M{"name": name} - return svc.GetRole(query, opts) -} - -func (svc *Service) GetRoleByKey(key string, opts *mongo.FindOptions) (res *models2.Role, err error) { - query := bson.M{"key": key} - return svc.GetRole(query, opts) -} diff --git a/core/models/service/schedule_service.go b/core/models/service/schedule_service.go deleted file mode 100644 index 9d04cbf2..00000000 --- a/core/models/service/schedule_service.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeSchedule(d interface{}, err error) (res *models2.Schedule, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Schedule) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetScheduleById(id primitive.ObjectID) (res *models2.Schedule, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdSchedule).GetById(id) - return convertTypeSchedule(d, err) -} - -func (svc *Service) GetSchedule(query bson.M, opts *mongo.FindOptions) (res *models2.Schedule, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdSchedule).Get(query, opts) - return convertTypeSchedule(d, err) -} - -func (svc *Service) GetScheduleList(query bson.M, opts *mongo.FindOptions) (res []models2.Schedule, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdSchedule).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.Schedule) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/service.go b/core/models/service/service.go deleted file mode 100644 index bb62246b..00000000 --- a/core/models/service/service.go +++ /dev/null @@ -1,62 +0,0 @@ -package service - -import ( - "context" - "github.com/crawlab-team/crawlab/core/color" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - mongo2 "go.mongodb.org/mongo-driver/mongo" -) - -type Service struct { - env string - colorSvc interfaces.ColorService -} - -func (svc *Service) DropAll() (err error) { - db := mongo.GetMongoDb("") - colNames, err := db.ListCollectionNames(context.Background(), bson.M{}) - if err != nil { - if err == mongo2.ErrNoDocuments { - return nil - } - return err - } - for _, colName := range colNames { - col := db.Collection(colName) - if err := col.Drop(context.Background()); err != nil { - return err - } - } - return nil -} - -func (svc *Service) GetBaseService(id interfaces.ModelId) (svc2 interfaces.ModelBaseService) { - return GetBaseService(id) -} - -func NewService() (svc2 ModelService, err error) { - // service - svc := &Service{} - - svc.colorSvc, err = color.NewService() - if err != nil { - return nil, err - } - - return svc, nil -} - -var modelSvc ModelService - -func GetService() (svc ModelService, err error) { - if modelSvc != nil { - return modelSvc, nil - } - modelSvc, err = NewService() - if err != nil { - return nil, err - } - return modelSvc, nil -} diff --git a/core/models/service/setting_service.go b/core/models/service/setting_service.go deleted file mode 100644 index 41dd5c45..00000000 --- a/core/models/service/setting_service.go +++ /dev/null @@ -1,45 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeSetting(d interface{}, err error) (res *models2.Setting, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Setting) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetSettingById(id primitive.ObjectID) (res *models2.Setting, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdSetting).GetById(id) - return convertTypeSetting(d, err) -} - -func (svc *Service) GetSetting(query bson.M, opts *mongo.FindOptions) (res *models2.Setting, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdSetting).Get(query, opts) - return convertTypeSetting(d, err) -} - -func (svc *Service) GetSettingList(query bson.M, opts *mongo.FindOptions) (res []models2.Setting, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdSetting).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.Setting) - res = append(res, *d) - } - return res, nil -} - -func (svc *Service) GetSettingByKey(key string, opts *mongo.FindOptions) (res *models2.Setting, err error) { - query := bson.M{"key": key} - return svc.GetSetting(query, opts) -} diff --git a/core/models/service/spider_service.go b/core/models/service/spider_service.go deleted file mode 100644 index cdd9fbad..00000000 --- a/core/models/service/spider_service.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeSpider(d interface{}, err error) (res *models2.Spider, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Spider) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetSpiderById(id primitive.ObjectID) (res *models2.Spider, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdSpider).GetById(id) - return convertTypeSpider(d, err) -} - -func (svc *Service) GetSpider(query bson.M, opts *mongo.FindOptions) (res *models2.Spider, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdSpider).Get(query, opts) - return convertTypeSpider(d, err) -} - -func (svc *Service) GetSpiderList(query bson.M, opts *mongo.FindOptions) (res []models2.Spider, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdSpider).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.Spider) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/spider_stat_service.go b/core/models/service/spider_stat_service.go deleted file mode 100644 index 569d2fc9..00000000 --- a/core/models/service/spider_stat_service.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeSpiderStat(d interface{}, err error) (res *models2.SpiderStat, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.SpiderStat) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetSpiderStatById(id primitive.ObjectID) (res *models2.SpiderStat, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdSpiderStat).GetById(id) - return convertTypeSpiderStat(d, err) -} - -func (svc *Service) GetSpiderStat(query bson.M, opts *mongo.FindOptions) (res *models2.SpiderStat, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdSpiderStat).Get(query, opts) - return convertTypeSpiderStat(d, err) -} - -func (svc *Service) GetSpiderStatList(query bson.M, opts *mongo.FindOptions) (res []models2.SpiderStat, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdSpiderStat).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.SpiderStat) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/tag_service.go b/core/models/service/tag_service.go deleted file mode 100644 index 9013b022..00000000 --- a/core/models/service/tag_service.go +++ /dev/null @@ -1,118 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" -) - -func convertTypeTag(d interface{}, err error) (res *models2.Tag, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Tag) - if !ok { - return nil, trace.TraceError(errors.ErrorModelInvalidType) - } - return res, nil -} - -func (svc *Service) GetTagById(id primitive.ObjectID) (res *models2.Tag, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdTag).GetById(id) - return convertTypeTag(d, err) -} - -func (svc *Service) GetTag(query bson.M, opts *mongo.FindOptions) (res *models2.Tag, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdTag).Get(query, opts) - return convertTypeTag(d, err) -} - -func (svc *Service) GetTagList(query bson.M, opts *mongo.FindOptions) (res []models2.Tag, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdTag).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.Tag) - res = append(res, *d) - } - return res, nil -} - -func (svc *Service) GetTagIds(colName string, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error) { - // iterate tag names - for _, tag := range tags { - // count of tags with the name - tagDb, err := svc.GetTag(bson.M{"name": tag.GetName(), "col": colName}, nil) - if err == nil { - // tag exists - tag = tagDb - } else if err == mongo2.ErrNoDocuments { - // add new tag if not exists - colorHex := tag.GetColor() - if colorHex == "" { - color, _ := svc.colorSvc.GetRandom() - colorHex = color.GetHex() - } - tag = &models2.Tag{ - Id: primitive.NewObjectID(), - Name: tag.GetName(), - Color: colorHex, - Col: colName, - } - if err := delegate.NewModelDelegate(tag).Add(); err != nil { - return tagIds, trace.TraceError(err) - } - } - - // add to tag ids - tagIds = append(tagIds, tag.GetId()) - } - - return tagIds, nil -} - -func (svc *Service) UpdateTagsById(colName string, id primitive.ObjectID, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error) { - // get tag ids to update - tagIds, err = svc.GetTagIds(colName, tags) - if err != nil { - return tagIds, trace.TraceError(err) - } - - // update in db - a, err := svc.GetArtifactById(id) - if err != nil { - return tagIds, trace.TraceError(err) - } - a.TagIds = tagIds - if err := mongo.GetMongoCol(interfaces.ModelColNameArtifact).ReplaceId(id, a); err != nil { - return tagIds, err - } - return tagIds, nil -} - -func (svc *Service) UpdateTags(colName string, query bson.M, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error) { - // tag ids to update - tagIds, err = svc.GetTagIds(colName, tags) - if err != nil { - return tagIds, trace.TraceError(err) - } - - // update - update := bson.M{ - "_tid": tagIds, - } - - // fields - fields := []string{"_tid"} - - // update in db - if err := svc.GetBaseService(interfaces.ModelIdTag).Update(query, update, fields); err != nil { - return tagIds, trace.TraceError(err) - } - - return tagIds, nil -} diff --git a/core/models/service/tag_service_legacy.go b/core/models/service/tag_service_legacy.go deleted file mode 100644 index a78cf067..00000000 --- a/core/models/service/tag_service_legacy.go +++ /dev/null @@ -1,118 +0,0 @@ -package service - -// -//import ( -// "github.com/crawlab-team/crawlab/core/interfaces" -// "github.com/crawlab-team/crawlab/db/mongo" -// "go.mongodb.org/mongo-driver/bson" -// "go.mongodb.org/mongo-driver/bson/primitive" -// mongo2 "go.mongodb.org/mongo-driver/mongo" -//) -// -//type TagServiceInterface interface { -// getTagIds(colName string, tags []Tag) (tagIds []primitive.ObjectID, err error) -// GetModelById(id primitive.ObjectID) (res Tag, err error) -// GetModel(query bson.M, opts *mongo.FindOptions) (res Tag, err error) -// GetModelList(query bson.M, opts *mongo.FindOptions) (res []Tag, err error) -// UpdateTagsById(colName string, id primitive.ObjectID, tags []Tag) (tagIds []primitive.ObjectID, err error) -// UpdateTags(colName string, query bson.M, tags []Tag) (tagIds []primitive.ObjectID, err error) -//} -// -//type tagService struct { -// *baseService -//} -// -//func (svc *tagService) getTagIds(colName string, tags []Tag) (tagIds []primitive.ObjectID, err error) { -// // iterate tag names -// for _, tag := range tags { -// // count of tags with the name -// tagDb, err := MustGetRootService().GetTag(bson.M{"name": tag.Name, "col": colName}, nil) -// if err == nil { -// // tag exists -// tag = tagDb -// } else if err == mongo2.ErrNoDocuments { -// // add new tag if not exists -// colorHex := tag.Color -// if colorHex == "" { -// color, _ := ColorService.GetRandom() -// colorHex = color.Hex -// } -// tag = Tag{ -// Name: tag.Name, -// Color: colorHex, -// Col: colName, -// } -// if err := tag.Add(); err != nil { -// return tagIds, err -// } -// } -// -// // add to tag ids -// tagIds = append(tagIds, tag.Id) -// } -// -// return tagIds, nil -//} -// -//func (svc *tagService) GetModelById(id primitive.ObjectID) (res Tag, err error) { -// err = svc.findId(id).One(&res) -// return res, err -//} -// -//func (svc *tagService) GetModel(query bson.M, opts *mongo.FindOptions) (res Tag, err error) { -// err = svc.find(query, opts).One(&res) -// return res, err -//} -// -//func (svc *tagService) GetModelList(query bson.M, opts *mongo.FindOptions) (res []Tag, err error) { -// err = svc.find(query, opts).All(&res) -// return res, err -//} -// -//func (svc *tagService) UpdateTagsById(colName string, id primitive.ObjectID, tags []Tag) (tagIds []primitive.ObjectID, err error) { -// // get tag ids to update -// tagIds, err = svc.getTagIds(colName, tags) -// if err != nil { -// return tagIds, err -// } -// -// // update in db -// a, err := MustGetRootService().GetArtifactById(id) -// if err != nil { -// return tagIds, err -// } -// a.TagIds = tagIds -// if err := mongo.GetMongoCol(interfaces.ModelColNameArtifact).ReplaceId(id, a); err != nil { -// return tagIds, err -// } -// return tagIds, nil -//} -// -//func (svc *tagService) UpdateTags(colName string, query bson.M, tags []Tag) (tagIds []primitive.ObjectID, err error) { -// // tag ids to update -// tagIds, err = svc.getTagIds(colName, tags) -// if err != nil { -// return tagIds, err -// } -// -// // update -// update := bson.M{ -// "_tid": tagIds, -// } -// -// // fields -// fields := []string{"_tid"} -// -// // update in db -// if err := ArtifactService.Update(query, update, fields); err != nil { -// return tagIds, err -// } -// -// return tagIds, nil -//} -// -//func NewTagService() (svc *tagService) { -// return &tagService{svc.GetBaseService(interfaces.ModelIdTag)} -//} -// -//var TagService *tagService diff --git a/core/models/service/task_queue_service.go b/core/models/service/task_queue_service.go deleted file mode 100644 index 7ade0326..00000000 --- a/core/models/service/task_queue_service.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeTaskQueueItem(d interface{}, err error) (res *models2.TaskQueueItem, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.TaskQueueItem) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetTaskQueueItemById(id primitive.ObjectID) (res *models2.TaskQueueItem, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdTaskQueue).GetById(id) - return convertTypeTaskQueueItem(d, err) -} - -func (svc *Service) GetTaskQueueItem(query bson.M, opts *mongo.FindOptions) (res *models2.TaskQueueItem, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdTaskQueue).Get(query, opts) - return convertTypeTaskQueueItem(d, err) -} - -func (svc *Service) GetTaskQueueItemList(query bson.M, opts *mongo.FindOptions) (res []models2.TaskQueueItem, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdTaskQueue).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.TaskQueueItem) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/task_service.go b/core/models/service/task_service.go deleted file mode 100644 index 3e5e64a5..00000000 --- a/core/models/service/task_service.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeTask(d interface{}, err error) (res *models2.Task, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Task) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetTaskById(id primitive.ObjectID) (res *models2.Task, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdTask).GetById(id) - return convertTypeTask(d, err) -} - -func (svc *Service) GetTask(query bson.M, opts *mongo.FindOptions) (res *models2.Task, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdTask).Get(query, opts) - return convertTypeTask(d, err) -} - -func (svc *Service) GetTaskList(query bson.M, opts *mongo.FindOptions) (res []models2.Task, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdTask).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.Task) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/task_stat_service.go b/core/models/service/task_stat_service.go deleted file mode 100644 index 3d335104..00000000 --- a/core/models/service/task_stat_service.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeTaskStat(d interface{}, err error) (res *models2.TaskStat, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.TaskStat) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetTaskStatById(id primitive.ObjectID) (res *models2.TaskStat, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdTaskStat).GetById(id) - return convertTypeTaskStat(d, err) -} - -func (svc *Service) GetTaskStat(query bson.M, opts *mongo.FindOptions) (res *models2.TaskStat, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdTaskStat).Get(query, opts) - return convertTypeTaskStat(d, err) -} - -func (svc *Service) GetTaskStatList(query bson.M, opts *mongo.FindOptions) (res []models2.TaskStat, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdTaskStat).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.TaskStat) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/token_service.go b/core/models/service/token_service.go deleted file mode 100644 index 02689353..00000000 --- a/core/models/service/token_service.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeToken(d interface{}, err error) (res *models2.Token, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Token) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetTokenById(id primitive.ObjectID) (res *models2.Token, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdToken).GetById(id) - return convertTypeToken(d, err) -} - -func (svc *Service) GetToken(query bson.M, opts *mongo.FindOptions) (res *models2.Token, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdToken).Get(query, opts) - return convertTypeToken(d, err) -} - -func (svc *Service) GetTokenList(query bson.M, opts *mongo.FindOptions) (res []models2.Token, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdToken).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.Token) - res = append(res, *d) - } - return res, nil -} diff --git a/core/models/service/user_role_service.go b/core/models/service/user_role_service.go deleted file mode 100644 index fe610919..00000000 --- a/core/models/service/user_role_service.go +++ /dev/null @@ -1,51 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeUserRole(d interface{}, err error) (res *models2.UserRole, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.UserRole) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetUserRoleById(id primitive.ObjectID) (res *models2.UserRole, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdUserRole).GetById(id) - return convertTypeUserRole(d, err) -} - -func (svc *Service) GetUserRole(query bson.M, opts *mongo.FindOptions) (res *models2.UserRole, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdUserRole).Get(query, opts) - return convertTypeUserRole(d, err) -} - -func (svc *Service) GetUserRoleList(query bson.M, opts *mongo.FindOptions) (res []models2.UserRole, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdUserRole).GetList(query, opts) - if err != nil { - return nil, err - } - for _, doc := range l.GetModels() { - d := doc.(*models2.UserRole) - res = append(res, *d) - } - return res, nil -} - -func (svc *Service) GetUserRoleListByUserId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models2.UserRole, err error) { - return svc.GetUserRoleList(bson.M{"user_id": id}, opts) -} - -func (svc *Service) GetUserRoleListByRoleId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models2.UserRole, err error) { - return svc.GetUserRoleList(bson.M{"role_id": id}, opts) -} diff --git a/core/models/service/user_service.go b/core/models/service/user_service.go deleted file mode 100644 index ab9c29b1..00000000 --- a/core/models/service/user_service.go +++ /dev/null @@ -1,58 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeUser(d interface{}, err error) (res *models2.User, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.User) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetUserById(id primitive.ObjectID) (res *models2.User, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdUser).GetById(id) - return convertTypeUser(d, err) -} - -func (svc *Service) GetUser(query bson.M, opts *mongo.FindOptions) (res *models2.User, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdUser).Get(query, opts) - return convertTypeUser(d, err) -} - -func (svc *Service) GetUserList(query bson.M, opts *mongo.FindOptions) (res []models2.User, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdUser).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.User) - res = append(res, *d) - } - return res, nil -} - -func (svc *Service) GetUserByUsername(username string, opts *mongo.FindOptions) (res *models2.User, err error) { - query := bson.M{"username": username} - return svc.GetUser(query, opts) -} - -func (svc *Service) GetUserByUsernameWithPassword(username string, opts *mongo.FindOptions) (res *models2.User, err error) { - u, err := svc.GetUserByUsername(username, opts) - if err != nil { - return nil, err - } - p, err := svc.GetPasswordById(u.Id) - if err != nil { - return nil, err - } - u.Password = p.Password - return u, nil -} diff --git a/core/models/service/variable_service.go b/core/models/service/variable_service.go deleted file mode 100644 index 8b72371d..00000000 --- a/core/models/service/variable_service.go +++ /dev/null @@ -1,45 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func convertTypeVariable(d interface{}, err error) (res *models2.Variable, err2 error) { - if err != nil { - return nil, err - } - res, ok := d.(*models2.Variable) - if !ok { - return nil, errors.ErrorModelInvalidType - } - return res, nil -} - -func (svc *Service) GetVariableById(id primitive.ObjectID) (res *models2.Variable, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdVariable).GetById(id) - return convertTypeVariable(d, err) -} - -func (svc *Service) GetVariable(query bson.M, opts *mongo.FindOptions) (res *models2.Variable, err error) { - d, err := svc.GetBaseService(interfaces.ModelIdVariable).Get(query, opts) - return convertTypeVariable(d, err) -} - -func (svc *Service) GetVariableList(query bson.M, opts *mongo.FindOptions) (res []models2.Variable, err error) { - l, err := svc.GetBaseService(interfaces.ModelIdVariable).GetList(query, opts) - for _, doc := range l.GetModels() { - d := doc.(*models2.Variable) - res = append(res, *d) - } - return res, nil -} - -func (svc *Service) GetVariableByKey(key string, opts *mongo.FindOptions) (res *models2.Variable, err error) { - query := bson.M{"key": key} - return svc.GetVariable(query, opts) -} diff --git a/core/node/service/options.go b/core/node/service/options.go deleted file mode 100644 index 941879f5..00000000 --- a/core/node/service/options.go +++ /dev/null @@ -1,47 +0,0 @@ -package service - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "time" -) - -type Option func(svc interfaces.NodeService) - -func WithConfigPath(path string) Option { - return func(svc interfaces.NodeService) { - svc.SetConfigPath(path) - } -} - -func WithAddress(address interfaces.Address) Option { - return func(svc interfaces.NodeService) { - svc.SetAddress(address) - } -} - -func WithMonitorInterval(duration time.Duration) Option { - return func(svc interfaces.NodeService) { - svc2, ok := svc.(interfaces.NodeMasterService) - if ok { - svc2.SetMonitorInterval(duration) - } - } -} - -func WithStopOnError() Option { - return func(svc interfaces.NodeService) { - svc2, ok := svc.(interfaces.NodeMasterService) - if ok { - svc2.StopOnError() - } - } -} - -func WithHeartbeatInterval(duration time.Duration) Option { - return func(svc interfaces.NodeService) { - svc2, ok := svc.(interfaces.NodeWorkerService) - if ok { - svc2.SetHeartbeatInterval(duration) - } - } -} diff --git a/core/node/service/worker_service_v2.go b/core/node/service/worker_service_v2.go index a041877c..8e73b4ff 100644 --- a/core/node/service/worker_service_v2.go +++ b/core/node/service/worker_service_v2.go @@ -4,16 +4,15 @@ import ( "context" "encoding/json" "github.com/apex/log" - config2 "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/grpc/client" "github.com/crawlab-team/crawlab/core/interfaces" client2 "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/models" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models/v2" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/handler" "github.com/crawlab-team/crawlab/core/utils" - grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/crawlab/grpc" "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "sync" @@ -32,7 +31,7 @@ type WorkerServiceV2 struct { heartbeatInterval time.Duration // internals - n *models2.NodeV2 + n *models.NodeV2 s grpc.NodeService_SubscribeClient } @@ -88,7 +87,7 @@ func (svc *WorkerServiceV2) Register() { if err != nil { panic(err) } - svc.n, err = client2.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": svc.GetConfigService().GetNodeKey()}, nil) + svc.n, err = client2.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": svc.GetConfigService().GetNodeKey()}, nil) if err != nil { panic(err) } @@ -125,7 +124,7 @@ func (svc *WorkerServiceV2) handleStreamMessage(msg *grpc.StreamMessage) (err er return trace.TraceError(err) } case grpc.StreamMessageCode_RUN_TASK: - var t models.Task + var t models.TaskV2 if err := json.Unmarshal(msg.Data, &t); err != nil { return trace.TraceError(err) } @@ -133,7 +132,7 @@ func (svc *WorkerServiceV2) handleStreamMessage(msg *grpc.StreamMessage) (err er return trace.TraceError(err) } case grpc.StreamMessageCode_CANCEL_TASK: - var t models.Task + var t models.TaskV2 if err := json.Unmarshal(msg.Data, &t); err != nil { return trace.TraceError(err) } @@ -202,7 +201,7 @@ var workerServiceV2Once = new(sync.Once) func newWorkerServiceV2() (res *WorkerServiceV2, err error) { svc := &WorkerServiceV2{ - cfgPath: config2.GetConfigPath(), + cfgPath: config.GetConfigPath(), heartbeatInterval: 15 * time.Second, } diff --git a/core/process/daemon.go b/core/process/daemon.go deleted file mode 100644 index 058880dd..00000000 --- a/core/process/daemon.go +++ /dev/null @@ -1,170 +0,0 @@ -package process - -import ( - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/sys_exec" - "github.com/crawlab-team/crawlab/trace" - "math/rand" - "os/exec" - "time" -) - -const ( - SignalCreate = iota - SignalStart - SignalStopped - SignalError - SignalExited - SignalReachedMaxErrors -) - -type Daemon struct { - // settings - maxErrors int - exitTimeout time.Duration - - // internals - errors int - errMsg string - exitCode int - newCmdFn func() *exec.Cmd - cmd *exec.Cmd - stopped bool - ch chan int -} - -func (d *Daemon) Start() (err error) { - go d.handleSignal() - - for { - // command - d.cmd = d.newCmdFn() - d.ch <- SignalCreate - - // attempt to run - _ = d.cmd.Start() - d.ch <- SignalStart - - if err := d.cmd.Wait(); err != nil { - // stopped - d.ch <- SignalStopped - if d.stopped { - log.Infof("daemon stopped") - return nil - } - - // error - d.ch <- SignalError - d.errMsg = err.Error() - trace.PrintError(err) - } - - // exited - d.ch <- SignalExited - - // exit code - d.exitCode = d.cmd.ProcessState.ExitCode() - - // check exit code - if d.exitCode == 0 { - log.Infof("process exited with code 0") - return - } - - // error message - d.errMsg = errors.ErrorProcessDaemonProcessExited.Error() - - // increment errors - d.errors++ - - // validate if error count exceeds max errors - if d.errors >= d.maxErrors { - log.Infof("reached max errors: %d", d.maxErrors) - d.ch <- SignalReachedMaxErrors - return errors.ErrorProcessReachedMaxErrors - } - - // re-attempt - waitSec := rand.Intn(5) - log.Infof("re-attempt to start process in %d seconds...", waitSec) - time.Sleep(time.Duration(waitSec) * time.Second) - } -} - -func (d *Daemon) Stop() { - d.stopped = true - opts := &sys_exec.KillProcessOptions{ - Timeout: d.exitTimeout, - Force: false, - } - _ = sys_exec.KillProcess(d.cmd, opts) -} - -func (d *Daemon) GetMaxErrors() (maxErrors int) { - return d.maxErrors -} - -func (d *Daemon) SetMaxErrors(maxErrors int) { - d.maxErrors = maxErrors -} - -func (d *Daemon) GetExitTimeout() (timeout time.Duration) { - return d.exitTimeout -} - -func (d *Daemon) SetExitTimeout(timeout time.Duration) { - d.exitTimeout = timeout -} - -func (d *Daemon) GetCmd() (cmd *exec.Cmd) { - return d.cmd -} - -func (d *Daemon) GetCh() (ch chan int) { - return d.ch -} - -func (d *Daemon) handleSignal() { - for { - select { - case signal := <-d.ch: - switch signal { - case SignalCreate: - log.Infof("process created") - case SignalStart: - log.Infof("process started") - case SignalStopped: - log.Infof("process stopped") - case SignalError: - trace.PrintError(errors.NewProcessError(d.errMsg)) - case SignalExited: - log.Infof("process exited") - case SignalReachedMaxErrors: - log.Infof("reached max errors") - return - } - } - } -} - -func NewProcessDaemon(newCmdFn func() *exec.Cmd, opts ...DaemonOption) (d interfaces.ProcessDaemon) { - // daemon - d = &Daemon{ - maxErrors: 5, - exitTimeout: 15 * time.Second, - errors: 0, - errMsg: "", - newCmdFn: newCmdFn, - stopped: false, - ch: make(chan int), - } - - // apply options - for _, opt := range opts { - opt(d) - } - - return d -} diff --git a/core/process/daemon_test.go b/core/process/daemon_test.go deleted file mode 100644 index 7734eb59..00000000 --- a/core/process/daemon_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package process - -import ( - "github.com/stretchr/testify/require" - "os/exec" - "testing" -) - -func TestDaemon(t *testing.T) { - d := NewProcessDaemon(func() *exec.Cmd { - return exec.Command("echo", "hello") - }) - err := d.Start() - require.Nil(t, err) - - d = NewProcessDaemon(func() *exec.Cmd { - return exec.Command("return", "1") - }) - err = d.Start() - require.NotNil(t, err) -} diff --git a/core/process/manage.go b/core/process/manage.go deleted file mode 100644 index 0869ac33..00000000 --- a/core/process/manage.go +++ /dev/null @@ -1,60 +0,0 @@ -package process - -import ( - "github.com/crawlab-team/crawlab/trace" - "os/exec" - "regexp" - "runtime" - "strings" -) - -var pidRegexp, _ = regexp.Compile("(?:^|\\s+)\\d+(?:$|\\s+)") - -func ProcessIdExists(id int) (ok bool) { - lines, err := ListProcess(string(rune(id))) - if err != nil { - return false - } - for _, line := range lines { - matched := pidRegexp.MatchString(line) - if matched { - return true - } - } - return false -} - -func ListProcess(text string) (lines []string, err error) { - if runtime.GOOS == "windows" { - return listProcessWindow(text) - } else { - return listProcessLinuxMac(text) - } -} - -func listProcessWindow(text string) (lines []string, err error) { - cmd := exec.Command("tasklist", "/fi", text) - out, err := cmd.CombinedOutput() - _, ok := err.(*exec.ExitError) - if !ok { - return nil, trace.TraceError(err) - } - lines = strings.Split(string(out), "\n") - return lines, nil -} - -func listProcessLinuxMac(text string) (lines []string, err error) { - cmd := exec.Command("ps", "aux") - out, err := cmd.CombinedOutput() - _, ok := err.(*exec.ExitError) - if !ok { - return nil, trace.TraceError(err) - } - _lines := strings.Split(string(out), "\n") - for _, l := range _lines { - if strings.Contains(l, text) { - lines = append(lines, l) - } - } - return lines, nil -} diff --git a/core/process/options.go b/core/process/options.go deleted file mode 100644 index f0e046c9..00000000 --- a/core/process/options.go +++ /dev/null @@ -1,20 +0,0 @@ -package process - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "time" -) - -type DaemonOption func(d interfaces.ProcessDaemon) - -func WithDaemonMaxErrors(maxErrors int) DaemonOption { - return func(d interfaces.ProcessDaemon) { - d.SetMaxErrors(maxErrors) - } -} - -func WithExitTimeout(timeout time.Duration) DaemonOption { - return func(d interfaces.ProcessDaemon) { - - } -} diff --git a/core/result/options.go b/core/result/options.go deleted file mode 100644 index 59c63301..00000000 --- a/core/result/options.go +++ /dev/null @@ -1,16 +0,0 @@ -package result - -import "go.mongodb.org/mongo-driver/bson/primitive" - -type Option func(opts *Options) - -type Options struct { - registryKey string // registry key - SpiderId primitive.ObjectID // data source id -} - -func WithRegistryKey(key string) Option { - return func(opts *Options) { - opts.registryKey = key - } -} diff --git a/core/result/service.go b/core/result/service.go deleted file mode 100644 index 5497f3a6..00000000 --- a/core/result/service.go +++ /dev/null @@ -1,83 +0,0 @@ -package result - -import ( - "fmt" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson/primitive" - "sync" -) - -func NewResultService(registryKey string, s *models.Spider) (svc2 interfaces.ResultService, err error) { - // result service function - var fn interfaces.ResultServiceRegistryFn - - if registryKey == "" { - // default - fn = NewResultServiceMongo - } else { - // from registry - reg := GetResultServiceRegistry() - fn = reg.Get(registryKey) - if fn == nil { - return nil, errors.NewResultError(fmt.Sprintf("%s is not implemented", registryKey)) - } - } - - // generate result service - svc, err := fn(s.ColId, s.DataSourceId) - if err != nil { - return nil, trace.TraceError(err) - } - - return svc, nil -} - -var store = sync.Map{} - -func GetResultService(spiderId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { - // model service - modelSvc, err := service.GetService() - if err != nil { - return nil, trace.TraceError(err) - } - - // spider - s, err := modelSvc.GetSpiderById(spiderId) - if err != nil { - return nil, trace.TraceError(err) - } - - // store key - storeKey := s.ColId.Hex() + ":" + s.DataSourceId.Hex() - - // attempt to load result service from store - res, _ := store.Load(storeKey) - if res != nil { - svc, ok := res.(interfaces.ResultService) - if ok { - return svc, nil - } - } - - // registry key - var registryKey string - ds, _ := modelSvc.GetDataSourceById(s.DataSourceId) - if ds != nil { - registryKey = ds.Type - } - - // create a new result service if not exists - svc, err := NewResultService(registryKey, s) - if err != nil { - return nil, err - } - - // save into store - store.Store(storeKey, svc) - - return svc, nil -} diff --git a/core/result/service_mongo.go b/core/result/service_mongo.go deleted file mode 100644 index 504e2827..00000000 --- a/core/result/service_mongo.go +++ /dev/null @@ -1,146 +0,0 @@ -package result - -import ( - "github.com/crawlab-team/crawlab/trace" - "time" - - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -type ServiceMongo struct { - // dependencies - modelSvc service.ModelService - modelColSvc interfaces.ModelBaseService - - // internals - colId primitive.ObjectID // _id of models.DataCollection - dc *models.DataCollection // models.DataCollection - t time.Time -} - -func (svc *ServiceMongo) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { - _query := svc.getQuery(query) - _opts := svc.getOpts(opts) - return svc.getList(_query, _opts) -} - -func (svc *ServiceMongo) Count(query generic.ListQuery) (n int, err error) { - _query := svc.getQuery(query) - return svc.modelColSvc.Count(_query) -} - -func (svc *ServiceMongo) Insert(docs ...interface{}) (err error) { - if svc.dc.Dedup.Enabled && len(svc.dc.Dedup.Keys) > 0 { - for _, doc := range docs { - hash, err := utils.GetResultHash(doc, svc.dc.Dedup.Keys) - if err != nil { - return err - } - doc.(interfaces.Result).SetValue(constants.HashKey, hash) - query := bson.M{constants.HashKey: hash} - switch svc.dc.Dedup.Type { - case constants.DedupTypeOverwrite: - err = mongo.GetMongoCol(svc.dc.Name).ReplaceWithOptions(query, doc, &options.ReplaceOptions{Upsert: &[]bool{true}[0]}) - if err != nil { - return trace.TraceError(err) - } - default: - var o bson.M - err := mongo.GetMongoCol(svc.dc.Name).Find(query, &mongo.FindOptions{Limit: 1}).One(&o) - if err == nil { - // exists, ignore - continue - } - if err != mongo2.ErrNoDocuments { - // error - return trace.TraceError(err) - } - // not exists, insert - _, err = mongo.GetMongoCol(svc.dc.Name).Insert(doc) - if err != nil { - return trace.TraceError(err) - } - } - } - } else { - _, err = mongo.GetMongoCol(svc.dc.Name).InsertMany(docs) - if err != nil { - return trace.TraceError(err) - } - } - return nil -} - -func (svc *ServiceMongo) Index(fields []string) { - for _, field := range fields { - _ = mongo.GetMongoCol(svc.dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{field: 1}}) - } -} - -func (svc *ServiceMongo) SetTime(t time.Time) { - svc.t = t -} - -func (svc *ServiceMongo) GetTime() (t time.Time) { - return svc.t -} - -func (svc *ServiceMongo) getList(query bson.M, opts *mongo.FindOptions) (results []interface{}, err error) { - list, err := svc.modelColSvc.GetList(query, opts) - if err != nil { - return nil, err - } - for _, d := range list.GetModels() { - r, ok := d.(interfaces.Result) - if ok { - results = append(results, r) - } - } - return results, nil -} - -func (svc *ServiceMongo) getQuery(query generic.ListQuery) (res bson.M) { - return utils.GetMongoQuery(query) -} - -func (svc *ServiceMongo) getOpts(opts *generic.ListOptions) (res *mongo.FindOptions) { - return utils.GetMongoOpts(opts) -} - -func NewResultServiceMongo(colId primitive.ObjectID, _ primitive.ObjectID) (svc2 interfaces.ResultService, err error) { - // service - svc := &ServiceMongo{ - colId: colId, - t: time.Now(), - } - - // dependency injection - svc.modelSvc, err = service.GetService() - if err != nil { - return nil, err - } - - // data collection - svc.dc, _ = svc.modelSvc.GetDataCollectionById(colId) - go func() { - for { - time.Sleep(1 * time.Second) - svc.dc, _ = svc.modelSvc.GetDataCollectionById(colId) - } - }() - - // data collection model service - svc.modelColSvc = service.GetBaseServiceByColName(interfaces.ModelIdResult, svc.dc.Name) - - return svc, nil -} diff --git a/core/result/service_registry.go b/core/result/service_registry.go deleted file mode 100644 index 2c635397..00000000 --- a/core/result/service_registry.go +++ /dev/null @@ -1,48 +0,0 @@ -package result - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "sync" -) - -type ServiceRegistry struct { - // internals - services sync.Map -} - -func (r *ServiceRegistry) Register(key string, fn interfaces.ResultServiceRegistryFn) { - r.services.Store(key, fn) -} - -func (r *ServiceRegistry) Unregister(key string) { - r.services.Delete(key) -} - -func (r *ServiceRegistry) Get(key string) (fn interfaces.ResultServiceRegistryFn) { - res, ok := r.services.Load(key) - if ok { - fn, ok = res.(interfaces.ResultServiceRegistryFn) - if !ok { - return nil - } - return fn - } - return nil -} - -func NewResultServiceRegistry() (r interfaces.ResultServiceRegistry) { - r = &ServiceRegistry{ - services: sync.Map{}, - } - return r -} - -var _svc interfaces.ResultServiceRegistry - -func GetResultServiceRegistry() (r interfaces.ResultServiceRegistry) { - if _svc != nil { - return _svc - } - _svc = NewResultServiceRegistry() - return _svc -} diff --git a/core/schedule/options.go b/core/schedule/options.go deleted file mode 100644 index 6ad22f99..00000000 --- a/core/schedule/options.go +++ /dev/null @@ -1,37 +0,0 @@ -package schedule - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "time" -) - -type Option func(svc interfaces.ScheduleService) - -func WithConfigPath(path string) Option { - return func(svc interfaces.ScheduleService) { - svc.SetConfigPath(path) - } -} - -func WithLocation(loc *time.Location) Option { - return func(svc interfaces.ScheduleService) { - svc.SetLocation(loc) - } -} - -func WithDelayIfStillRunning() Option { - return func(svc interfaces.ScheduleService) { - svc.SetDelay(true) - } -} - -func WithSkipIfStillRunning() Option { - return func(svc interfaces.ScheduleService) { - svc.SetSkip(true) - } -} - -func WithUpdateInterval(interval time.Duration) Option { - return func(svc interfaces.ScheduleService) { - } -} diff --git a/core/schedule/service.go b/core/schedule/service.go deleted file mode 100644 index 1238700b..00000000 --- a/core/schedule/service.go +++ /dev/null @@ -1,287 +0,0 @@ -package schedule - -import ( - "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "github.com/robfig/cron/v3" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "sync" - "time" -) - -type Service struct { - // dependencies - interfaces.WithConfigPath - modelSvc service.ModelService - adminSvc interfaces.SpiderAdminService - - // settings variables - loc *time.Location - delay bool - skip bool - updateInterval time.Duration - - // internals - cron *cron.Cron - logger cron.Logger - schedules []models.Schedule - stopped bool - mu sync.Mutex -} - -func (svc *Service) GetLocation() (loc *time.Location) { - return svc.loc -} - -func (svc *Service) SetLocation(loc *time.Location) { - svc.loc = loc -} - -func (svc *Service) GetDelay() (delay bool) { - return svc.delay -} - -func (svc *Service) SetDelay(delay bool) { - svc.delay = delay -} - -func (svc *Service) GetSkip() (skip bool) { - return svc.skip -} - -func (svc *Service) SetSkip(skip bool) { - svc.skip = skip -} - -func (svc *Service) GetUpdateInterval() (interval time.Duration) { - return svc.updateInterval -} - -func (svc *Service) SetUpdateInterval(interval time.Duration) { - svc.updateInterval = interval -} - -func (svc *Service) Init() (err error) { - return svc.fetch() -} - -func (svc *Service) Start() { - svc.cron.Start() - go svc.Update() -} - -func (svc *Service) Wait() { - utils.DefaultWait() - svc.Stop() -} - -func (svc *Service) Stop() { - svc.stopped = true - svc.cron.Stop() -} - -func (svc *Service) Enable(s interfaces.Schedule, args ...interface{}) (err error) { - svc.mu.Lock() - defer svc.mu.Unlock() - - id, err := svc.cron.AddFunc(s.GetCron(), svc.schedule(s.GetId())) - if err != nil { - return trace.TraceError(err) - } - s.SetEnabled(true) - s.SetEntryId(id) - u := utils.GetUserFromArgs(args...) - return delegate.NewModelDelegate(s, u).Save() -} - -func (svc *Service) Disable(s interfaces.Schedule, args ...interface{}) (err error) { - svc.mu.Lock() - defer svc.mu.Unlock() - - svc.cron.Remove(s.GetEntryId()) - s.SetEnabled(false) - s.SetEntryId(-1) - u := utils.GetUserFromArgs(args...) - return delegate.NewModelDelegate(s, u).Save() -} - -func (svc *Service) Update() { - for { - if svc.stopped { - return - } - - svc.update() - - time.Sleep(svc.updateInterval) - } -} - -func (svc *Service) GetCron() (c *cron.Cron) { - return svc.cron -} - -func (svc *Service) update() { - // fetch enabled schedules - if err := svc.fetch(); err != nil { - trace.PrintError(err) - return - } - - // entry id map - entryIdsMap := svc.getEntryIdsMap() - - // iterate enabled schedules - for _, s := range svc.schedules { - _, ok := entryIdsMap[s.EntryId] - if ok { - entryIdsMap[s.EntryId] = true - } else { - if err := svc.Enable(&s); err != nil { - trace.PrintError(err) - continue - } - } - } - - // remove non-existent entries - for id, ok := range entryIdsMap { - if !ok { - svc.cron.Remove(id) - } - } -} - -func (svc *Service) getEntryIdsMap() (res map[cron.EntryID]bool) { - res = map[cron.EntryID]bool{} - for _, e := range svc.cron.Entries() { - res[e.ID] = false - } - return res -} - -func (svc *Service) fetch() (err error) { - query := bson.M{ - "enabled": true, - } - svc.schedules, err = svc.modelSvc.GetScheduleList(query, nil) - if err != nil { - return err - } - return nil -} - -func (svc *Service) schedule(id primitive.ObjectID) (fn func()) { - return func() { - // schedule - s, err := svc.modelSvc.GetScheduleById(id) - if err != nil { - trace.PrintError(err) - return - } - - // spider - spider, err := svc.modelSvc.GetSpiderById(s.GetSpiderId()) - if err != nil { - trace.PrintError(err) - return - } - - // options - opts := &interfaces.SpiderRunOptions{ - Mode: s.GetMode(), - NodeIds: s.GetNodeIds(), - Cmd: s.GetCmd(), - Param: s.GetParam(), - Priority: s.GetPriority(), - ScheduleId: s.GetId(), - UserId: s.UserId, - } - - // normalize options - if opts.Mode == "" { - opts.Mode = spider.Mode - } - if len(opts.NodeIds) == 0 { - opts.NodeIds = spider.NodeIds - } - if opts.Cmd == "" { - opts.Cmd = spider.Cmd - } - if opts.Param == "" { - opts.Param = spider.Param - } - if opts.Priority == 0 { - if spider.Priority > 0 { - opts.Priority = spider.Priority - } else { - opts.Priority = 5 - } - } - - // schedule or assign a task in the task queue - if _, err := svc.adminSvc.Schedule(s.GetSpiderId(), opts); err != nil { - trace.PrintError(err) - } - } -} - -func NewScheduleService() (svc2 interfaces.ScheduleService, err error) { - // service - svc := &Service{ - WithConfigPath: config.NewConfigPathService(), - loc: time.Local, - // TODO: implement delay and skip - delay: false, - skip: false, - updateInterval: 1 * time.Minute, - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - adminSvc interfaces.SpiderAdminService, - ) { - svc.modelSvc = modelSvc - svc.adminSvc = adminSvc - }); err != nil { - return nil, trace.TraceError(err) - } - - // logger - svc.logger = NewLogger() - - // cron - svc.cron = cron.New( - cron.WithLogger(svc.logger), - cron.WithLocation(svc.loc), - cron.WithChain(cron.Recover(svc.logger)), - ) - - // initialize - if err := svc.Init(); err != nil { - return nil, err - } - - return svc, nil -} - -var svc interfaces.ScheduleService - -func GetScheduleService() (res interfaces.ScheduleService, err error) { - if svc != nil { - return svc, nil - } - svc, err = NewScheduleService() - if err != nil { - return nil, err - } - return svc, nil -} diff --git a/core/schedule/test/base.go b/core/schedule/test/base.go deleted file mode 100644 index cd7c885e..00000000 --- a/core/schedule/test/base.go +++ /dev/null @@ -1,91 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/schedule" - "go.uber.org/dig" - "testing" -) - -func init() { - var err error - T, err = NewTest() - if err != nil { - panic(err) - } -} - -var T *Test - -type Test struct { - // dependencies - modelSvc service.ModelService - scheduleSvc interfaces.ScheduleService - - // test data - TestSchedule interfaces.Schedule - TestSpider interfaces.Spider - ScriptName string - Script string -} - -func (t *Test) Setup(t2 *testing.T) { - t.scheduleSvc.Start() - t2.Cleanup(t.Cleanup) -} - -func (t *Test) Cleanup() { - t.scheduleSvc.Stop() - _ = t.modelSvc.GetBaseService(interfaces.ModelIdTask).Delete(nil) -} - -func NewTest() (t *Test, err error) { - // test - t = &Test{ - TestSpider: &models.Spider{ - Name: "test_spider", - Cmd: "go run main.go", - }, - ScriptName: "main.go", - Script: `package main -import "fmt" -func main() { - fmt.Println("it works") -}`, - } - - // dependency injection - c := dig.New() - if err := c.Provide(service.GetService); err != nil { - return nil, err - } - if err := c.Provide(schedule.NewScheduleService); err != nil { - return nil, err - } - if err := c.Invoke(func(modelSvc service.ModelService, scheduleSvc interfaces.ScheduleService) { - t.modelSvc = modelSvc - t.scheduleSvc = scheduleSvc - }); err != nil { - return nil, err - } - - // add spider to db - if err := delegate.NewModelDelegate(t.TestSpider).Add(); err != nil { - return nil, err - } - - // test schedule - t.TestSchedule = &models.Schedule{ - Name: "test_schedule", - SpiderId: t.TestSpider.GetId(), - Cron: "* * * * *", - } - if err := delegate.NewModelDelegate(t.TestSchedule).Add(); err != nil { - return nil, err - } - - return t, nil -} diff --git a/core/schedule/test/schedule_service_test.go b/core/schedule/test/schedule_service_test.go deleted file mode 100644 index 7a815da1..00000000 --- a/core/schedule/test/schedule_service_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package test - -import ( - "github.com/stretchr/testify/require" - "testing" - "time" -) - -func TestScheduleService_Enable_Disable(t *testing.T) { - var err error - T.Setup(t) - - time.Sleep(1 * time.Second) - err = T.scheduleSvc.Enable(T.TestSchedule) - require.Nil(t, err) - time.Sleep(1 * time.Second) - - require.True(t, T.TestSchedule.GetEnabled()) - require.Greater(t, int(T.TestSchedule.GetEntryId()), -1) - e := T.scheduleSvc.GetCron().Entry(T.TestSchedule.GetEntryId()) - require.Equal(t, T.TestSchedule.GetEntryId(), e.ID) - time.Sleep(1 * time.Second) - - err = T.scheduleSvc.Disable(T.TestSchedule) - require.False(t, T.TestSchedule.GetEnabled()) - require.Equal(t, 0, len(T.scheduleSvc.GetCron().Entries())) -} - -func TestScheduleService_Run(t *testing.T) { - var err error - T.Setup(t) - - time.Sleep(1 * time.Second) - err = T.scheduleSvc.Enable(T.TestSchedule) - require.Nil(t, err) - time.Sleep(1 * time.Minute) - - tasks, err := T.modelSvc.GetTaskList(nil, nil) - require.Nil(t, err) - require.Greater(t, len(tasks), 0) - for _, task := range tasks { - require.False(t, task.ScheduleId.IsZero()) - } -} diff --git a/core/spider/admin/options.go b/core/spider/admin/options.go deleted file mode 100644 index 32e08e76..00000000 --- a/core/spider/admin/options.go +++ /dev/null @@ -1,11 +0,0 @@ -package admin - -import "github.com/crawlab-team/crawlab/core/interfaces" - -type Option func(svc interfaces.SpiderAdminService) - -func WithConfigPath(path string) Option { - return func(svc interfaces.SpiderAdminService) { - svc.SetConfigPath(path) - } -} diff --git a/core/spider/admin/service.go b/core/spider/admin/service.go deleted file mode 100644 index b4d13480..00000000 --- a/core/spider/admin/service.go +++ /dev/null @@ -1,352 +0,0 @@ -package admin - -import ( - "context" - "github.com/apex/log" - config2 "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "github.com/crawlab-team/crawlab/vcs" - "github.com/google/uuid" - "github.com/robfig/cron/v3" - "github.com/spf13/viper" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "os" - "path" - "path/filepath" - "sync" - "time" -) - -type Service struct { - // dependencies - nodeCfgSvc interfaces.NodeConfigService - modelSvc service.ModelService - schedulerSvc interfaces.TaskSchedulerService - cron *cron.Cron - syncLock bool - - // settings - cfgPath string -} - -func (svc *Service) GetConfigPath() (path string) { - return svc.cfgPath -} - -func (svc *Service) SetConfigPath(path string) { - svc.cfgPath = path -} - -func (svc *Service) Start() (err error) { - return svc.SyncGit() -} - -func (svc *Service) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { - // spider - s, err := svc.modelSvc.GetSpiderById(id) - if err != nil { - return nil, err - } - - // assign tasks - return svc.scheduleTasks(s, opts) -} - -func (svc *Service) Clone(id primitive.ObjectID, opts *interfaces.SpiderCloneOptions) (err error) { - // TODO: implement - return nil -} - -func (svc *Service) Delete(id primitive.ObjectID) (err error) { - panic("implement me") -} - -func (svc *Service) SyncGit() (err error) { - if _, err = svc.cron.AddFunc("* * * * *", svc.syncGit); err != nil { - return trace.TraceError(err) - } - svc.cron.Start() - return nil -} - -func (svc *Service) SyncGitOne(g interfaces.Git) (err error) { - svc.syncGitOne(g) - return nil -} - -func (svc *Service) Export(id primitive.ObjectID) (filePath string, err error) { - // spider fs - workspacePath := viper.GetString("workspace") - spiderFolderPath := filepath.Join(workspacePath, id.Hex()) - - // zip files in workspace - dirPath := spiderFolderPath - zipFilePath := path.Join(os.TempDir(), uuid.New().String()+".zip") - if err := utils.ZipDirectory(dirPath, zipFilePath); err != nil { - return "", trace.TraceError(err) - } - - return zipFilePath, nil -} - -func (svc *Service) scheduleTasks(s *models.Spider, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { - // main task - mainTask := &models.Task{ - SpiderId: s.Id, - Mode: opts.Mode, - NodeIds: opts.NodeIds, - Cmd: opts.Cmd, - Param: opts.Param, - ScheduleId: opts.ScheduleId, - Priority: opts.Priority, - UserId: opts.UserId, - CreateTs: time.Now(), - } - - // normalize - if mainTask.Mode == "" { - mainTask.Mode = s.Mode - } - if mainTask.NodeIds == nil { - mainTask.NodeIds = s.NodeIds - } - if mainTask.Cmd == "" { - mainTask.Cmd = s.Cmd - } - if mainTask.Param == "" { - mainTask.Param = s.Param - } - if mainTask.Priority == 0 { - mainTask.Priority = s.Priority - } - - if svc.isMultiTask(opts) { - // multi tasks - nodeIds, err := svc.getNodeIds(opts) - if err != nil { - return nil, err - } - for _, nodeId := range nodeIds { - t := &models.Task{ - SpiderId: s.Id, - Mode: opts.Mode, - Cmd: opts.Cmd, - Param: opts.Param, - NodeId: nodeId, - ScheduleId: opts.ScheduleId, - Priority: opts.Priority, - UserId: opts.UserId, - CreateTs: time.Now(), - } - t2, err := svc.schedulerSvc.Enqueue(t) - if err != nil { - return nil, err - } - taskIds = append(taskIds, t2.GetId()) - } - } else { - // single task - nodeIds, err := svc.getNodeIds(opts) - if err != nil { - return nil, err - } - if len(nodeIds) > 0 { - mainTask.NodeId = nodeIds[0] - } - t2, err := svc.schedulerSvc.Enqueue(mainTask) - if err != nil { - return nil, err - } - taskIds = append(taskIds, t2.GetId()) - } - - return taskIds, nil -} - -func (svc *Service) getNodeIds(opts *interfaces.SpiderRunOptions) (nodeIds []primitive.ObjectID, err error) { - if opts.Mode == constants.RunTypeAllNodes { - query := bson.M{ - "active": true, - "enabled": true, - "status": constants.NodeStatusOnline, - } - nodes, err := svc.modelSvc.GetNodeList(query, nil) - if err != nil { - return nil, err - } - for _, node := range nodes { - nodeIds = append(nodeIds, node.GetId()) - } - } else if opts.Mode == constants.RunTypeSelectedNodes { - nodeIds = opts.NodeIds - } - return nodeIds, nil -} - -func (svc *Service) isMultiTask(opts *interfaces.SpiderRunOptions) (res bool) { - if opts.Mode == constants.RunTypeAllNodes { - query := bson.M{ - "active": true, - "enabled": true, - "status": constants.NodeStatusOnline, - } - nodes, err := svc.modelSvc.GetNodeList(query, nil) - if err != nil { - trace.PrintError(err) - return false - } - return len(nodes) > 1 - } else if opts.Mode == constants.RunTypeRandom { - return false - } else if opts.Mode == constants.RunTypeSelectedNodes { - return len(opts.NodeIds) > 1 - } else { - return false - } -} - -func (svc *Service) syncGit() { - if svc.syncLock { - log.Infof("[SpiderAdminService] sync git is locked, skip") - return - } - log.Infof("[SpiderAdminService] start to sync git") - - svc.syncLock = true - defer func() { - svc.syncLock = false - }() - - // spiders - spiders, err := svc.modelSvc.GetSpiderList(nil, nil) - if err != nil { - trace.PrintError(err) - return - } - - // spider ids - var spiderIds []primitive.ObjectID - for _, s := range spiders { - spiderIds = append(spiderIds, s.Id) - } - - if len(spiderIds) > 0 { - // gits - gits, err := svc.modelSvc.GetGitList(bson.M{ - "_id": bson.M{ - "$in": spiderIds, - }, - "auto_pull": true, - }, nil) - if err != nil { - trace.PrintError(err) - return - } - - wg := sync.WaitGroup{} - wg.Add(len(gits)) - for _, g := range gits { - go func(g models.Git) { - svc.syncGitOne(&g) - wg.Done() - }(g) - } - wg.Wait() - } - - log.Infof("[SpiderAdminService] finished sync git") -} - -func (svc *Service) syncGitOne(g interfaces.Git) { - log.Infof("[SpiderAdminService] sync git %s", g.GetId()) - - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - - // git client - workspacePath := viper.GetString("workspace") - gitClient, err := vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, g.GetId().Hex()))) - if err != nil { - return - } - - // set auth - utils.InitGitClientAuth(g, gitClient) - - // check if remote has changes - ok, err := gitClient.IsRemoteChanged() - if err != nil { - trace.PrintError(err) - return - } - if !ok { - // no change - return - } - - // pull and sync to workspace - if err := gitClient.Reset(); err != nil { - trace.PrintError(err) - return - } - if err := gitClient.Pull(); err != nil { - trace.PrintError(err) - return - } - - // wait for context to end - <-ctx.Done() -} - -func NewSpiderAdminService(opts ...Option) (svc2 interfaces.SpiderAdminService, err error) { - svc := &Service{ - cfgPath: config2.GetConfigPath(), - } - - // apply options - for _, opt := range opts { - opt(svc) - } - - // dependency injection - if err := container.GetContainer().Invoke(func(nodeCfgSvc interfaces.NodeConfigService, modelSvc service.ModelService, schedulerSvc interfaces.TaskSchedulerService) { - svc.nodeCfgSvc = nodeCfgSvc - svc.modelSvc = modelSvc - svc.schedulerSvc = schedulerSvc - }); err != nil { - return nil, trace.TraceError(err) - } - - // cron - svc.cron = cron.New() - - // validate node type - if !svc.nodeCfgSvc.IsMaster() { - return nil, trace.TraceError(errors.ErrorSpiderForbidden) - } - - return svc, nil -} - -var _service interfaces.SpiderAdminService - -func GetSpiderAdminService() (svc2 interfaces.SpiderAdminService, err error) { - if _service != nil { - return _service, nil - } - - _service, err = NewSpiderAdminService() - if err != nil { - return nil, err - } - - return _service, nil -} diff --git a/core/stats/options.go b/core/stats/options.go deleted file mode 100644 index b40e16c3..00000000 --- a/core/stats/options.go +++ /dev/null @@ -1,5 +0,0 @@ -package stats - -import "github.com/crawlab-team/crawlab/core/interfaces" - -type Option func(svc interfaces.StatsService) diff --git a/core/stats/service.go b/core/stats/service.go index 75c803c8..ae34ca69 100644 --- a/core/stats/service.go +++ b/core/stats/service.go @@ -4,6 +4,7 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/db/mongo" "go.mongodb.org/mongo-driver/bson" mongo2 "go.mongodb.org/mongo-driver/mongo" @@ -16,7 +17,7 @@ func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) stats := bson.M{} // nodes - stats["nodes"], err = mongo.GetMongoCol(interfaces.ModelColNameNode).Count(bson.M{"active": true}) + stats["nodes"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.NodeV2{})).Count(bson.M{"active": true}) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -25,7 +26,7 @@ func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) } // projects - stats["projects"], err = mongo.GetMongoCol(interfaces.ModelColNameProject).Count(nil) + stats["projects"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.ProjectV2{})).Count(nil) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -34,7 +35,7 @@ func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) } // spiders - stats["spiders"], err = mongo.GetMongoCol(interfaces.ModelColNameSpider).Count(nil) + stats["spiders"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.SpiderV2{})).Count(nil) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -43,7 +44,7 @@ func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) } // schedules - stats["schedules"], err = mongo.GetMongoCol(interfaces.ModelColNameSchedule).Count(nil) + stats["schedules"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.ScheduleV2{})).Count(nil) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -52,7 +53,7 @@ func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) } // tasks - stats["tasks"], err = mongo.GetMongoCol(interfaces.ModelColNameTask).Count(nil) + stats["tasks"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskV2{})).Count(nil) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -61,7 +62,7 @@ func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) } // error tasks - stats["error_tasks"], err = mongo.GetMongoCol(interfaces.ModelColNameTask).Count(bson.M{"status": constants.TaskStatusError}) + stats["error_tasks"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskV2{})).Count(bson.M{"status": constants.TaskStatusError}) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -79,7 +80,7 @@ func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) } // users - stats["users"], err = mongo.GetMongoCol(interfaces.ModelColNameUser).Count(nil) + stats["users"], err = mongo.GetMongoCol(models.GetCollectionNameByInstance(models.UserV2{})).Count(nil) if err != nil { if err.Error() != mongo2.ErrNoDocuments.Error() { return nil, err @@ -153,7 +154,7 @@ func (svc *Service) getDailyTasksStats(query bson.M) (data interface{}, err erro }}, } var results []entity.StatsDailyItem - if err := mongo.GetMongoCol(interfaces.ModelColNameTaskStat).Aggregate(pipeline, nil).All(&results); err != nil { + if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskStatV2{})).Aggregate(pipeline, nil).All(&results); err != nil { return nil, err } return results, nil @@ -171,7 +172,7 @@ func (svc *Service) getOverviewResults(query bson.M) (data interface{}, err erro }}, } var res bson.M - if err := mongo.GetMongoCol(interfaces.ModelColNameTaskStat).Aggregate(pipeline, nil).One(&res); err != nil { + if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskStatV2{})).Aggregate(pipeline, nil).One(&res); err != nil { return nil, err } return res["results"], nil @@ -196,7 +197,7 @@ func (svc *Service) getTaskStatsByStatus(query bson.M) (data interface{}, err er }}, } var results []bson.M - if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).All(&results); err != nil { + if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskV2{})).Aggregate(pipeline, nil).All(&results); err != nil { return nil, err } return results, nil @@ -215,7 +216,7 @@ func (svc *Service) getTaskStatsByNode(query bson.M) (data interface{}, err erro {{ "$lookup", bson.M{ - "from": interfaces.ModelColNameNode, + "from": models.GetCollectionNameByInstance(models.NodeV2{}), "localField": "_id", "foreignField": "_id", "as": "_n", @@ -232,7 +233,7 @@ func (svc *Service) getTaskStatsByNode(query bson.M) (data interface{}, err erro }}, } var results []bson.M - if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).All(&results); err != nil { + if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskV2{})).Aggregate(pipeline, nil).All(&results); err != nil { return nil, err } return results, nil @@ -251,7 +252,7 @@ func (svc *Service) getTaskStatsBySpider(query bson.M) (data interface{}, err er {{ "$lookup", bson.M{ - "from": interfaces.ModelColNameSpider, + "from": models.GetCollectionNameByInstance(models.SpiderV2{}), "localField": "_id", "foreignField": "_id", "as": "_s", @@ -269,7 +270,7 @@ func (svc *Service) getTaskStatsBySpider(query bson.M) (data interface{}, err er {{"$limit", 10}}, } var results []bson.M - if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).All(&results); err != nil { + if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskV2{})).Aggregate(pipeline, nil).All(&results); err != nil { return nil, err } return results, nil @@ -281,7 +282,7 @@ func (svc *Service) getTaskStatsHistogram(query bson.M) (data interface{}, err e {{ "$lookup", bson.M{ - "from": interfaces.ModelColNameTaskStat, + "from": models.GetCollectionNameByInstance(models.TaskStatV2{}), "localField": "_id", "foreignField": "_id", "as": "_ts", @@ -303,7 +304,7 @@ func (svc *Service) getTaskStatsHistogram(query bson.M) (data interface{}, err e }}, } var res bson.M - if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).One(&res); err != nil { + if err := mongo.GetMongoCol(models.GetCollectionNameByInstance(models.TaskV2{})).Aggregate(pipeline, nil).One(&res); err != nil { return nil, err } return res, nil diff --git a/core/system/service.go b/core/system/service.go deleted file mode 100644 index 94cd3140..00000000 --- a/core/system/service.go +++ /dev/null @@ -1,89 +0,0 @@ -package system - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - mongo2 "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Service struct { - col *mongo2.Col - modelSvc service.ModelService -} - -func (svc *Service) Init() (err error) { - // initialize data - if err := svc.initData(); err != nil { - return err - } - - return nil -} - -func (svc *Service) initData() (err error) { - total, err := svc.col.Count(bson.M{ - "key": "customize", - }) - if err != nil { - return err - } - if total > 0 { - return nil - } - - // data to initialize - settings := []models.Setting{ - { - Id: primitive.NewObjectID(), - Key: "customize", - Value: bson.M{ - "show_custom_title": false, - "custom_title": "", - "show_custom_logo": false, - "custom_logo": "", - "hide_platform_version": false, - }, - }, - } - var data []interface{} - for _, s := range settings { - data = append(data, s) - } - _, err = svc.col.InsertMany(data) - if err != nil { - return err - } - return nil -} - -func NewService() *Service { - // service - svc := &Service{ - col: mongo2.GetMongoCol(interfaces.ModelColNameSetting), - } - - // model service - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - svc.modelSvc = modelSvc - - if err := svc.Init(); err != nil { - panic(err) - } - - return svc -} - -var _service *Service - -func GetService() *Service { - if _service == nil { - _service = NewService() - } - return _service -} diff --git a/core/task/handler/options.go b/core/task/handler/options.go deleted file mode 100644 index c2c03558..00000000 --- a/core/task/handler/options.go +++ /dev/null @@ -1,40 +0,0 @@ -package handler - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "time" -) - -type Option func(svc interfaces.TaskHandlerService) - -func WithConfigPath(path string) Option { - return func(svc interfaces.TaskHandlerService) { - svc.SetConfigPath(path) - } -} - -func WithExitWatchDuration(duration time.Duration) Option { - return func(svc interfaces.TaskHandlerService) { - svc.SetExitWatchDuration(duration) - } -} - -func WithReportInterval(interval time.Duration) Option { - return func(svc interfaces.TaskHandlerService) { - svc.SetReportInterval(interval) - } -} - -func WithCancelTimeout(timeout time.Duration) Option { - return func(svc interfaces.TaskHandlerService) { - svc.SetCancelTimeout(timeout) - } -} - -type RunnerOption func(r interfaces.TaskRunner) - -func WithSubscribeTimeout(timeout time.Duration) RunnerOption { - return func(r interfaces.TaskRunner) { - r.SetSubscribeTimeout(timeout) - } -} diff --git a/core/task/handler/runner_test.go b/core/task/handler/runner_test.go deleted file mode 100644 index 175117a3..00000000 --- a/core/task/handler/runner_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package handler - -import ( - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/google/uuid" - "github.com/spf13/viper" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -type MockRunner struct { - mock.Mock - Runner -} - -func (m *MockRunner) downloadFile(url string, filePath string) error { - args := m.Called(url, filePath) - return args.Error(0) -} - -func newMockRunner() *MockRunner { - r := &MockRunner{} - r.s = &models.Spider{} - return r -} - -func TestSyncFiles_SuccessWithDummyFiles(t *testing.T) { - // Create a test server that responds with a list of files - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasSuffix(r.URL.Path, "/scan") { - w.Write([]byte(`{"file1.txt":{"path": "file1.txt", "hash": "hash1"}, "file2.txt":{"path": "file2.txt", "hash": "hash2"}}`)) - return - } - - if strings.HasSuffix(r.URL.Path, "/download") { - w.Write([]byte("file content")) - return - } - })) - defer ts.Close() - - // Create a mock runner - r := newMockRunner() - r.On("downloadFile", mock.Anything, mock.Anything).Return(nil) - - // Set the master URL to the test server URL - viper.Set("api.endpoint", ts.URL) - - localPath := filepath.Join(os.TempDir(), uuid.New().String()) - os.MkdirAll(filepath.Join(localPath, r.s.GetId().Hex()), os.ModePerm) - defer os.RemoveAll(localPath) - viper.Set("workspace", localPath) - - // Call the method under test - err := r.syncFiles() - assert.NoError(t, err) - - // Assert that the files were downloaded - assert.FileExists(t, filepath.Join(localPath, r.s.GetId().Hex(), "file1.txt")) - assert.FileExists(t, filepath.Join(localPath, r.s.GetId().Hex(), "file2.txt")) -} - -func TestSyncFiles_DeletesFilesNotOnMaster(t *testing.T) { - // Create a test server that responds with an empty list of files - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasSuffix(r.URL.Path, "/scan") { - w.Write([]byte(`{}`)) - return - } - })) - defer ts.Close() - - // Create a mock runner - r := newMockRunner() - r.On("downloadFile", mock.Anything, mock.Anything).Return(nil) - - // Set the master URL to the test server URL - viper.Set("api.endpoint", ts.URL) - - localPath := filepath.Join(os.TempDir(), uuid.New().String()) - os.MkdirAll(filepath.Join(localPath, r.s.GetId().Hex()), os.ModePerm) - defer os.RemoveAll(localPath) - viper.Set("workspace", localPath) - - // Create a dummy file that should be deleted - dummyFilePath := filepath.Join(localPath, r.s.GetId().Hex(), "dummy.txt") - dummyFile, _ := os.Create(dummyFilePath) - dummyFile.Close() - - // Call the method under test - err := r.syncFiles() - assert.NoError(t, err) - - // Assert that the dummy file was deleted - assert.NoFileExists(t, dummyFilePath) -} diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go index 3788661f..8690aa6e 100644 --- a/core/task/handler/runner_v2.go +++ b/core/task/handler/runner_v2.go @@ -10,13 +10,12 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" - fs2 "github.com/crawlab-team/crawlab/core/fs" + "github.com/crawlab-team/crawlab/core/fs" client2 "github.com/crawlab-team/crawlab/core/grpc/client" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/models" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" - service2 "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/sys_exec" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/grpc" @@ -45,17 +44,16 @@ type RunnerV2 struct { bufferSize int // internals - cmd *exec.Cmd // process command instance - pid int // process id - tid primitive.ObjectID // task id - t *models2.TaskV2 // task model.Task - s *models2.SpiderV2 // spider model.Spider - ch chan constants.TaskSignal // channel to communicate between Service and RunnerV2 - err error // standard process error - envs []models.Env // environment variables - cwd string // working directory - c *client2.GrpcClientV2 // grpc client - sub grpc.TaskService_SubscribeClient // grpc task service stream client + cmd *exec.Cmd // process command instance + pid int // process id + tid primitive.ObjectID // task id + t *models.TaskV2 // task model.Task + s *models.SpiderV2 // spider model.Spider + ch chan constants.TaskSignal // channel to communicate between Service and RunnerV2 + err error // standard process error + cwd string // working directory + c *client2.GrpcClientV2 // grpc client + sub grpc.TaskService_SubscribeClient // grpc task service stream client // log internals scannerStdout *bufio.Reader @@ -316,7 +314,7 @@ func (r *RunnerV2) configureEnv() { } // global environment variables - envs, err := client.NewModelServiceV2[models2.EnvironmentV2]().GetMany(nil, nil) + envs, err := client.NewModelServiceV2[models.EnvironmentV2]().GetMany(nil, nil) if err != nil { trace.PrintError(err) return @@ -511,12 +509,12 @@ func (r *RunnerV2) updateTask(status string, e error) (err error) { r.t.Error = e.Error() } if r.svc.GetNodeConfigService().IsMaster() { - err = service2.NewModelServiceV2[models2.TaskV2]().ReplaceById(r.t.Id, *r.t) + err = service.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t) if err != nil { return err } } else { - err = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(r.t.Id, *r.t) + err = client.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t) if err != nil { return err } @@ -567,7 +565,7 @@ func (r *RunnerV2) writeLogLines(lines []string) { } func (r *RunnerV2) _updateTaskStat(status string) { - ts, err := client.NewModelServiceV2[models2.TaskStatV2]().GetById(r.tid) + ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid) if err != nil { trace.PrintError(err) return @@ -588,13 +586,13 @@ func (r *RunnerV2) _updateTaskStat(status string) { ts.TotalDuration = ts.EndTs.Sub(ts.BaseModelV2.CreatedAt).Milliseconds() } if r.svc.GetNodeConfigService().IsMaster() { - err = service2.NewModelServiceV2[models2.TaskStatV2]().ReplaceById(ts.Id, *ts) + err = service.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts) if err != nil { trace.PrintError(err) return } } else { - err = client.NewModelServiceV2[models2.TaskStatV2]().ReplaceById(ts.Id, *ts) + err = client.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts) if err != nil { trace.PrintError(err) return @@ -617,7 +615,7 @@ func (r *RunnerV2) sendNotification() { func (r *RunnerV2) _updateSpiderStat(status string) { // task stat - ts, err := client.NewModelServiceV2[models2.TaskStatV2]().GetById(r.tid) + ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid) if err != nil { trace.PrintError(err) return @@ -655,13 +653,13 @@ func (r *RunnerV2) _updateSpiderStat(status string) { // perform update if r.svc.GetNodeConfigService().IsMaster() { - err = service2.NewModelServiceV2[models2.SpiderStatV2]().UpdateById(r.s.Id, update) + err = service.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update) if err != nil { trace.PrintError(err) return } } else { - err = client.NewModelServiceV2[models2.SpiderStatV2]().UpdateById(r.s.Id, update) + err = client.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update) if err != nil { trace.PrintError(err) return @@ -709,7 +707,7 @@ func NewTaskRunnerV2(id primitive.ObjectID, svc *ServiceV2) (r2 *RunnerV2, err e } // task fs service - r.fsSvc = fs2.NewFsServiceV2(filepath.Join(viper.GetString("workspace"), r.s.Id.Hex())) + r.fsSvc = fs.NewFsServiceV2(filepath.Join(viper.GetString("workspace"), r.s.Id.Hex())) // grpc client r.c = client2.GetGrpcClientV2() diff --git a/core/task/log/constants.go b/core/task/log/constants.go index 73676851..d89ac7a6 100644 --- a/core/task/log/constants.go +++ b/core/task/log/constants.go @@ -6,7 +6,6 @@ const ( const ( DriverTypeFile = "file" // raw file - DriverTypeFs = "fs" // file system (SeaweedFS) DriverTypeMongo = "mongo" // mongodb DriverTypeEs = "es" // elastic search ) diff --git a/core/task/scheduler/options.go b/core/task/scheduler/options.go deleted file mode 100644 index a6600fab..00000000 --- a/core/task/scheduler/options.go +++ /dev/null @@ -1,20 +0,0 @@ -package scheduler - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "time" -) - -type Option func(svc interfaces.TaskSchedulerService) - -func WithConfigPath(path string) Option { - return func(svc interfaces.TaskSchedulerService) { - svc.SetConfigPath(path) - } -} - -func WithInterval(interval time.Duration) Option { - return func(svc interfaces.TaskSchedulerService) { - svc.SetInterval(interval) - } -} diff --git a/core/task/stats/options.go b/core/task/stats/options.go deleted file mode 100644 index 37e7a823..00000000 --- a/core/task/stats/options.go +++ /dev/null @@ -1,11 +0,0 @@ -package stats - -import "github.com/crawlab-team/crawlab/core/interfaces" - -type Option func(service interfaces.TaskStatsService) - -func WithConfigPath(path string) Option { - return func(svc interfaces.TaskStatsService) { - svc.SetConfigPath(path) - } -} diff --git a/core/user/options.go b/core/user/options.go deleted file mode 100644 index 81c208d7..00000000 --- a/core/user/options.go +++ /dev/null @@ -1,11 +0,0 @@ -package user - -import "github.com/crawlab-team/crawlab/core/interfaces" - -type Option func(svc interfaces.UserService) - -func WithJwtSecret(secret string) Option { - return func(svc interfaces.UserService) { - svc.SetJwtSecret(secret) - } -} diff --git a/core/user/service.go b/core/user/service.go deleted file mode 100644 index 7294df4f..00000000 --- a/core/user/service.go +++ /dev/null @@ -1,238 +0,0 @@ -package user - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - mongo2 "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "github.com/golang-jwt/jwt/v5" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "time" -) - -type Service struct { - // settings variables - jwtSecret string - jwtSigningMethod jwt.SigningMethod - - // dependencies - modelSvc service.ModelService -} - -func (svc *Service) Init() (err error) { - _, err = svc.modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil) - if err == nil { - return nil - } - if err.Error() != mongo.ErrNoDocuments.Error() { - return err - } - return svc.Create(&interfaces.UserCreateOptions{ - Username: constants.DefaultAdminUsername, - Password: constants.DefaultAdminPassword, - Role: constants.RoleAdmin, - }) -} - -func (svc *Service) SetJwtSecret(secret string) { - svc.jwtSecret = secret -} - -func (svc *Service) SetJwtSigningMethod(method jwt.SigningMethod) { - svc.jwtSigningMethod = method -} - -func (svc *Service) Create(opts *interfaces.UserCreateOptions, args ...interface{}) (err error) { - actor := utils.GetUserFromArgs(args...) - - // validate options - if opts.Username == "" || opts.Password == "" { - return trace.TraceError(errors.ErrorUserMissingRequiredFields) - } - if len(opts.Password) < 5 { - return trace.TraceError(errors.ErrorUserInvalidPassword) - } - - // normalize options - if opts.Role == "" { - opts.Role = constants.RoleNormal - } - - // check if user exists - if u, err := svc.modelSvc.GetUserByUsername(opts.Username, nil); err == nil && u != nil && !u.Id.IsZero() { - return trace.TraceError(errors.ErrorUserAlreadyExists) - } - - // transaction - return mongo2.RunTransaction(func(ctx mongo.SessionContext) error { - // add user - u := &models.User{ - Username: opts.Username, - Role: opts.Role, - Email: opts.Email, - } - if err := delegate.NewModelDelegate(u, actor).Add(); err != nil { - return err - } - - // add password - p := &models.Password{ - Id: u.Id, - Password: utils.EncryptMd5(opts.Password), - } - if err := delegate.NewModelDelegate(p, actor).Add(); err != nil { - return err - } - - return nil - }) -} - -func (svc *Service) Login(opts *interfaces.UserLoginOptions) (token string, u interfaces.User, err error) { - u, err = svc.modelSvc.GetUserByUsername(opts.Username, nil) - if err != nil { - return "", nil, err - } - p, err := svc.modelSvc.GetPasswordById(u.GetId()) - if err != nil { - return "", nil, err - } - if p.Password != utils.EncryptMd5(opts.Password) { - return "", nil, errors.ErrorUserMismatch - } - token, err = svc.makeToken(u) - if err != nil { - return "", nil, err - } - return token, u, nil -} - -func (svc *Service) CheckToken(tokenStr string) (u interfaces.User, err error) { - return svc.checkToken(tokenStr) -} - -func (svc *Service) ChangePassword(id primitive.ObjectID, password string, args ...interface{}) (err error) { - actor := utils.GetUserFromArgs(args...) - - p, err := svc.modelSvc.GetPasswordById(id) - if err != nil { - return err - } - p.Password = utils.EncryptMd5(password) - if err := delegate.NewModelDelegate(p, actor).Save(); err != nil { - return err - } - return nil -} - -func (svc *Service) MakeToken(user interfaces.User) (tokenStr string, err error) { - return svc.makeToken(user) -} - -func (svc *Service) GetCurrentUser(c *gin.Context) (user interfaces.User, err error) { - // token string - tokenStr := c.GetHeader("Authorization") - - // user - u, err := userSvc.CheckToken(tokenStr) - if err != nil { - return nil, err - } - - return u, nil -} - -func (svc *Service) makeToken(user interfaces.User) (tokenStr string, err error) { - token := jwt.NewWithClaims(svc.jwtSigningMethod, jwt.MapClaims{ - "id": user.GetId(), - "username": user.GetUsername(), - "nbf": time.Now().Unix(), - }) - return token.SignedString([]byte(svc.jwtSecret)) -} - -func (svc *Service) checkToken(tokenStr string) (user interfaces.User, err error) { - token, err := jwt.Parse(tokenStr, svc.getSecretFunc()) - if err != nil { - return - } - - claim, ok := token.Claims.(jwt.MapClaims) - if !ok { - err = errors.ErrorUserInvalidType - return - } - - if !token.Valid { - err = errors.ErrorUserInvalidToken - return - } - - id, err := primitive.ObjectIDFromHex(claim["id"].(string)) - if err != nil { - return user, err - } - username := claim["username"].(string) - user, err = svc.modelSvc.GetUserById(id) - if err != nil { - err = errors.ErrorUserNotExists - return - } - - if username != user.GetUsername() { - err = errors.ErrorUserMismatch - return - } - - return -} - -func (svc *Service) getSecretFunc() jwt.Keyfunc { - return func(token *jwt.Token) (interface{}, error) { - return []byte(svc.jwtSecret), nil - } -} - -func NewUserService() (svc2 interfaces.UserService, err error) { - // service - svc := &Service{ - jwtSecret: "crawlab", - jwtSigningMethod: jwt.SigningMethodHS256, - } - - // dependency injection - if err := container.GetContainer().Invoke(func(modelSvc service.ModelService) { - svc.modelSvc = modelSvc - }); err != nil { - return nil, trace.TraceError(err) - } - - // initialize - if err := svc.Init(); err != nil { - return nil, trace.TraceError(err) - } - - return svc, nil -} - -var userSvc interfaces.UserService - -func GetUserService() (svc interfaces.UserService, err error) { - if userSvc != nil { - return userSvc, nil - } - svc, err = NewUserService() - if err != nil { - return nil, err - } - userSvc = svc - return svc, nil -} diff --git a/core/user/service_v2.go b/core/user/service_v2.go index 696b7d95..eb618a16 100644 --- a/core/user/service_v2.go +++ b/core/user/service_v2.go @@ -4,13 +4,11 @@ import ( "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" mongo2 "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" "github.com/golang-jwt/jwt/v5" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -119,19 +117,6 @@ func (svc *ServiceV2) MakeToken(user *models.UserV2) (tokenStr string, err error return svc.makeToken(user) } -func (svc *ServiceV2) GetCurrentUser(c *gin.Context) (user interfaces.User, err error) { - // token string - tokenStr := c.GetHeader("Authorization") - - // user - u, err := userSvc.CheckToken(tokenStr) - if err != nil { - return nil, err - } - - return u, nil -} - func (svc *ServiceV2) makeToken(user *models.UserV2) (tokenStr string, err error) { token := jwt.NewWithClaims(svc.jwtSigningMethod, jwt.MapClaims{ "id": user.Id, diff --git a/core/user/test/base.go b/core/user/test/base.go deleted file mode 100644 index 392a7e06..00000000 --- a/core/user/test/base.go +++ /dev/null @@ -1,65 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/user" - "go.uber.org/dig" - "testing" -) - -func init() { - var err error - T, err = NewTest() - if err != nil { - panic(err) - } -} - -var T *Test - -type Test struct { - // dependencies - modelSvc service.ModelService - userSvc interfaces.UserService - - // test data - TestUsername string - TestPassword string - TestNewPassword string -} - -func (t *Test) Setup(t2 *testing.T) { - var err error - t.userSvc, err = user.NewUserService() - if err != nil { - panic(err) - } - t2.Cleanup(t.Cleanup) -} - -func (t *Test) Cleanup() { - _ = t.modelSvc.GetBaseService(interfaces.ModelIdUser).DeleteList(nil) -} - -func NewTest() (t *Test, err error) { - // test - t = &Test{ - TestUsername: "test_username", - TestPassword: "test_password", - TestNewPassword: "test_new_password", - } - - // dependency injection - c := dig.New() - if err := c.Provide(service.GetService); err != nil { - return nil, err - } - if err := c.Invoke(func(modelSvc service.ModelService) { - t.modelSvc = modelSvc - }); err != nil { - return nil, err - } - - return t, nil -} diff --git a/core/user/test/user_service_test.go b/core/user/test/user_service_test.go deleted file mode 100644 index 0806c31e..00000000 --- a/core/user/test/user_service_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/stretchr/testify/require" - "testing" -) - -func TestUserService_Init(t *testing.T) { - var err error - T.Setup(t) - - u, err := T.modelSvc.GetUserByUsernameWithPassword(constants.DefaultAdminUsername, nil) - require.Nil(t, err) - require.Equal(t, constants.DefaultAdminUsername, u.Username) - require.Equal(t, utils.EncryptMd5(constants.DefaultAdminPassword), u.Password) -} - -func TestUserService_Create_Login_CheckToken(t *testing.T) { - var err error - T.Setup(t) - - err = T.userSvc.Create(&interfaces.UserCreateOptions{ - Username: T.TestUsername, - Password: T.TestPassword, - }) - require.Nil(t, err) - - u, err := T.modelSvc.GetUserByUsernameWithPassword(T.TestUsername, nil) - require.Nil(t, err) - require.Equal(t, T.TestUsername, u.Username) - require.Equal(t, utils.EncryptMd5(T.TestPassword), u.Password) - - token, u2, err := T.userSvc.Login(&interfaces.UserLoginOptions{ - Username: T.TestUsername, - Password: T.TestPassword, - }) - require.Nil(t, err) - require.Greater(t, len(token), 10) - require.Equal(t, u.Username, u2.GetUsername()) - - u3, err := T.userSvc.CheckToken(token) - require.Nil(t, err) - require.Equal(t, u.Username, u3.GetUsername()) -} - -func TestUserService_ChangePassword(t *testing.T) { - var err error - T.Setup(t) - - u, err := T.modelSvc.GetUserByUsernameWithPassword(constants.DefaultAdminUsername, nil) - require.Nil(t, err) - err = T.userSvc.ChangePassword(u.Id, T.TestNewPassword) - require.Nil(t, err) - - u2, err := T.modelSvc.GetUserByUsernameWithPassword(constants.DefaultAdminUsername, nil) - require.Nil(t, err) - require.Equal(t, utils.EncryptMd5(T.TestNewPassword), u2.Password) -} diff --git a/core/utils/args.go b/core/utils/args.go deleted file mode 100644 index e83f540a..00000000 --- a/core/utils/args.go +++ /dev/null @@ -1,17 +0,0 @@ -package utils - -import "github.com/crawlab-team/crawlab/core/interfaces" - -func GetUserFromArgs(args ...interface{}) (u interfaces.User) { - for _, arg := range args { - switch arg.(type) { - case interfaces.User: - var ok bool - u, ok = arg.(interfaces.User) - if ok { - return u - } - } - } - return nil -} diff --git a/core/utils/array.go b/core/utils/array.go deleted file mode 100644 index a5e958c6..00000000 --- a/core/utils/array.go +++ /dev/null @@ -1,46 +0,0 @@ -package utils - -import ( - "errors" - "math/rand" - "reflect" - "time" -) - -func StringArrayContains(arr []string, str string) bool { - for _, s := range arr { - if s == str { - return true - } - } - return false -} - -func GetArrayItems(array interface{}) (res []interface{}, err error) { - switch reflect.TypeOf(array).Kind() { - case reflect.Slice, reflect.Array: - s := reflect.ValueOf(array) - for i := 0; i < s.Len(); i++ { - obj, ok := s.Index(i).Interface().(interface{}) - if !ok { - return nil, errors.New("invalid type") - } - res = append(res, obj) - } - default: - return nil, errors.New("invalid type") - } - return res, nil -} - -func ShuffleArray(slice []interface{}) (err error) { - r := rand.New(rand.NewSource(time.Now().Unix())) - for len(slice) > 0 { - n := len(slice) - randIndex := r.Intn(n) - slice[n-1], slice[randIndex] = slice[randIndex], slice[n-1] - slice = slice[:n-1] - } - - return nil -} diff --git a/core/utils/binders/binder_col_name.go b/core/utils/binders/binder_col_name.go deleted file mode 100644 index 9e76224d..00000000 --- a/core/utils/binders/binder_col_name.go +++ /dev/null @@ -1,99 +0,0 @@ -package binders - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" -) - -func NewColNameBinder(id interfaces.ModelId) (b *ColNameBinder) { - return &ColNameBinder{id: id} -} - -type ColNameBinder struct { - id interfaces.ModelId -} - -func (b *ColNameBinder) Bind() (res interface{}, err error) { - switch b.id { - // system models - case interfaces.ModelIdArtifact: - return interfaces.ModelColNameArtifact, nil - case interfaces.ModelIdTag: - return interfaces.ModelColNameTag, nil - - // operation models - case interfaces.ModelIdNode: - return interfaces.ModelColNameNode, nil - case interfaces.ModelIdProject: - return interfaces.ModelColNameProject, nil - case interfaces.ModelIdSpider: - return interfaces.ModelColNameSpider, nil - case interfaces.ModelIdTask: - return interfaces.ModelColNameTask, nil - case interfaces.ModelIdJob: - return interfaces.ModelColNameJob, nil - case interfaces.ModelIdSchedule: - return interfaces.ModelColNameSchedule, nil - case interfaces.ModelIdUser: - return interfaces.ModelColNameUser, nil - case interfaces.ModelIdSetting: - return interfaces.ModelColNameSetting, nil - case interfaces.ModelIdToken: - return interfaces.ModelColNameToken, nil - case interfaces.ModelIdVariable: - return interfaces.ModelColNameVariable, nil - case interfaces.ModelIdTaskQueue: - return interfaces.ModelColNameTaskQueue, nil - case interfaces.ModelIdTaskStat: - return interfaces.ModelColNameTaskStat, nil - case interfaces.ModelIdSpiderStat: - return interfaces.ModelColNameSpiderStat, nil - case interfaces.ModelIdDataSource: - return interfaces.ModelColNameDataSource, nil - case interfaces.ModelIdDataCollection: - return interfaces.ModelColNameDataCollection, nil - case interfaces.ModelIdPassword: - return interfaces.ModelColNamePasswords, nil - case interfaces.ModelIdExtraValue: - return interfaces.ModelColNameExtraValues, nil - case interfaces.ModelIdGit: - return interfaces.ModelColNameGit, nil - case interfaces.ModelIdRole: - return interfaces.ModelColNameRole, nil - case interfaces.ModelIdUserRole: - return interfaces.ModelColNameUserRole, nil - case interfaces.ModelIdPermission: - return interfaces.ModelColNamePermission, nil - case interfaces.ModelIdRolePermission: - return interfaces.ModelColNameRolePermission, nil - case interfaces.ModelIdEnvironment: - return interfaces.ModelColNameEnvironment, nil - case interfaces.ModelIdDependencySetting: - return interfaces.ModelColNameDependencySetting, nil - - // invalid - default: - return res, errors.ErrorModelNotImplemented - } -} - -func (b *ColNameBinder) MustBind() (res interface{}) { - res, err := b.Bind() - if err != nil { - panic(err) - } - return res -} - -func (b *ColNameBinder) BindString() (res string, err error) { - res_, err := b.Bind() - if err != nil { - return "", err - } - res = res_.(string) - return res, nil -} - -func (b *ColNameBinder) MustBindString() (res string) { - return b.MustBind().(string) -} diff --git a/core/utils/cache.go b/core/utils/cache.go deleted file mode 100644 index e64e147b..00000000 --- a/core/utils/cache.go +++ /dev/null @@ -1,57 +0,0 @@ -package utils - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "time" -) - -func GetFromDbCache(key string, getFn func() (string, error)) (res string, err error) { - col := mongo.GetMongoCol(constants.CacheColName) - - var d bson.M - if err := col.Find(bson.M{ - constants.CacheColKey: key, - }, nil).One(&d); err != nil { - if err != mongo2.ErrNoDocuments { - return "", err - } - - // get cache value - res, err = getFn() - if err != nil { - return "", err - } - - // save cache - d = bson.M{ - constants.CacheColKey: key, - constants.CacheColValue: res, - constants.CacheColTime: time.Now(), - } - if _, err := col.Insert(d); err != nil { - return "", err - } - return res, nil - } - - // type conversion - r, ok := d[constants.CacheColValue] - if !ok { - if err := col.Delete(bson.M{constants.CacheColKey: key}); err != nil { - return "", err - } - return GetFromDbCache(key, getFn) - } - res, ok = r.(string) - if !ok { - if err := col.Delete(bson.M{constants.CacheColKey: key}); err != nil { - return "", err - } - return GetFromDbCache(key, getFn) - } - - return res, nil -} diff --git a/core/utils/chan.go b/core/utils/chan.go deleted file mode 100644 index c0144340..00000000 --- a/core/utils/chan.go +++ /dev/null @@ -1,40 +0,0 @@ -package utils - -import ( - "sync" -) - -var TaskExecChanMap = NewChanMap() - -type ChanMap struct { - m sync.Map -} - -func NewChanMap() *ChanMap { - return &ChanMap{m: sync.Map{}} -} - -func (cm *ChanMap) Chan(key string) chan string { - if ch, ok := cm.m.Load(key); ok { - return ch.(interface{}).(chan string) - } - ch := make(chan string, 10) - cm.m.Store(key, ch) - return ch -} - -func (cm *ChanMap) ChanBlocked(key string) chan string { - if ch, ok := cm.m.Load(key); ok { - return ch.(interface{}).(chan string) - } - ch := make(chan string) - cm.m.Store(key, ch) - return ch -} - -func (cm *ChanMap) HasChanKey(key string) bool { - if _, ok := cm.m.Load(key); ok { - return true - } - return false -} diff --git a/core/utils/chan_test.go b/core/utils/chan_test.go deleted file mode 100644 index 4bc75917..00000000 --- a/core/utils/chan_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package utils - -import ( - . "github.com/smartystreets/goconvey/convey" - "sync" - "testing" -) - -func TestNewChanMap(t *testing.T) { - mapTest := sync.Map{} - chanTest := make(chan string) - test := "test" - - Convey("Call NewChanMap to generate ChanMap", t, func() { - mapTest.Store("test", chanTest) - chanMapTest := ChanMap{mapTest} - chanMap := NewChanMap() - chanMap.m.Store("test", chanTest) - - Convey(test, func() { - v1, ok := chanMap.m.Load("test") - So(ok, ShouldBeTrue) - v2, ok := chanMapTest.m.Load("test") - So(ok, ShouldBeTrue) - So(v1, ShouldResemble, v2) - }) - }) -} - -func TestChan(t *testing.T) { - mapTest := sync.Map{} - chanTest := make(chan string) - mapTest.Store("test", chanTest) - chanMapTest := ChanMap{mapTest} - - Convey("Test Chan use exist key", t, func() { - ch1 := chanMapTest.Chan("test") - Convey("ch1 should equal chanTest", func() { - So(ch1, ShouldEqual, chanTest) - }) - }) - Convey("Test Chan use no-exist key", t, func() { - ch2 := chanMapTest.Chan("test2") - Convey("ch2 should equal chanMapTest.m[test2]", func() { - v, ok := chanMapTest.m.Load("test2") - So(ok, ShouldBeTrue) - So(v, ShouldEqual, ch2) - }) - Convey("Cap of chanMapTest.m[test2] should equal 10", func() { - So(10, ShouldEqual, cap(ch2)) - }) - }) -} - -func TestChanBlocked(t *testing.T) { - mapTest := sync.Map{} - chanTest := make(chan string) - mapTest.Store("test", chanTest) - chanMapTest := ChanMap{mapTest} - - Convey("Test Chan use exist key", t, func() { - ch1 := chanMapTest.ChanBlocked("test") - Convey("ch1 should equal chanTest", func() { - So(ch1, ShouldEqual, chanTest) - }) - }) - Convey("Test Chan use no-exist key", t, func() { - ch2 := chanMapTest.ChanBlocked("test2") - Convey("ch2 should equal chanMapTest.m[test2]", func() { - v, ok := chanMapTest.m.Load("test2") - So(ok, ShouldBeTrue) - So(v, ShouldEqual, ch2) - }) - Convey("Cap of chanMapTest.m[test2] should equal 10", func() { - So(0, ShouldEqual, cap(ch2)) - }) - }) -} diff --git a/core/utils/cockroachdb.go b/core/utils/cockroachdb.go deleted file mode 100644 index 012da375..00000000 --- a/core/utils/cockroachdb.go +++ /dev/null @@ -1,106 +0,0 @@ -package utils - -import ( - "context" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/models" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" - "github.com/upper/db/v4" - "github.com/upper/db/v4/adapter/mssql" - "time" -) - -func GetCockroachdbSession(ds *models.DataSource) (s db.Session, err error) { - return getCockroachdbSession(context.Background(), ds) -} - -func GetCockroachdbSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getCockroachdbSession(ctx, ds) -} - -func getCockroachdbSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { - // normalize settings - host := ds.Host - port := ds.Port - if ds.Host == "" { - host = constants.DefaultHost - } - if ds.Port == 0 { - port = constants.DefaultCockroachdbPort - } - - // connect settings - settings := mssql.ConnectionURL{ - User: ds.Username, - Password: ds.Password, - Database: ds.Database, - Host: fmt.Sprintf("%s:%s", host, port), - Options: nil, - } - - // session - done := make(chan struct{}) - go func() { - s, err = mssql.Open(settings) - close(done) - }() - - // wait for done - select { - case <-ctx.Done(): - if ctx.Err() != nil { - err = ctx.Err() - } - case <-done: - } - - return s, err -} - -func GetCockroachdbSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getCockroachdbSessionV2(ctx, ds) -} - -func getCockroachdbSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { - // normalize settings - host := ds.Host - port := ds.Port - if ds.Host == "" { - host = constants.DefaultHost - } - if ds.Port == 0 { - port = constants.DefaultCockroachdbPort - } - - // connect settings - settings := mssql.ConnectionURL{ - User: ds.Username, - Password: ds.Password, - Database: ds.Database, - Host: fmt.Sprintf("%s:%s", host, port), - Options: nil, - } - - // session - done := make(chan struct{}) - go func() { - s, err = mssql.Open(settings) - close(done) - }() - - // wait for done - select { - case <-ctx.Done(): - if ctx.Err() != nil { - err = ctx.Err() - } - case <-done: - } - - return s, err -} diff --git a/core/utils/cron.go b/core/utils/cron.go deleted file mode 100644 index e8838d2f..00000000 --- a/core/utils/cron.go +++ /dev/null @@ -1,176 +0,0 @@ -package utils - -import ( - "fmt" - "math" - "strconv" - "strings" -) - -// cronBounds provides a range of acceptable values (plus a map of name to value). -type cronBounds struct { - min, max uint - names map[string]uint -} - -type cronUtils struct { - // The cronBounds for each field. - seconds cronBounds - minutes cronBounds - hours cronBounds - dom cronBounds - months cronBounds - dow cronBounds - - // Set the top bit if a star was included in the expression. - starBit uint64 -} - -// getRange returns the bits indicated by the given expression: -// number | number "-" number [ "/" number ] -// or error parsing range. -func (u *cronUtils) getRange(expr string, r cronBounds) (uint64, error) { - var ( - start, end, step uint - rangeAndStep = strings.Split(expr, "/") - lowAndHigh = strings.Split(rangeAndStep[0], "-") - singleDigit = len(lowAndHigh) == 1 - err error - ) - - var extra uint64 - if lowAndHigh[0] == "*" || lowAndHigh[0] == "?" { - start = r.min - end = r.max - extra = CronUtils.starBit - } else { - start, err = u.parseIntOrName(lowAndHigh[0], r.names) - if err != nil { - return 0, err - } - switch len(lowAndHigh) { - case 1: - end = start - case 2: - end, err = u.parseIntOrName(lowAndHigh[1], r.names) - if err != nil { - return 0, err - } - default: - return 0, fmt.Errorf("too many hyphens: %s", expr) - } - } - - switch len(rangeAndStep) { - case 1: - step = 1 - case 2: - step, err = u.mustParseInt(rangeAndStep[1]) - if err != nil { - return 0, err - } - - // Special handling: "N/step" means "N-max/step". - if singleDigit { - end = r.max - } - if step > 1 { - extra = 0 - } - default: - return 0, fmt.Errorf("too many slashes: %s", expr) - } - - if start < r.min { - return 0, fmt.Errorf("beginning of range (%d) below minimum (%d): %s", start, r.min, expr) - } - if end > r.max { - return 0, fmt.Errorf("end of range (%d) above maximum (%d): %s", end, r.max, expr) - } - if start > end { - return 0, fmt.Errorf("beginning of range (%d) beyond end of range (%d): %s", start, end, expr) - } - if step == 0 { - return 0, fmt.Errorf("step of range should be a positive number: %s", expr) - } - - return u.getBits(start, end, step) | extra, nil -} - -// parseIntOrName returns the (possibly-named) integer contained in expr. -func (u *cronUtils) parseIntOrName(expr string, names map[string]uint) (uint, error) { - if names != nil { - if namedInt, ok := names[strings.ToLower(expr)]; ok { - return namedInt, nil - } - } - return u.mustParseInt(expr) -} - -// mustParseInt parses the given expression as an int or returns an error. -func (u *cronUtils) mustParseInt(expr string) (uint, error) { - num, err := strconv.Atoi(expr) - if err != nil { - return 0, fmt.Errorf("failed to parse int from %s: %s", expr, err) - } - if num < 0 { - return 0, fmt.Errorf("negative number (%d) not allowed: %s", num, expr) - } - - return uint(num), nil -} - -// getBits sets all bits in the range [min, max], modulo the given step size. -func (u *cronUtils) getBits(min, max, step uint) uint64 { - var bits uint64 - - // If step is 1, use shifts. - if step == 1 { - return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min) - } - - // Else, use a simple loop. - for i := min; i <= max; i += step { - bits |= 1 << i - } - return bits -} - -// all returns all bits within the given cronBounds. (plus the star bit) -func (u *cronUtils) all(r cronBounds) uint64 { - return u.getBits(r.min, r.max, 1) | CronUtils.starBit -} - -var CronUtils = cronUtils{ - // The cronBounds for each field. - seconds: cronBounds{0, 59, nil}, - minutes: cronBounds{0, 59, nil}, - hours: cronBounds{0, 23, nil}, - dom: cronBounds{1, 31, nil}, - months: cronBounds{1, 12, map[string]uint{ - "jan": 1, - "feb": 2, - "mar": 3, - "apr": 4, - "may": 5, - "jun": 6, - "jul": 7, - "aug": 8, - "sep": 9, - "oct": 10, - "nov": 11, - "dec": 12, - }}, - dow: cronBounds{0, 6, map[string]uint{ - "sun": 0, - "mon": 1, - "tue": 2, - "wed": 3, - "thu": 4, - "fri": 5, - "sat": 6, - }}, - - // Set the top bit if a star was included in the expression. - starBit: 1 << 63, -} diff --git a/core/utils/demo.go b/core/utils/demo.go deleted file mode 100644 index 70b45044..00000000 --- a/core/utils/demo.go +++ /dev/null @@ -1,66 +0,0 @@ -package utils - -import ( - "fmt" - "github.com/crawlab-team/crawlab/core/sys_exec" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "github.com/spf13/viper" -) - -func GetApiAddress() (res string) { - apiAddress := viper.GetString("api.address") - if apiAddress == "" { - return "http://localhost:8000" - } - return apiAddress -} - -func IsDemo() (ok bool) { - return EnvIsTrue("demo", true) -} - -func InitializedDemo() (ok bool) { - col := mongo.GetMongoCol("users") - n, err := col.Count(nil) - if err != nil { - return true - } - return n > 0 -} - -func ImportDemo() (err error) { - cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo import", GetApiAddress()) - cmd, err := sys_exec.BuildCmd(cmdStr) - if err != nil { - return err - } - if err := cmd.Run(); err != nil { - trace.PrintError(err) - } - return nil -} - -func ReimportDemo() (err error) { - cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo reimport", GetApiAddress()) - cmd, err := sys_exec.BuildCmd(cmdStr) - if err != nil { - return err - } - if err := cmd.Run(); err != nil { - trace.PrintError(err) - } - return nil -} - -func CleanupDemo() (err error) { - cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo reimport", GetApiAddress()) - cmd, err := sys_exec.BuildCmd(cmdStr) - if err != nil { - return err - } - if err := cmd.Run(); err != nil { - trace.PrintError(err) - } - return nil -} diff --git a/core/utils/di.go b/core/utils/di.go deleted file mode 100644 index 8d69a733..00000000 --- a/core/utils/di.go +++ /dev/null @@ -1,18 +0,0 @@ -package utils - -import ( - "github.com/crawlab-team/crawlab/trace" - "github.com/spf13/viper" - "go.uber.org/dig" - "os" -) - -func VisualizeContainer(c *dig.Container) (err error) { - if !viper.GetBool("debug.di.visualize") { - return nil - } - if err := dig.Visualize(c, os.Stdout); err != nil { - return trace.TraceError(err) - } - return nil -} diff --git a/core/utils/es.go b/core/utils/es.go deleted file mode 100644 index ee70f187..00000000 --- a/core/utils/es.go +++ /dev/null @@ -1,201 +0,0 @@ -package utils - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "github.com/cenkalti/backoff/v4" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/models" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" - "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/crawlab/trace" - "github.com/elastic/go-elasticsearch/v8" - "github.com/elastic/go-elasticsearch/v8/esapi" - "go.mongodb.org/mongo-driver/bson/primitive" - "time" -) - -func GetElasticsearchClient(ds *models.DataSource) (c *elasticsearch.Client, err error) { - return getElasticsearchClient(context.Background(), ds) -} - -func GetElasticsearchClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c *elasticsearch.Client, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getElasticsearchClient(ctx, ds) -} - -func getElasticsearchClient(ctx context.Context, ds *models.DataSource) (c *elasticsearch.Client, err error) { - // normalize settings - host := ds.Host - port := ds.Port - if ds.Host == "" { - host = constants.DefaultHost - } - if ds.Port == 0 { - port = constants.DefaultElasticsearchPort - } - - // es hosts - addresses := []string{ - fmt.Sprintf("http://%s:%s", host, port), - } - - // retry backoff - rb := backoff.NewExponentialBackOff() - - // es client options - cfg := elasticsearch.Config{ - Addresses: addresses, - Username: ds.Username, - Password: ds.Password, - RetryBackoff: func(i int) time.Duration { - if i == 1 { - rb.Reset() - } - return rb.NextBackOff() - }, - } - - // es client - done := make(chan struct{}) - go func() { - c, err = elasticsearch.NewClient(cfg) - if err != nil { - return - } - var res *esapi.Response - res, err = c.Info() - fmt.Println(res) - close(done) - }() - - // wait for done - select { - case <-ctx.Done(): - if ctx.Err() != nil { - err = ctx.Err() - } - case <-done: - } - - return c, err -} - -func GetElasticsearchClientWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (c *elasticsearch.Client, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getElasticsearchClientV2(ctx, ds) -} - -func getElasticsearchClientV2(ctx context.Context, ds *models2.DatabaseV2) (c *elasticsearch.Client, err error) { - // normalize settings - host := ds.Host - port := ds.Port - if ds.Host == "" { - host = constants.DefaultHost - } - if ds.Port == 0 { - port = constants.DefaultElasticsearchPort - } - - // es hosts - addresses := []string{ - fmt.Sprintf("http://%s:%s", host, port), - } - - // retry backoff - rb := backoff.NewExponentialBackOff() - - // es client options - cfg := elasticsearch.Config{ - Addresses: addresses, - Username: ds.Username, - Password: ds.Password, - RetryBackoff: func(i int) time.Duration { - if i == 1 { - rb.Reset() - } - return rb.NextBackOff() - }, - } - - // es client - done := make(chan struct{}) - go func() { - c, err = elasticsearch.NewClient(cfg) - if err != nil { - return - } - var res *esapi.Response - res, err = c.Info() - fmt.Println(res) - close(done) - }() - - // wait for done - select { - case <-ctx.Done(): - if ctx.Err() != nil { - err = ctx.Err() - } - case <-done: - } - - return c, err -} - -func GetElasticsearchQuery(query generic.ListQuery) (buf *bytes.Buffer) { - q := map[string]interface{}{} - if len(query) > 0 { - match := getElasticsearchQueryMatch(query) - q["query"] = map[string]interface{}{ - "match": match, - } - } - buf = &bytes.Buffer{} - if err := json.NewEncoder(buf).Encode(q); err != nil { - trace.PrintError(err) - } - return buf -} - -func GetElasticsearchQueryWithOptions(query generic.ListQuery, opts *generic.ListOptions) (buf *bytes.Buffer) { - q := map[string]interface{}{ - "size": opts.Limit, - "from": opts.Skip, - // TODO: sort - } - if len(query) > 0 { - match := getElasticsearchQueryMatch(query) - q["query"] = map[string]interface{}{ - "match": match, - } - } - buf = &bytes.Buffer{} - if err := json.NewEncoder(buf).Encode(q); err != nil { - trace.PrintError(err) - } - return buf -} - -func getElasticsearchQueryMatch(query generic.ListQuery) (match map[string]interface{}) { - match = map[string]interface{}{} - for _, c := range query { - switch c.Value.(type) { - case primitive.ObjectID: - c.Value = c.Value.(primitive.ObjectID).Hex() - } - switch c.Op { - case generic.OpEqual: - match[c.Key] = c.Value - default: - match[c.Key] = map[string]interface{}{ - c.Op: c.Value, - } - } - } - return match -} diff --git a/core/utils/git.go b/core/utils/git.go deleted file mode 100644 index 272d0691..00000000 --- a/core/utils/git.go +++ /dev/null @@ -1,21 +0,0 @@ -package utils - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - vcs "github.com/crawlab-team/crawlab/vcs" -) - -func InitGitClientAuth(g interfaces.Git, gitClient *vcs.GitClient) { - // set auth - switch g.GetAuthType() { - case constants.GitAuthTypeHttp: - gitClient.SetAuthType(vcs.GitAuthTypeHTTP) - gitClient.SetUsername(g.GetUsername()) - gitClient.SetPassword(g.GetPassword()) - case constants.GitAuthTypeSsh: - gitClient.SetAuthType(vcs.GitAuthTypeSSH) - gitClient.SetUsername(g.GetUsername()) - gitClient.SetPrivateKey(g.GetPassword()) - } -} diff --git a/core/utils/helpers.go b/core/utils/helpers.go index 3f63f12b..f3fcd56e 100644 --- a/core/utils/helpers.go +++ b/core/utils/helpers.go @@ -4,13 +4,8 @@ import ( "github.com/crawlab-team/crawlab/trace" "io" "reflect" - "unsafe" ) -func BytesToString(b []byte) string { - return *(*string)(unsafe.Pointer(&b)) -} - func Close(c io.Closer) { err := c.Close() if err != nil { diff --git a/core/utils/kafka.go b/core/utils/kafka.go deleted file mode 100644 index c07f19ea..00000000 --- a/core/utils/kafka.go +++ /dev/null @@ -1,69 +0,0 @@ -package utils - -import ( - "context" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/models" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" - "github.com/segmentio/kafka-go" - "time" -) - -func GetKafkaConnection(ds *models.DataSource) (c *kafka.Conn, err error) { - return getKafkaConnection(context.Background(), ds) -} - -func GetKafkaConnectionWithTimeout(ds *models.DataSource, timeout time.Duration) (c *kafka.Conn, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getKafkaConnection(ctx, ds) -} - -func getKafkaConnection(ctx context.Context, ds *models.DataSource) (c *kafka.Conn, err error) { - // normalize settings - host := ds.Host - port := ds.Port - if ds.Host == "" { - host = constants.DefaultHost - } - if ds.Port == 0 { - port = constants.DefaultKafkaPort - } - - // kafka connection address - network := "tcp" - address := fmt.Sprintf("%s:%s", host, port) - topic := ds.Database - partition := 0 // TODO: parameterize - - // kafka connection - return kafka.DialLeader(ctx, network, address, topic, partition) -} - -func GetKafkaConnectionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (c *kafka.Conn, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getKafkaConnectionV2(ctx, ds) -} - -func getKafkaConnectionV2(ctx context.Context, ds *models2.DatabaseV2) (c *kafka.Conn, err error) { - // normalize settings - host := ds.Host - port := ds.Port - if ds.Host == "" { - host = constants.DefaultHost - } - if ds.Port == 0 { - port = constants.DefaultKafkaPort - } - - // kafka connection address - network := "tcp" - address := fmt.Sprintf("%s:%s", host, port) - topic := ds.Database - partition := 0 // TODO: parameterize - - // kafka connection - return kafka.DialLeader(ctx, network, address, topic, partition) -} diff --git a/core/utils/mongo.go b/core/utils/mongo.go index f576cec5..c223d3ef 100644 --- a/core/utils/mongo.go +++ b/core/utils/mongo.go @@ -1,15 +1,9 @@ package utils import ( - "context" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/models" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/db/generic" "github.com/crawlab-team/crawlab/db/mongo" "go.mongodb.org/mongo-driver/bson" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "time" ) func GetMongoQuery(query generic.ListQuery) (res bson.M) { @@ -44,80 +38,3 @@ func GetMongoOpts(opts *generic.ListOptions) (res *mongo.FindOptions) { Sort: sort, } } - -func GetMongoClient(ds *models.DataSource) (c *mongo2.Client, err error) { - return getMongoClient(context.Background(), ds) -} - -func GetMongoClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c *mongo2.Client, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getMongoClient(ctx, ds) -} - -func GetMongoClientWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (c *mongo2.Client, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getMongoClientV2(ctx, ds) -} - -func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Client, err error) { - // normalize settings - if ds.Host == "" { - ds.Host = constants.DefaultHost - } - if ds.Port == 0 { - ds.Port = constants.DefaultMongoPort - } - - // options - var opts []mongo.ClientOption - opts = append(opts, mongo.WithContext(ctx)) - opts = append(opts, mongo.WithUri(ds.Url)) - opts = append(opts, mongo.WithHost(ds.Host)) - opts = append(opts, mongo.WithPort(ds.Port)) - opts = append(opts, mongo.WithDb(ds.Database)) - opts = append(opts, mongo.WithUsername(ds.Username)) - opts = append(opts, mongo.WithPassword(ds.Password)) - - // extra - if ds.Extra != nil { - // auth source - authSource, ok := ds.Extra["auth_source"] - if ok { - opts = append(opts, mongo.WithAuthSource(authSource)) - } - - // auth mechanism - authMechanism, ok := ds.Extra["auth_mechanism"] - if ok { - opts = append(opts, mongo.WithAuthMechanism(authMechanism)) - } - } - - // client - return mongo.GetMongoClient(opts...) -} - -func getMongoClientV2(ctx context.Context, ds *models2.DatabaseV2) (c *mongo2.Client, err error) { - // normalize settings - if ds.Host == "" { - ds.Host = constants.DefaultHost - } - if ds.Port == 0 { - ds.Port = constants.DefaultMongoPort - } - - // options - var opts []mongo.ClientOption - opts = append(opts, mongo.WithContext(ctx)) - opts = append(opts, mongo.WithUri(ds.URI)) - opts = append(opts, mongo.WithHost(ds.Host)) - opts = append(opts, mongo.WithPort(ds.Port)) - opts = append(opts, mongo.WithDb(ds.Database)) - opts = append(opts, mongo.WithUsername(ds.Username)) - opts = append(opts, mongo.WithPassword(ds.Password)) - - // client - return mongo.GetMongoClient(opts...) -} diff --git a/core/utils/mssql.go b/core/utils/mssql.go deleted file mode 100644 index 1190a28e..00000000 --- a/core/utils/mssql.go +++ /dev/null @@ -1,106 +0,0 @@ -package utils - -import ( - "context" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/models" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" - "github.com/upper/db/v4" - "github.com/upper/db/v4/adapter/mssql" - "time" -) - -func GetMssqlSession(ds *models.DataSource) (s db.Session, err error) { - return getMssqlSession(context.Background(), ds) -} - -func GetMssqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getMssqlSession(ctx, ds) -} - -func getMssqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { - // normalize settings - host := ds.Host - port := ds.Port - if ds.Host == "" { - host = constants.DefaultHost - } - if ds.Port == 0 { - port = constants.DefaultMssqlPort - } - - // connect settings - settings := mssql.ConnectionURL{ - User: ds.Username, - Password: ds.Password, - Database: ds.Database, - Host: fmt.Sprintf("%s:%s", host, port), - Options: nil, - } - - // session - done := make(chan struct{}) - go func() { - s, err = mssql.Open(settings) - close(done) - }() - - // wait for done - select { - case <-ctx.Done(): - if ctx.Err() != nil { - err = ctx.Err() - } - case <-done: - } - - return s, err -} - -func GetMssqlSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getMssqlSessionV2(ctx, ds) -} - -func getMssqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { - // normalize settings - host := ds.Host - port := ds.Port - if ds.Host == "" { - host = constants.DefaultHost - } - if ds.Port == 0 { - port = constants.DefaultMssqlPort - } - - // connect settings - settings := mssql.ConnectionURL{ - User: ds.Username, - Password: ds.Password, - Database: ds.Database, - Host: fmt.Sprintf("%s:%s", host, port), - Options: nil, - } - - // session - done := make(chan struct{}) - go func() { - s, err = mssql.Open(settings) - close(done) - }() - - // wait for done - select { - case <-ctx.Done(): - if ctx.Err() != nil { - err = ctx.Err() - } - case <-done: - } - - return s, err -} diff --git a/core/utils/mysql.go b/core/utils/mysql.go deleted file mode 100644 index 34c8bcfb..00000000 --- a/core/utils/mysql.go +++ /dev/null @@ -1,106 +0,0 @@ -package utils - -import ( - "context" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/models" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" - "github.com/upper/db/v4" - "github.com/upper/db/v4/adapter/mysql" - "time" -) - -func GetMysqlSession(ds *models.DataSource) (s db.Session, err error) { - return getMysqlSession(context.Background(), ds) -} - -func GetMysqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getMysqlSession(ctx, ds) -} - -func getMysqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { - // normalize settings - host := ds.Host - port := ds.Port - if ds.Host == "" { - host = constants.DefaultHost - } - if ds.Port == 0 { - port = constants.DefaultMysqlPort - } - - // connect settings - settings := mysql.ConnectionURL{ - User: ds.Username, - Password: ds.Password, - Database: ds.Database, - Host: fmt.Sprintf("%s:%s", host, port), - Options: nil, - } - - // session - done := make(chan struct{}) - go func() { - s, err = mysql.Open(settings) - close(done) - }() - - // wait for done - select { - case <-ctx.Done(): - if ctx.Err() != nil { - err = ctx.Err() - } - case <-done: - } - - return s, err -} - -func GetMysqlSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getMysqlSessionV2(ctx, ds) -} - -func getMysqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { - // normalize settings - host := ds.Host - port := ds.Port - if ds.Host == "" { - host = constants.DefaultHost - } - if ds.Port == 0 { - port = constants.DefaultMysqlPort - } - - // connect settings - settings := mysql.ConnectionURL{ - User: ds.Username, - Password: ds.Password, - Database: ds.Database, - Host: fmt.Sprintf("%s:%s", host, port), - Options: nil, - } - - // session - done := make(chan struct{}) - go func() { - s, err = mysql.Open(settings) - close(done) - }() - - // wait for done - select { - case <-ctx.Done(): - if ctx.Err() != nil { - err = ctx.Err() - } - case <-done: - } - - return s, err -} diff --git a/core/utils/postgresql.go b/core/utils/postgresql.go deleted file mode 100644 index 0971c10e..00000000 --- a/core/utils/postgresql.go +++ /dev/null @@ -1,106 +0,0 @@ -package utils - -import ( - "context" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/models" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" - "github.com/upper/db/v4" - "github.com/upper/db/v4/adapter/postgresql" - "time" -) - -func GetPostgresqlSession(ds *models.DataSource) (s db.Session, err error) { - return getPostgresqlSession(context.Background(), ds) -} - -func GetPostgresqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getPostgresqlSession(ctx, ds) -} - -func getPostgresqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { - // normalize settings - host := ds.Host - port := ds.Port - if ds.Host == "" { - host = constants.DefaultHost - } - if ds.Port == 0 { - port = constants.DefaultPostgresqlPort - } - - // connect settings - settings := postgresql.ConnectionURL{ - User: ds.Username, - Password: ds.Password, - Database: ds.Database, - Host: fmt.Sprintf("%s:%s", host, port), - Options: nil, - } - - // session - done := make(chan struct{}) - go func() { - s, err = postgresql.Open(settings) - close(done) - }() - - // wait for done - select { - case <-ctx.Done(): - if ctx.Err() != nil { - err = ctx.Err() - } - case <-done: - } - - return s, err -} - -func GetPostgresqlSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getPostgresqlSessionV2(ctx, ds) -} - -func getPostgresqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { - // normalize settings - host := ds.Host - port := ds.Port - if ds.Host == "" { - host = constants.DefaultHost - } - if ds.Port == 0 { - port = constants.DefaultPostgresqlPort - } - - // connect settings - settings := postgresql.ConnectionURL{ - User: ds.Username, - Password: ds.Password, - Database: ds.Database, - Host: fmt.Sprintf("%s:%s", host, port), - Options: nil, - } - - // session - done := make(chan struct{}) - go func() { - s, err = postgresql.Open(settings) - close(done) - }() - - // wait for done - select { - case <-ctx.Done(): - if ctx.Err() != nil { - err = ctx.Err() - } - case <-done: - } - - return s, err -} diff --git a/core/utils/process.go b/core/utils/process.go new file mode 100644 index 00000000..3ebb3a42 --- /dev/null +++ b/core/utils/process.go @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2024. Core Digital Limited + * 版权所有 (c) 2024. 重庆科锐数研科技有限公司 (Core Digital Limited) + * All rights reserved. 保留所有权利。 + * + * 该软件由 重庆科锐数研科技有限公司 (Core Digital Limited) 开发，未经明确书面许可，任何人不得使用、复制、修改或分发该软件的任何部分。 + * This software is developed by Core Digital Limited. No one is permitted to use, copy, modify, or distribute this software without explicit written permission. + * + * 许可证： + * 该软件仅供授权使用。授权用户有权在授权范围内使用、复制、修改和分发该软件。 + * License: + * This software is for authorized use only. Authorized users are permitted to use, copy, modify, and distribute this software within the scope of their authorization. + * + * 免责声明： + * 该软件按“原样”提供，不附带任何明示或暗示的担保，包括但不限于对适销性和适用于特定目的的担保。在任何情况下，版权持有者或其许可方对因使用该软件而产生的任何损害或其他责任概不负责。 + * Disclaimer: + * This software is provided "as is," without any express or implied warranties, including but not limited to warranties of merchantability and fitness for a particular purpose. In no event shall the copyright holder or its licensors be liable for any damages or other liability arising from the use of this software. + * + */ + +package utils + +import ( + "github.com/crawlab-team/crawlab/trace" + "os/exec" + "regexp" + "runtime" + "strings" +) + +var pidRegexp, _ = regexp.Compile("(?:^|\\s+)\\d+(?:$|\\s+)") + +func ProcessIdExists(id int) (ok bool) { + lines, err := ListProcess(string(rune(id))) + if err != nil { + return false + } + for _, line := range lines { + matched := pidRegexp.MatchString(line) + if matched { + return true + } + } + return false +} + +func ListProcess(text string) (lines []string, err error) { + if runtime.GOOS == "windows" { + return listProcessWindow(text) + } else { + return listProcessLinuxMac(text) + } +} + +func listProcessWindow(text string) (lines []string, err error) { + cmd := exec.Command("tasklist", "/fi", text) + out, err := cmd.CombinedOutput() + _, ok := err.(*exec.ExitError) + if !ok { + return nil, trace.TraceError(err) + } + lines = strings.Split(string(out), "\n") + return lines, nil +} + +func listProcessLinuxMac(text string) (lines []string, err error) { + cmd := exec.Command("ps", "aux") + out, err := cmd.CombinedOutput() + _, ok := err.(*exec.ExitError) + if !ok { + return nil, trace.TraceError(err) + } + _lines := strings.Split(string(out), "\n") + for _, l := range _lines { + if strings.Contains(l, text) { + lines = append(lines, l) + } + } + return lines, nil +} diff --git a/core/utils/rpc.go b/core/utils/rpc.go deleted file mode 100644 index 03414199..00000000 --- a/core/utils/rpc.go +++ /dev/null @@ -1,14 +0,0 @@ -package utils - -import "encoding/json" - -// Object 转化为 String -func ObjectToString(params interface{}) string { - bytes, _ := json.Marshal(params) - return BytesToString(bytes) -} - -// 获取 RPC 参数 -func GetRpcParam(key string, params map[string]string) string { - return params[key] -} diff --git a/core/utils/sql.go b/core/utils/sql.go deleted file mode 100644 index 4eccce86..00000000 --- a/core/utils/sql.go +++ /dev/null @@ -1,27 +0,0 @@ -package utils - -import ( - "github.com/crawlab-team/crawlab/db/generic" - "github.com/upper/db/v4" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func GetSqlQuery(query generic.ListQuery) (res db.Cond) { - res = db.Cond{} - for _, c := range query { - switch c.Value.(type) { - case primitive.ObjectID: - c.Value = c.Value.(primitive.ObjectID).Hex() - } - switch c.Op { - case generic.OpEqual: - res[c.Key] = c.Value - default: - res[c.Key] = db.Cond{ - c.Op: c.Value, - } - } - } - // TODO: sort - return res -} diff --git a/core/utils/sqlite.go b/core/utils/sqlite.go deleted file mode 100644 index 8e39db2a..00000000 --- a/core/utils/sqlite.go +++ /dev/null @@ -1,78 +0,0 @@ -package utils - -import ( - "context" - "github.com/crawlab-team/crawlab/core/models/models" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" - "github.com/upper/db/v4" - "github.com/upper/db/v4/adapter/sqlite" - "time" -) - -func GetSqliteSession(ds *models.DataSource) (s db.Session, err error) { - return getSqliteSession(context.Background(), ds) -} - -func GetSqliteSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getSqliteSession(ctx, ds) -} - -func getSqliteSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { - // connect settings - settings := sqlite.ConnectionURL{ - Database: ds.Database, - Options: nil, - } - - // session - done := make(chan struct{}) - go func() { - s, err = sqlite.Open(settings) - close(done) - }() - - // wait for done - select { - case <-ctx.Done(): - if ctx.Err() != nil { - err = ctx.Err() - } - case <-done: - } - - return s, err -} - -func GetSqliteSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return getSqliteSessionV2(ctx, ds) -} - -func getSqliteSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { - // connect settings - settings := sqlite.ConnectionURL{ - Database: ds.Database, - Options: nil, - } - - // session - done := make(chan struct{}) - go func() { - s, err = sqlite.Open(settings) - close(done) - }() - - // wait for done - select { - case <-ctx.Done(): - if ctx.Err() != nil { - err = ctx.Err() - } - case <-done: - } - - return s, err -}