git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: code cleanup

Browse Source
This commit is contained in:
Marvin Zhang
2024-10-18 15:03:32 +08:00
parent 354327ffb1
commit 1b852fb96a
171 changed files with 131 additions and 12522 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
core/apps/docker.go
View File

@@ -167,16 +167,6 @@ func (app *Docker) startSeaweedFs() {
}
}
func (app *Docker) importDemo() {
for {
if app.Ready() {
break
}
time.Sleep(1 * time.Second)
}
_ = utils.ImportDemo()
}
func NewDocker(svr ServerApp) *Docker {
dck := &Docker{
parent: svr,

17
core/cmd/server_test.go
View File

@@ -1,17 +0,0 @@
package cmd
import (
"github.com/crawlab-team/crawlab/core/apps"
"os"
"testing"
)
func TestCmdServer(t *testing.T) {
_ = os.Setenv("CRAWLAB_PPROF", "true")
// app
svr := apps.GetServerV2()
// start
apps.Start(svr)
}

82
core/color/service.go
View File

@@ -1,82 +0,0 @@
package color
import (
"encoding/hex"
"encoding/json"
"github.com/crawlab-team/crawlab/core/data"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/trace"
"math/rand"
"strconv"
"strings"
)
func NewService() (svc interfaces.ColorService, err error) {
var cl []*entity.Color
cm := map[string]*entity.Color{}
if err := json.Unmarshal([]byte(data.ColorsDataText), &cl); err != nil {
return nil, trace.TraceError(err)
}
for _, c := range cl {
cm[c.Name] = c
}
return &Service{
cl: cl,
cm: cm,
}, nil
}
type Service struct {
cl []*entity.Color
cm map[string]*entity.Color
}
func (svc *Service) Inject() (err error) {
return nil
}
func (svc *Service) GetByName(name string) (res interfaces.Color, err error) {
res, ok := svc.cm[name]
if !ok {
return res, errors.ErrorModelNotFound
}
return res, err
}
func (svc *Service) GetRandom() (res interfaces.Color, err error) {
if len(svc.cl) == 0 {
hexStr, err := svc.getRandomColorHex()
if err != nil {
return res, err
}
return &entity.Color{Hex: hexStr}, nil
}
idx := rand.Intn(len(svc.cl))
return svc.cl[idx], nil
}
func (svc *Service) getRandomColorHex() (res string, err error) {
n := 6
arr := make([]string, n)
for i := 0; i < n; i++ {
arr[i], err = svc.getRandomHexChar()
if err != nil {
return res, err
}
}
return strings.Join(arr, ""), nil
}
func (svc *Service) getRandomHexChar() (res string, err error) {
n := rand.Intn(16)
b := []byte(strconv.Itoa(n))
h := make([]byte, 1)
hex.Encode(h, b)
return string(h), nil
}

5
core/controllers/router_v2.go
View File

@@ -220,11 +220,6 @@ func InitRoutes(app *gin.Engine) (err error) {
Path: "/:id/logs",
HandlerFunc: GetTaskLogs,
},
{
Method: http.MethodGet,
Path: "/:id/data",
HandlerFunc: GetTaskData,
},
}...))
RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models2.TokenV2]([]Action{
{

61
core/controllers/task_v2.go
View File

@@ -7,12 +7,10 @@ import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/result"
"github.com/crawlab-team/crawlab/core/spider/admin"
"github.com/crawlab-team/crawlab/core/task/log"
"github.com/crawlab-team/crawlab/core/task/scheduler"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/gin-gonic/gin"
"github.com/spf13/viper"
@@ -441,62 +439,3 @@ func GetTaskLogs(c *gin.Context) {
HandleSuccessWithListData(c, logs, total)
}
func GetTaskData(c *gin.Context) {
// id
id, err := primitive.ObjectIDFromHex(c.Param("id"))
if err != nil {
HandleErrorBadRequest(c, err)
return
}
// pagination
p, err := GetPagination(c)
if err != nil {
HandleErrorBadRequest(c, err)
return
}
// task
t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// result service
resultSvc, err := result.GetResultService(t.SpiderId)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// query
query := generic.ListQuery{
generic.ListQueryCondition{
Key: constants.TaskKey,
Op: generic.OpEqual,
Value: t.Id,
},
}
// list
data, err := resultSvc.List(query, &generic.ListOptions{
Skip: (p.Page - 1) * p.Size,
Limit: p.Size,
Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}},
})
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// total
total, err := resultSvc.Count(query)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
HandleSuccessWithListData(c, data, total)
}

70
core/ds/cockroachdb.go
View File

@@ -1,70 +0,0 @@
package ds
import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/trace"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type CockroachdbService struct {
SqlService
}
func NewDataSourceCockroachdbService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
// service
svc := &CockroachdbService{}
// dependency injection
svc.modelSvc, err = service.GetService()
if err != nil {
return nil, trace.TraceError(err)
}
// data source
if dsId.IsZero() {
svc.ds = &models.DataSource{}
} else {
svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
if err != nil {
return nil, trace.TraceError(err)
}
}
// data source defaults
if svc.ds.Host == "" {
svc.ds.Host = constants.DefaultHost
}
if svc.ds.Port == 0 {
svc.ds.Port = constants.DefaultCockroachdbPort
}
// data source password
pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id)
if err == nil {
svc.ds.Password, err = utils.DecryptAES(pwd.Password)
if err != nil {
return nil, err
}
}
// data collection
svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
if err != nil {
return nil, trace.TraceError(err)
}
// session
svc.s, err = utils.GetCockroachdbSession(svc.ds)
if err != nil {
return nil, trace.TraceError(err)
}
// collection
svc.col = svc.s.Collection(svc.dc.Name)
return svc, nil
}

1
core/ds/default.go
View File

@@ -1 +0,0 @@
package ds

220
core/ds/es.go
View File

@@ -1,220 +0,0 @@
package ds
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/crawlab-team/crawlab/core/constants"
constants2 "github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
entity2 "github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/trace"
"github.com/elastic/go-elasticsearch/v8"
"github.com/elastic/go-elasticsearch/v8/esapi"
"github.com/google/uuid"
"go.mongodb.org/mongo-driver/bson/primitive"
"strings"
"sync"
"time"
)
type ElasticsearchService struct {
// dependencies
modelSvc service.ModelService
// internals
dc *models.DataCollection // models.DataCollection
ds *models.DataSource // models.DataSource
c *elasticsearch.Client // elasticsearch.Client
t time.Time
}
func (svc *ElasticsearchService) Insert(records ...interface{}) (err error) {
// wait group
var wg sync.WaitGroup
wg.Add(len(records))
// iterate records
for _, r := range records {
// async operation
go func(r interface{}) {
switch r.(type) {
case entity.Result:
// convert type to entity.Result
d := r.(entity.Result)
// get document id
id := d.GetValue("id")
var docId string
switch id.(type) {
case string:
docId = id.(string)
}
if docId == "" {
docId = uuid.New().String() // generate new uuid if id is empty
}
// collection
d[constants2.DataCollectionKey] = svc.dc.Name
// index request
req := esapi.IndexRequest{
Index: svc.getIndexName(),
DocumentID: docId,
Body: strings.NewReader(d.String()),
}
// perform request
res, err := req.Do(context.Background(), svc.c)
if err != nil {
trace.PrintError(err)
wg.Done()
return
}
defer res.Body.Close()
if res.IsError() {
trace.PrintError(errors.New(fmt.Sprintf("[ElasticsearchService] [%s] error inserting record: %v", res.Status(), r)))
}
// release
wg.Done()
default:
wg.Done()
return
}
}(r)
}
// wait
wg.Wait()
return nil
}
func (svc *ElasticsearchService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) {
data, err := svc.getListResponse(query, opts, false)
if err != nil {
return nil, err
}
for _, hit := range data.Hits.Hits {
results = append(results, hit.Source)
}
return results, nil
}
func (svc *ElasticsearchService) Count(query generic.ListQuery) (n int, err error) {
data, err := svc.getListResponse(query, nil, true)
if err != nil {
return n, err
}
return int(data.Hits.Total.Value), nil
}
func (svc *ElasticsearchService) getListResponse(query generic.ListQuery, opts *generic.ListOptions, trackTotalHits bool) (data *entity2.ElasticsearchResponseData, err error) {
if opts == nil {
opts = &generic.ListOptions{}
}
query = append(query, generic.ListQueryCondition{
Key: constants2.DataCollectionKey,
Op: constants2.FilterOpEqual,
Value: svc.dc.Name,
})
res, err := svc.c.Search(
svc.c.Search.WithContext(context.Background()),
svc.c.Search.WithIndex(svc.getIndexName()),
svc.c.Search.WithBody(utils.GetElasticsearchQueryWithOptions(query, opts)),
svc.c.Search.WithTrackTotalHits(trackTotalHits),
)
if err != nil {
return nil, trace.TraceError(err)
}
defer res.Body.Close()
if res.IsError() {
err = errors.New(fmt.Sprintf("[ElasticsearchService] [%s] error listing records: response=%s, query=%v opts=%v", res.Status(), res.String(), query, opts))
trace.PrintError(err)
return nil, err
}
data = &entity2.ElasticsearchResponseData{}
if err := json.NewDecoder(res.Body).Decode(data); err != nil {
return nil, trace.TraceError(err)
}
return data, nil
}
func (svc *ElasticsearchService) getIndexName() (index string) {
if svc.ds.Database == "" {
return svc.dc.Name
} else {
return svc.ds.Name
}
}
func NewDataSourceElasticsearchService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
// service
svc := &ElasticsearchService{}
// dependency injection
svc.modelSvc, err = service.GetService()
if err != nil {
return nil, err
}
// data source
if dsId.IsZero() {
svc.ds = &models.DataSource{}
} else {
svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
if err != nil {
return nil, err
}
}
// data source defaults
if svc.ds.Host == "" {
svc.ds.Host = constants.DefaultHost
}
if svc.ds.Port == 0 {
svc.ds.Port = constants.DefaultElasticsearchPort
}
// data source password
pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id)
if err == nil {
svc.ds.Password, err = utils.DecryptAES(pwd.Password)
if err != nil {
return nil, err
}
}
// data collection
svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
if err != nil {
return nil, err
}
// client
svc.c, err = utils.GetElasticsearchClient(svc.ds)
if err != nil {
return nil, err
}
return svc, nil
}
func (svc *ElasticsearchService) Index(fields []string) {
// TODO: implement me
}
func (svc *ElasticsearchService) SetTime(t time.Time) {
svc.t = t
}
func (svc *ElasticsearchService) GetTime() (t time.Time) {
return svc.t
}

116
core/ds/kafka.go
View File

@@ -1,116 +0,0 @@
package ds
import (
"github.com/cenkalti/backoff/v4"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/trace"
"github.com/segmentio/kafka-go"
"go.mongodb.org/mongo-driver/bson/primitive"
"time"
)
type KafkaService struct {
// dependencies
modelSvc service.ModelService
// internals
dc *models.DataCollection // models.DataCollection
ds *models.DataSource // models.DataSource
c *kafka.Conn // kafka.Conn
rb backoff.BackOff
t time.Time
}
func (svc *KafkaService) Insert(records ...interface{}) (err error) {
var messages []kafka.Message
for _, r := range records {
switch r.(type) {
case entity.Result:
d := r.(entity.Result)
messages = append(messages, kafka.Message{
Topic: svc.ds.Database,
Key: []byte(d.GetTaskId().Hex()),
Value: d.Bytes(),
})
}
}
_, err = svc.c.WriteMessages(messages...)
if err != nil {
return trace.TraceError(err)
}
return nil
}
func (svc *KafkaService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) {
// N/A
return nil, nil
}
func (svc *KafkaService) Count(query generic.ListQuery) (n int, err error) {
// N/A
return 0, nil
}
func NewDataSourceKafkaService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
// service
svc := &KafkaService{}
// dependency injection
svc.modelSvc, err = service.GetService()
if err != nil {
return nil, err
}
// data source
if dsId.IsZero() {
svc.ds = &models.DataSource{}
} else {
svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
if err != nil {
return nil, err
}
}
// data source defaults
if svc.ds.Host == "" {
svc.ds.Host = constants.DefaultHost
}
if svc.ds.Port == 0 {
svc.ds.Port = constants.DefaultKafkaPort
}
// data source password
pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id)
if err == nil {
svc.ds.Password, err = utils.DecryptAES(pwd.Password)
if err != nil {
return nil, err
}
}
// data collection
svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
if err != nil {
return nil, err
}
return svc, nil
}
func (svc *KafkaService) Index(fields []string) {
// TODO: implement me
}
func (svc *KafkaService) SetTime(t time.Time) {
svc.t = t
}
func (svc *KafkaService) GetTime() (t time.Time) {
return svc.t
}

118
core/ds/mongo.go
View File

@@ -1,118 +0,0 @@
package ds
import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
utils2 "github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson/primitive"
mongo2 "go.mongodb.org/mongo-driver/mongo"
"time"
)
type MongoService struct {
// dependencies
modelSvc service.ModelService
// internals
dc *models.DataCollection // models.DataCollection
ds *models.DataSource // models.DataSource
c *mongo2.Client
db *mongo2.Database
col *mongo.Col
t time.Time
}
func (svc *MongoService) Insert(records ...interface{}) (err error) {
_, err = svc.col.InsertMany(records)
return err
}
func (svc *MongoService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) {
var docs []models.Result
if err := svc.col.Find(utils.GetMongoQuery(query), utils.GetMongoOpts(opts)).All(&docs); err != nil {
return nil, err
}
for i := range docs {
results = append(results, &docs[i])
}
return results, nil
}
func (svc *MongoService) Count(query generic.ListQuery) (n int, err error) {
return svc.col.Count(utils.GetMongoQuery(query))
}
func NewDataSourceMongoService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
// service
svc := &MongoService{}
// dependency injection
svc.modelSvc, err = service.GetService()
if err != nil {
return nil, err
}
// data source
if dsId.IsZero() {
svc.ds = &models.DataSource{}
} else {
svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
if err != nil {
return nil, err
}
}
// data source defaults
if svc.ds.Host == "" {
svc.ds.Host = constants.DefaultHost
}
if svc.ds.Port == 0 {
svc.ds.Port = constants.DefaultMongoPort
}
// data source password
pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id)
if err == nil {
svc.ds.Password, err = utils.DecryptAES(pwd.Password)
if err != nil {
return nil, err
}
}
// data collection
svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
if err != nil {
return nil, err
}
// mongo client
svc.c, err = utils2.GetMongoClient(svc.ds)
if err != nil {
return nil, err
}
// mongo database
svc.db = mongo.GetMongoDb(svc.ds.Database, mongo.WithDbClient(svc.c))
// mongo col
svc.col = mongo.GetMongoColWithDb(svc.dc.Name, svc.db)
return svc, nil
}
func (svc *MongoService) Index(fields []string) {
// TODO: implement me
}
func (svc *MongoService) SetTime(t time.Time) {
svc.t = t
}
func (svc *MongoService) GetTime() (t time.Time) {
return svc.t
}

71
core/ds/mssql.go
View File

@@ -1,71 +0,0 @@
package ds
import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
utils2 "github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/trace"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type MssqlService struct {
SqlService
}
func NewDataSourceMssqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
// service
svc := &MssqlService{}
// dependency injection
svc.modelSvc, err = service.GetService()
if err != nil {
return nil, trace.TraceError(err)
}
// data source
if dsId.IsZero() {
svc.ds = &models.DataSource{}
} else {
svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
if err != nil {
return nil, trace.TraceError(err)
}
}
// data source defaults
if svc.ds.Host == "" {
svc.ds.Host = constants.DefaultHost
}
if svc.ds.Port == 0 {
svc.ds.Port = constants.DefaultMssqlPort
}
// data source password
pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id)
if err == nil {
svc.ds.Password, err = utils.DecryptAES(pwd.Password)
if err != nil {
return nil, err
}
}
// data collection
svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
if err != nil {
return nil, trace.TraceError(err)
}
// session
svc.s, err = utils2.GetMssqlSession(svc.ds)
if err != nil {
return nil, trace.TraceError(err)
}
// collection
svc.col = svc.s.Collection(svc.dc.Name)
return svc, nil
}

8
core/ds/mssql_test.go
View File

@@ -1,8 +0,0 @@
package ds
import "testing"
func TestNewDataSourceMssqlService(t *testing.T) {
t.Run("insert", func(t *testing.T) {
})
}

69
core/ds/mysql.go
View File

@@ -1,69 +0,0 @@
package ds
import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
utils2 "github.com/crawlab-team/crawlab/core/utils"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type MysqlService struct {
SqlService
}
func NewDataSourceMysqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
// service
svc := &MysqlService{}
// dependency injection
svc.modelSvc, err = service.GetService()
if err != nil {
return nil, err
}
// data source
if dsId.IsZero() {
svc.ds = &models.DataSource{}
} else {
svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
if err != nil {
return nil, err
}
}
// data source defaults
if svc.ds.Host == "" {
svc.ds.Host = constants.DefaultHost
}
if svc.ds.Port == 0 {
svc.ds.Port = constants.DefaultMysqlPort
}
// data source password
pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id)
if err == nil {
svc.ds.Password, err = utils.DecryptAES(pwd.Password)
if err != nil {
return nil, err
}
}
// data collection
svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
if err != nil {
return nil, err
}
// session
svc.s, err = utils2.GetMysqlSession(svc.ds)
if err != nil {
return nil, err
}
// collection
svc.col = svc.s.Collection(svc.dc.Name)
return svc, nil
}

14
core/ds/options.go
View File

@@ -1,14 +0,0 @@
package ds
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"time"
)
type DataSourceServiceOption func(svc interfaces.DataSourceService)
func WithMonitorInterval(duration time.Duration) DataSourceServiceOption {
return func(svc interfaces.DataSourceService) {
svc.SetMonitorInterval(duration)
}
}

71
core/ds/postgresql.go
View File

@@ -1,71 +0,0 @@
package ds
import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
utils2 "github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/trace"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type PostgresqlService struct {
SqlService
}
func NewDataSourcePostgresqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
// service
svc := &PostgresqlService{}
// dependency injection
svc.modelSvc, err = service.GetService()
if err != nil {
return nil, trace.TraceError(err)
}
// data source
if dsId.IsZero() {
svc.ds = &models.DataSource{}
} else {
svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
if err != nil {
return nil, trace.TraceError(err)
}
}
// data source defaults
if svc.ds.Host == "" {
svc.ds.Host = constants.DefaultHost
}
if svc.ds.Port == 0 {
svc.ds.Port = constants.DefaultPostgresqlPort
}
// data source password
pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id)
if err == nil {
svc.ds.Password, err = utils.DecryptAES(pwd.Password)
if err != nil {
return nil, err
}
}
// data collection
svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
if err != nil {
return nil, trace.TraceError(err)
}
// session
svc.s, err = utils2.GetPostgresqlSession(svc.ds)
if err != nil {
return nil, trace.TraceError(err)
}
// collection
svc.col = svc.s.Collection(svc.dc.Name)
return svc, nil
}

305
core/ds/service.go
View File

@@ -1,305 +0,0 @@
package ds
import (
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/constants"
constants2 "github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/delegate"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/result"
"github.com/crawlab-team/crawlab/core/utils"
utils2 "github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/trace"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"sync"
"time"
)
type Service struct {
// dependencies
modelSvc service.ModelService
// internals
timeout time.Duration
monitorInterval time.Duration
stopped bool
}
func (svc *Service) Init() {
// result service registry
reg := result.GetResultServiceRegistry()
// register result services
reg.Register(constants.DataSourceTypeMongo, NewDataSourceMongoService)
reg.Register(constants.DataSourceTypeMysql, NewDataSourceMysqlService)
reg.Register(constants.DataSourceTypePostgresql, NewDataSourcePostgresqlService)
reg.Register(constants.DataSourceTypeMssql, NewDataSourceMssqlService)
reg.Register(constants.DataSourceTypeSqlite, NewDataSourceSqliteService)
reg.Register(constants.DataSourceTypeCockroachdb, NewDataSourceCockroachdbService)
reg.Register(constants.DataSourceTypeElasticSearch, NewDataSourceElasticsearchService)
reg.Register(constants.DataSourceTypeKafka, NewDataSourceKafkaService)
}
func (svc *Service) Start() {
// start monitoring
go svc.Monitor()
}
func (svc *Service) Wait() {
utils.DefaultWait()
}
func (svc *Service) Stop() {
svc.stopped = true
}
func (svc *Service) ChangePassword(id primitive.ObjectID, password string) (err error) {
p, err := svc.modelSvc.GetPasswordById(id)
if err == nil {
// exists, save
encryptedPassword, err := utils.EncryptAES(password)
if err != nil {
return err
}
p.Password = encryptedPassword
if err := delegate.NewModelDelegate(p).Save(); err != nil {
return err
}
return nil
} else if err.Error() == mongo.ErrNoDocuments.Error() {
// not exists, add
encryptedPassword, err := utils.EncryptAES(password)
if err != nil {
return err
}
p = &models.Password{
Id: id,
Password: encryptedPassword,
}
if err := delegate.NewModelDelegate(p).Add(); err != nil {
return err
}
return nil
} else {
// error
return err
}
}
func (svc *Service) Monitor() {
for {
// return if stopped
if svc.stopped {
return
}
// monitor
if err := svc.monitor(); err != nil {
trace.PrintError(err)
}
// wait
time.Sleep(svc.monitorInterval)
}
}
func (svc *Service) CheckStatus(id primitive.ObjectID) (err error) {
ds, err := svc.modelSvc.GetDataSourceById(id)
if err != nil {
return err
}
return svc.checkStatus(ds, true)
}
func (svc *Service) SetTimeout(duration time.Duration) {
svc.timeout = duration
}
func (svc *Service) SetMonitorInterval(duration time.Duration) {
svc.monitorInterval = duration
}
func (svc *Service) monitor() (err error) {
// start
tic := time.Now()
log.Debugf("[DataSourceService] start monitoring")
// data source list
dsList, err := svc.modelSvc.GetDataSourceList(nil, nil)
if err != nil {
return err
}
// waiting group
wg := sync.WaitGroup{}
wg.Add(len(dsList))
// iterate data source list
for _, ds := range dsList {
// async operation
go func(ds models.DataSource) {
// check status and save
_ = svc.checkStatus(&ds, true)
// release
wg.Done()
}(ds)
}
// wait
wg.Wait()
// finish
toc := time.Now()
log.Debugf("[DataSourceService] finished monitoring. elapsed: %d ms", (toc.Sub(tic)).Milliseconds())
return nil
}
func (svc *Service) checkStatus(ds *models.DataSource, save bool) (err error) {
// password
if ds.Password == "" {
pwd, err := svc.modelSvc.GetPasswordById(ds.Id)
if err == nil {
ds.Password, err = utils.DecryptAES(pwd.Password)
if err != nil {
return err
}
} else if err.Error() != mongo.ErrNoDocuments.Error() {
return trace.TraceError(err)
}
}
// check status
if err := svc._checkStatus(ds); err != nil {
ds.Status = constants2.DataSourceStatusOffline
ds.Error = err.Error()
} else {
ds.Status = constants2.DataSourceStatusOnline
ds.Error = ""
}
// save
if save {
return svc._save(ds)
}
return nil
}
func (svc *Service) _save(ds *models.DataSource) (err error) {
log.Debugf("[DataSourceService] saving data source: name=%s, type=%s, status=%s, error=%s", ds.Name, ds.Type, ds.Status, ds.Error)
return delegate.NewModelDelegate(ds).Save()
}
func (svc *Service) _checkStatus(ds *models.DataSource) (err error) {
switch ds.Type {
case constants.DataSourceTypeMongo:
_, err := utils2.GetMongoClientWithTimeout(ds, svc.timeout)
if err != nil {
return err
}
case constants.DataSourceTypeMysql:
s, err := utils2.GetMysqlSessionWithTimeout(ds, svc.timeout)
if err != nil {
return err
}
if s != nil {
s.Close()
}
case constants.DataSourceTypePostgresql:
s, err := utils2.GetPostgresqlSessionWithTimeout(ds, svc.timeout)
if err != nil {
return err
}
if s != nil {
s.Close()
}
case constants.DataSourceTypeMssql:
s, err := utils2.GetMssqlSessionWithTimeout(ds, svc.timeout)
if err != nil {
return err
}
if s != nil {
s.Close()
}
case constants.DataSourceTypeSqlite:
s, err := utils2.GetSqliteSessionWithTimeout(ds, svc.timeout)
if err != nil {
return err
}
if s != nil {
s.Close()
}
case constants.DataSourceTypeCockroachdb:
s, err := utils2.GetCockroachdbSessionWithTimeout(ds, svc.timeout)
if err != nil {
return err
}
if s != nil {
s.Close()
}
case constants.DataSourceTypeElasticSearch:
_, err := utils2.GetElasticsearchClientWithTimeout(ds, svc.timeout)
if err != nil {
return err
}
case constants.DataSourceTypeKafka:
c, err := utils2.GetKafkaConnectionWithTimeout(ds, svc.timeout)
if err != nil {
return err
}
if c != nil {
c.Close()
}
default:
log.Warnf("[DataSourceService] invalid data source type: %s", ds.Type)
}
return nil
}
func NewDataSourceService(opts ...DataSourceServiceOption) (svc2 interfaces.DataSourceService, err error) {
// service
svc := &Service{
monitorInterval: 15 * time.Second,
timeout: 10 * time.Second,
}
// apply options
for _, opt := range opts {
opt(svc)
}
// dependency injection
if err := container.GetContainer().Invoke(func(modelSvc service.ModelService) {
svc.modelSvc = modelSvc
}); err != nil {
return nil, trace.TraceError(err)
}
// initialize
svc.Init()
// start
svc.Start()
return svc, nil
}
var _dsSvc interfaces.DataSourceService
func GetDataSourceService() (svc interfaces.DataSourceService, err error) {
if _dsSvc != nil {
return _dsSvc, nil
}
svc, err = NewDataSourceService()
if err != nil {
return nil, err
}
_dsSvc = svc
return svc, nil
}

76
core/ds/sql.go
View File

@@ -1,76 +0,0 @@
package ds
import (
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
utils2 "github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/trace"
"github.com/upper/db/v4"
"time"
)
type SqlService struct {
// dependencies
modelSvc service.ModelService
// internals
ds *models.DataSource
dc *models.DataCollection
s db.Session
col db.Collection
t time.Time
}
func (svc *SqlService) Insert(records ...interface{}) (err error) {
for _, d := range records {
var r entity.Result
switch d.(type) {
case entity.Result:
r = d.(entity.Result)
default:
continue
}
_r := r.Flatten()
if _, err = svc.col.Insert(_r); err != nil {
trace.PrintError(err)
continue
}
}
return nil
}
func (svc *SqlService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) {
var docs []entity.Result
if err := svc.col.Find(utils2.GetSqlQuery(query)).
Offset(opts.Skip).
Limit(opts.Limit).All(&docs); err != nil {
return nil, trace.TraceError(err)
}
for i := range docs {
d := docs[i].ToJSON()
results = append(results, &d)
}
return results, nil
}
func (svc *SqlService) Count(query generic.ListQuery) (n int, err error) {
nInt64, err := svc.col.Find(utils2.GetSqlQuery(query)).Count()
if err != nil {
return n, err
}
return int(nInt64), nil
}
func (svc *SqlService) Index(fields []string) {
// TODO: implement me
}
func (svc *SqlService) SetTime(t time.Time) {
svc.t = t
}
func (svc *SqlService) GetTime() (t time.Time) {
return svc.t
}

6
core/ds/sql_options.go
View File

@@ -1,6 +0,0 @@
package ds
type SqlOptions struct {
DefaultHost string
DefaultPort string
}

52
core/ds/sqlite.go
View File

@@ -1,52 +0,0 @@
package ds
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
utils2 "github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/trace"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type SqliteService struct {
SqlService
}
func NewDataSourceSqliteService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) {
// service
svc := &SqliteService{}
// dependency injection
svc.modelSvc, err = service.GetService()
if err != nil {
return nil, trace.TraceError(err)
}
// data source
if dsId.IsZero() {
svc.ds = &models.DataSource{}
} else {
svc.ds, err = svc.modelSvc.GetDataSourceById(dsId)
if err != nil {
return nil, trace.TraceError(err)
}
}
// data collection
svc.dc, err = svc.modelSvc.GetDataCollectionById(colId)
if err != nil {
return nil, trace.TraceError(err)
}
// session
svc.s, err = utils2.GetSqliteSession(svc.ds)
if err != nil {
return nil, trace.TraceError(err)
}
// collection
svc.col = svc.s.Collection(svc.dc.Name)
return svc, nil
}

420
core/grpc/client/client.go
View File

@@ -1,420 +0,0 @@
package client
import (
"context"
"encoding/json"
"github.com/apex/log"
"github.com/cenkalti/backoff/v4"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/grpc/middlewares"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/utils"
grpc2 "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/crawlab/trace"
"github.com/spf13/viper"
"google.golang.org/grpc"
"google.golang.org/grpc/connectivity"
"io"
"time"
)
type Client struct {
// dependencies
nodeCfgSvc interfaces.NodeConfigService
// settings
cfgPath string
address interfaces.Address
timeout time.Duration
subscribeType string
handleMessage bool
// internals
conn *grpc.ClientConn
stream grpc2.NodeService_SubscribeClient
msgCh chan *grpc2.StreamMessage
err error
// grpc clients
ModelDelegateClient grpc2.ModelDelegateClient
ModelBaseServiceClient grpc2.ModelBaseServiceClient
NodeClient grpc2.NodeServiceClient
TaskClient grpc2.TaskServiceClient
MessageClient grpc2.MessageServiceClient
}
func (c *Client) Init() (err error) {
// do nothing
return nil
}
func (c *Client) Start() (err error) {
// connect
if err := c.connect(); err != nil {
return err
}
// register rpc services
if err := c.Register(); err != nil {
return err
}
// subscribe
if err := c.subscribe(); err != nil {
return err
}
// handle stream message
if c.handleMessage {
go c.handleStreamMessage()
}
return nil
}
func (c *Client) Stop() (err error) {
// skip if connection is nil
if c.conn == nil {
return nil
}
// grpc server address
address := c.address.String()
// unsubscribe
if err := c.unsubscribe(); err != nil {
return err
}
log.Infof("grpc client unsubscribed from %s", address)
// close connection
if err := c.conn.Close(); err != nil {
return err
}
log.Infof("grpc client disconnected from %s", address)
return nil
}
func (c *Client) Register() (err error) {
// model delegate
c.ModelDelegateClient = grpc2.NewModelDelegateClient(c.conn)
// model base service
c.ModelBaseServiceClient = grpc2.NewModelBaseServiceClient(c.conn)
// node
c.NodeClient = grpc2.NewNodeServiceClient(c.conn)
// task
c.TaskClient = grpc2.NewTaskServiceClient(c.conn)
// message
c.MessageClient = grpc2.NewMessageServiceClient(c.conn)
// log
log.Infof("[GrpcClient] grpc client registered client services")
log.Debugf("[GrpcClient] ModelDelegateClient: %v", c.ModelDelegateClient)
log.Debugf("[GrpcClient] ModelBaseServiceClient: %v", c.ModelBaseServiceClient)
log.Debugf("[GrpcClient] NodeClient: %v", c.NodeClient)
log.Debugf("[GrpcClient] TaskClient: %v", c.TaskClient)
log.Debugf("[GrpcClient] MessageClient: %v", c.MessageClient)
return nil
}
func (c *Client) GetModelDelegateClient() (res grpc2.ModelDelegateClient) {
return c.ModelDelegateClient
}
func (c *Client) GetModelBaseServiceClient() (res grpc2.ModelBaseServiceClient) {
return c.ModelBaseServiceClient
}
func (c *Client) GetNodeClient() grpc2.NodeServiceClient {
return c.NodeClient
}
func (c *Client) GetTaskClient() grpc2.TaskServiceClient {
return c.TaskClient
}
func (c *Client) GetMessageClient() grpc2.MessageServiceClient {
return c.MessageClient
}
func (c *Client) SetAddress(address interfaces.Address) {
c.address = address
}
func (c *Client) SetTimeout(timeout time.Duration) {
c.timeout = timeout
}
func (c *Client) SetSubscribeType(value string) {
c.subscribeType = value
}
func (c *Client) SetHandleMessage(handleMessage bool) {
c.handleMessage = handleMessage
}
func (c *Client) Context() (ctx context.Context, cancel context.CancelFunc) {
return context.WithTimeout(context.Background(), c.timeout)
}
func (c *Client) NewRequest(d interface{}) (req *grpc2.Request) {
return &grpc2.Request{
NodeKey: c.nodeCfgSvc.GetNodeKey(),
Data: c.getRequestData(d),
}
}
func (c *Client) GetConfigPath() (path string) {
return c.cfgPath
}
func (c *Client) SetConfigPath(path string) {
c.cfgPath = path
}
func (c *Client) NewModelBaseServiceRequest(id interfaces.ModelId, params interfaces.GrpcBaseServiceParams) (req *grpc2.Request, err error) {
data, err := json.Marshal(params)
if err != nil {
return nil, trace.TraceError(err)
}
msg := &entity.GrpcBaseServiceMessage{
ModelId: id,
Data: data,
}
return c.NewRequest(msg), nil
}
func (c *Client) GetMessageChannel() (msgCh chan *grpc2.StreamMessage) {
return c.msgCh
}
func (c *Client) Restart() (err error) {
if c.needRestart() {
return c.Start()
}
return nil
}
func (c *Client) IsStarted() (res bool) {
return c.conn != nil
}
func (c *Client) IsClosed() (res bool) {
if c.conn != nil {
return c.conn.GetState() == connectivity.Shutdown
}
return false
}
func (c *Client) Err() (err error) {
return c.err
}
func (c *Client) GetStream() (stream grpc2.NodeService_SubscribeClient) {
return c.stream
}
func (c *Client) connect() (err error) {
return backoff.RetryNotify(c._connect, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client connect"))
}
func (c *Client) _connect() (err error) {
// grpc server address
address := c.address.String()
// timeout context
ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
defer cancel()
// connection
// TODO: configure dial options
var opts []grpc.DialOption
opts = append(opts, grpc.WithInsecure())
opts = append(opts, grpc.WithBlock())
opts = append(opts, grpc.WithChainUnaryInterceptor(middlewares.GetAuthTokenUnaryChainInterceptor(c.nodeCfgSvc)))
opts = append(opts, grpc.WithChainStreamInterceptor(middlewares.GetAuthTokenStreamChainInterceptor(c.nodeCfgSvc)))
c.conn, err = grpc.DialContext(ctx, address, opts...)
if err != nil {
_ = trace.TraceError(err)
return errors.ErrorGrpcClientFailedToStart
}
log.Infof("[GrpcClient] grpc client connected to %s", address)
return nil
}
func (c *Client) subscribe() (err error) {
var op func() error
switch c.subscribeType {
case constants.GrpcSubscribeTypeNode:
op = c._subscribeNode
default:
return errors.ErrorGrpcInvalidType
}
return backoff.RetryNotify(op, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client subscribe"))
}
func (c *Client) _subscribeNode() (err error) {
req := c.NewRequest(&entity.NodeInfo{
Key: c.nodeCfgSvc.GetNodeKey(),
IsMaster: false,
})
c.stream, err = c.GetNodeClient().Subscribe(context.Background(), req)
if err != nil {
return trace.TraceError(err)
}
// log
log.Infof("[GrpcClient] grpc client subscribed to remote server")
return nil
}
func (c *Client) unsubscribe() (err error) {
req := c.NewRequest(&entity.NodeInfo{
Key: c.nodeCfgSvc.GetNodeKey(),
IsMaster: false,
})
if _, err = c.GetNodeClient().Unsubscribe(context.Background(), req); err != nil {
return trace.TraceError(err)
}
return nil
}
func (c *Client) handleStreamMessage() {
log.Infof("[GrpcClient] start handling stream message...")
for {
// resubscribe if stream is set to nil
if c.stream == nil {
if err := backoff.RetryNotify(c.subscribe, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client subscribe")); err != nil {
log.Errorf("subscribe")
return
}
}
// receive stream message
msg, err := c.stream.Recv()
log.Debugf("[GrpcClient] received message: %v", msg)
if err != nil {
// set error
c.err = err
// end
if err == io.EOF {
log.Infof("[GrpcClient] received EOF signal, disconnecting")
return
}
// connection closed
if c.IsClosed() {
return
}
// error
trace.PrintError(err)
c.stream = nil
time.Sleep(1 * time.Second)
continue
}
// send stream message to channel
c.msgCh <- msg
// reset error
c.err = nil
}
}
func (c *Client) needRestart() bool {
switch c.conn.GetState() {
case connectivity.Shutdown, connectivity.TransientFailure:
return true
case connectivity.Idle, connectivity.Connecting, connectivity.Ready:
return false
default:
return false
}
}
func (c *Client) getRequestData(d interface{}) (data []byte) {
if d == nil {
return data
}
switch d.(type) {
case []byte:
data = d.([]byte)
default:
var err error
data, err = json.Marshal(d)
if err != nil {
panic(err)
}
}
return data
}
func NewClient() (res interfaces.GrpcClient, err error) {
// client
client := &Client{
address: entity.NewAddress(&entity.AddressOptions{
Host: constants.DefaultGrpcClientRemoteHost,
Port: constants.DefaultGrpcClientRemotePort,
}),
timeout: 10 * time.Second,
msgCh: make(chan *grpc2.StreamMessage),
subscribeType: constants.GrpcSubscribeTypeNode,
handleMessage: true,
}
if viper.GetString("grpc.address") != "" {
client.address, err = entity.NewAddressFromString(viper.GetString("grpc.address"))
if err != nil {
return nil, trace.TraceError(err)
}
}
// dependency injection
if err := container.GetContainer().Invoke(func(nodeCfgSvc interfaces.NodeConfigService) {
client.nodeCfgSvc = nodeCfgSvc
}); err != nil {
return nil, err
}
// init
if err := client.Init(); err != nil {
return nil, err
}
return client, nil
}
var _client interfaces.GrpcClient
func GetClient() (c interfaces.GrpcClient, err error) {
if _client != nil {
return _client, nil
}
_client, err = createClient()
if err != nil {
return nil, err
}
return _client, nil
}
func createClient() (client2 interfaces.GrpcClient, err error) {
if err := container.GetContainer().Invoke(func(client interfaces.GrpcClient) {
client2 = client
}); err != nil {
return nil, trace.TraceError(err)
}
return client2, nil
}

38
core/grpc/client/options.go
View File

@@ -2,50 +2,12 @@ package client
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"time"
)
type Option func(client interfaces.GrpcClient)
func WithConfigPath(path string) Option {
return func(c interfaces.GrpcClient) {
c.SetConfigPath(path)
}
}
func WithAddress(address interfaces.Address) Option {
return func(c interfaces.GrpcClient) {
c.SetAddress(address)
}
}
func WithTimeout(timeout time.Duration) Option {
return func(c interfaces.GrpcClient) {
}
}
func WithSubscribeType(subscribeType string) Option {
return func(c interfaces.GrpcClient) {
c.SetSubscribeType(subscribeType)
}
}
func WithHandleMessage(handleMessage bool) Option {
return func(c interfaces.GrpcClient) {
c.SetHandleMessage(handleMessage)
}
}
type PoolOption func(p interfaces.GrpcClientPool)
func WithPoolConfigPath(path string) PoolOption {
return func(c interfaces.GrpcClientPool) {
c.SetConfigPath(path)
}
}
func WithPoolSize(size int) PoolOption {
return func(c interfaces.GrpcClientPool) {
c.SetSize(size)
}
}

88
core/grpc/client/pool.go
View File

@@ -1,88 +0,0 @@
package client
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/trace"
"github.com/emirpasic/gods/lists/arraylist"
"math/rand"
)
type Pool struct {
// settings
size int
cfgPath string
// internals
clients *arraylist.List
}
func (p *Pool) GetConfigPath() (path string) {
return p.cfgPath
}
func (p *Pool) SetConfigPath(path string) {
p.cfgPath = path
}
func (p *Pool) Init() (err error) {
for i := 0; i < p.size; i++ {
if err := p.NewClient(); err != nil {
return err
}
}
return nil
}
func (p *Pool) NewClient() (err error) {
c, err := NewClient()
if err != nil {
return trace.TraceError(err)
}
if err := c.Start(); err != nil {
return err
}
p.clients.Add(c)
return nil
}
func (p *Pool) GetClient() (c interfaces.GrpcClient, err error) {
idx := p.getRandomIndex()
res, ok := p.clients.Get(idx)
if !ok {
return nil, trace.TraceError(errors.ErrorGrpcClientNotExists)
}
c, ok = res.(interfaces.GrpcClient)
if !ok {
return nil, trace.TraceError(errors.ErrorGrpcInvalidType)
}
return c, nil
}
func (p *Pool) SetSize(size int) {
p.size = size
}
func (p *Pool) getRandomIndex() (idx int) {
return rand.Intn(p.clients.Size())
}
func NewPool(opts ...PoolOption) (p interfaces.GrpcClientPool, err error) {
// pool
p = &Pool{
size: 1,
clients: arraylist.New(),
}
// apply options
for _, opt := range opts {
opt(p)
}
// initialize
if err := p.Init(); err != nil {
return nil, err
}
return p, nil
}

5
core/grpc/client/utils_proto.go
View File

@@ -1,5 +0,0 @@
package client
import grpc2 "github.com/crawlab-team/crawlab/grpc"
var EmptyRequest = &grpc2.Request{}

93
core/grpc/server/message_server.go
View File

@@ -1,93 +0,0 @@
package server
import (
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/service"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/crawlab/trace"
"io"
)
type MessageServer struct {
grpc.UnimplementedMessageServiceServer
// dependencies
modelSvc service.ModelService
cfgSvc interfaces.NodeConfigService
// internals
server interfaces.GrpcServer
}
func (svr MessageServer) Connect(stream grpc.MessageService_ConnectServer) (err error) {
finished := make(chan bool)
for {
msg, err := stream.Recv()
nodeKey := "unknown node key"
if msg != nil {
nodeKey = msg.NodeKey
}
if err == io.EOF {
log.Infof("[MessageServer] received signal EOF from node[%s], now quit", nodeKey)
return nil
}
if err != nil {
log.Errorf("[MessageServer] receiving message error from node[%s]: %v", nodeKey, err)
return err
}
switch msg.Code {
case grpc.StreamMessageCode_CONNECT:
log.Infof("[MessageServer] received connect request from node[%s], key: %s", nodeKey, msg.Key)
svr.server.SetSubscribe(msg.Key, &entity.GrpcSubscribe{
Stream: stream,
Finished: finished,
})
case grpc.StreamMessageCode_DISCONNECT:
log.Infof("[MessageServer] received disconnect request from node[%s], key: %s", nodeKey, msg.Key)
svr.server.DeleteSubscribe(msg.Key)
return nil
case grpc.StreamMessageCode_SEND:
log.Debugf("[MessageServer] received send request from node[%s] to %s", nodeKey, msg.To)
sub, err := svr.server.GetSubscribe(msg.To)
if err != nil {
return err
}
svr.redirectMessage(sub, msg)
}
}
}
func (svr MessageServer) redirectMessage(sub interfaces.GrpcSubscribe, msg *grpc.StreamMessage) {
stream := sub.GetStream()
if stream == nil {
trace.PrintError(errors.ErrorGrpcStreamNotFound)
return
}
log.Debugf("[MessageServer] redirect message: %v", msg)
if err := stream.Send(msg); err != nil {
trace.PrintError(err)
return
}
}
func NewMessageServer() (res *MessageServer, err error) {
// message server
svr := &MessageServer{}
// dependency injection
if err := container.GetContainer().Invoke(func(
modelSvc service.ModelService,
cfgSvc interfaces.NodeConfigService,
) {
svr.modelSvc = modelSvc
svr.cfgSvc = cfgSvc
}); err != nil {
return nil, err
}
return svr, nil
}

139
core/grpc/server/model_base_service_server.go
View File

@@ -1,139 +0,0 @@
package server
import (
"context"
"encoding/json"
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/crawlab/trace"
)
type ModelBaseServiceServer struct {
grpc.UnimplementedModelBaseServiceServer
// dependencies
modelSvc interfaces.ModelService
}
func (svr ModelBaseServiceServer) GetById(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) {
return svc.GetById(params.Id)
})
}
func (svr ModelBaseServiceServer) Get(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) {
return svc.Get(utils.NormalizeBsonMObjectId(params.Query), params.FindOptions)
})
}
func (svr ModelBaseServiceServer) GetList(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) {
list, err := svc.GetList(utils.NormalizeBsonMObjectId(params.Query), params.FindOptions)
if err != nil {
return nil, err
}
data, err := json.Marshal(list)
if err != nil {
return nil, err
}
return data, nil
})
}
func (svr ModelBaseServiceServer) DeleteById(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) {
err := svc.DeleteById(params.Id, params.User)
return nil, err
})
}
func (svr ModelBaseServiceServer) Delete(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) {
err := svc.Delete(utils.NormalizeBsonMObjectId(params.Query), params.User)
return nil, err
})
}
func (svr ModelBaseServiceServer) DeleteList(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) {
err := svc.DeleteList(utils.NormalizeBsonMObjectId(params.Query), params.User)
return nil, err
})
}
func (svr ModelBaseServiceServer) ForceDeleteList(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) {
err := svc.ForceDeleteList(utils.NormalizeBsonMObjectId(params.Query), params.User)
return nil, err
})
}
func (svr ModelBaseServiceServer) UpdateById(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) {
err := svc.UpdateById(params.Id, params.Update)
return nil, err
})
}
func (svr ModelBaseServiceServer) Update(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) {
err := svc.Update(utils.NormalizeBsonMObjectId(params.Query), params.Update, params.Fields, params.User)
return nil, err
})
}
func (svr ModelBaseServiceServer) UpdateDoc(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) {
err := svc.UpdateDoc(utils.NormalizeBsonMObjectId(params.Query), params.Doc, params.Fields, params.User)
return nil, err
})
}
func (svr ModelBaseServiceServer) Insert(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) {
err := svc.Insert(params.User, params.Docs...)
return nil, err
})
}
func (svr ModelBaseServiceServer) Count(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) {
return svc.Count(utils.NormalizeBsonMObjectId(params.Query))
})
}
func (svr ModelBaseServiceServer) handleRequest(req *grpc.Request, handle handleBaseServiceRequest) (res *grpc.Response, err error) {
params, msg, err := NewModelBaseServiceBinder(req).BindWithBaseServiceMessage()
if err != nil {
return HandleError(err)
}
svc := svr.modelSvc.GetBaseService(msg.GetModelId())
d, err := handle(params, svc)
if err != nil {
return HandleError(err)
}
if d == nil {
return HandleSuccess()
}
return HandleSuccessWithData(d)
}
type handleBaseServiceRequest func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error)
func NewModelBaseServiceServer() (svr2 *ModelBaseServiceServer, err error) {
svr := &ModelBaseServiceServer{}
// dependency injection
if err := container.GetContainer().Invoke(func(modelSvc service.ModelService) {
svr.modelSvc = modelSvc
}); err != nil {
return nil, trace.TraceError(err)
}
return svr, nil
}

120
core/grpc/server/model_delegate_binder.go
View File

@@ -1,120 +0,0 @@
package server
import (
"encoding/json"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/grpc"
)
func NewModelDelegateBinder(req *grpc.Request) (b *ModelDelegateBinder) {
return &ModelDelegateBinder{
req: req,
msg: &entity.GrpcDelegateMessage{},
}
}
type ModelDelegateBinder struct {
req *grpc.Request
msg interfaces.GrpcModelDelegateMessage
}
func (b *ModelDelegateBinder) Bind() (res interface{}, err error) {
if err := b.bindDelegateMessage(); err != nil {
return nil, err
}
m := models.NewModelMap()
switch b.msg.GetModelId() {
case interfaces.ModelIdArtifact:
return b.process(&m.Artifact, interfaces.ModelIdTag)
case interfaces.ModelIdTag:
return b.process(&m.Tag, interfaces.ModelIdTag)
case interfaces.ModelIdNode:
return b.process(&m.Node, interfaces.ModelIdTag)
case interfaces.ModelIdProject:
return b.process(&m.Project, interfaces.ModelIdTag)
case interfaces.ModelIdSpider:
return b.process(&m.Spider, interfaces.ModelIdTag)
case interfaces.ModelIdTask:
return b.process(&m.Task)
case interfaces.ModelIdJob:
return b.process(&m.Job)
case interfaces.ModelIdSchedule:
return b.process(&m.Schedule)
case interfaces.ModelIdUser:
return b.process(&m.User)
case interfaces.ModelIdSetting:
return b.process(&m.Setting)
case interfaces.ModelIdToken:
return b.process(&m.Token)
case interfaces.ModelIdVariable:
return b.process(&m.Variable)
case interfaces.ModelIdTaskQueue:
return b.process(&m.TaskQueueItem)
case interfaces.ModelIdTaskStat:
return b.process(&m.TaskStat)
case interfaces.ModelIdSpiderStat:
return b.process(&m.SpiderStat)
case interfaces.ModelIdDataSource:
return b.process(&m.DataSource)
case interfaces.ModelIdDataCollection:
return b.process(&m.DataCollection)
case interfaces.ModelIdResult:
return b.process(&m.Result)
case interfaces.ModelIdPassword:
return b.process(&m.Password)
case interfaces.ModelIdExtraValue:
return b.process(&m.ExtraValue)
case interfaces.ModelIdGit:
return b.process(&m.Git)
case interfaces.ModelIdRole:
return b.process(&m.Role)
case interfaces.ModelIdUserRole:
return b.process(&m.UserRole)
case interfaces.ModelIdPermission:
return b.process(&m.Permission)
case interfaces.ModelIdRolePermission:
return b.process(&m.RolePermission)
case interfaces.ModelIdEnvironment:
return b.process(&m.Environment)
case interfaces.ModelIdDependencySetting:
return b.process(&m.DependencySetting)
default:
return nil, errors.ErrorModelInvalidModelId
}
}
func (b *ModelDelegateBinder) MustBind() (res interface{}) {
res, err := b.Bind()
if err != nil {
panic(err)
}
return res
}
func (b *ModelDelegateBinder) BindWithDelegateMessage() (res interface{}, msg interfaces.GrpcModelDelegateMessage, err error) {
if err := json.Unmarshal(b.req.Data, b.msg); err != nil {
return nil, nil, err
}
res, err = b.Bind()
if err != nil {
return nil, nil, err
}
return res, b.msg, nil
}
func (b *ModelDelegateBinder) process(d interface{}, fieldIds ...interfaces.ModelId) (res interface{}, err error) {
if err := json.Unmarshal(b.msg.GetData(), d); err != nil {
return nil, err
}
//return models.AssignFields(d, fieldIds...) // TODO: do we need to assign fields?
return d, nil
}
func (b *ModelDelegateBinder) bindDelegateMessage() (err error) {
return json.Unmarshal(b.req.Data, b.msg)
}

67
core/grpc/server/model_delegate_server.go
View File

@@ -1,67 +0,0 @@
package server
import (
"context"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/delegate"
grpc "github.com/crawlab-team/crawlab/grpc"
)
type ModelDelegateServer struct {
grpc.UnimplementedModelDelegateServer
}
// Do and perform an RPC action of constants.Delegate
func (svr ModelDelegateServer) Do(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
// bind message
obj, msg, err := NewModelDelegateBinder(req).BindWithDelegateMessage()
if err != nil {
return HandleError(err)
}
// convert to model
doc, ok := obj.(interfaces.Model)
if !ok {
return HandleError(errors.ErrorModelInvalidType)
}
// model delegate
d := delegate.NewModelDelegate(doc)
// apply method
switch msg.GetMethod() {
case interfaces.ModelDelegateMethodAdd:
err = d.Add()
case interfaces.ModelDelegateMethodSave:
err = d.Save()
case interfaces.ModelDelegateMethodDelete:
err = d.Delete()
case interfaces.ModelDelegateMethodGetArtifact, interfaces.ModelDelegateMethodRefresh:
err = d.Refresh()
}
if err != nil {
return HandleError(err)
}
// model
m := d.GetModel()
if msg.GetMethod() == interfaces.ModelDelegateMethodGetArtifact {
m, err = d.GetArtifact()
if err != nil {
return nil, err
}
}
// json bytes
data, err := d.ToBytes(m)
if err != nil {
return nil, err
}
return HandleSuccessWithData(data)
}
func NewModelDelegateServer() (svr *ModelDelegateServer) {
return &ModelDelegateServer{}
}

199
core/grpc/server/node_server.go
View File

@@ -1,199 +0,0 @@
package server
import (
"context"
"encoding/json"
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/delegate"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
"github.com/crawlab-team/crawlab/grpc"
"go.mongodb.org/mongo-driver/mongo"
)
type NodeServer struct {
grpc.UnimplementedNodeServiceServer
// dependencies
modelSvc service.ModelService
cfgSvc interfaces.NodeConfigService
// internals
server interfaces.GrpcServer
}
// Register from handler/worker to master
func (svr NodeServer) Register(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
// unmarshall data
var nodeInfo entity.NodeInfo
if req.Data != nil {
if err := json.Unmarshal(req.Data, &nodeInfo); err != nil {
return HandleError(err)
}
if nodeInfo.IsMaster {
// error: cannot register master node
return HandleError(errors.ErrorGrpcNotAllowed)
}
}
// node key
var nodeKey string
if req.NodeKey != "" {
nodeKey = req.NodeKey
} else {
nodeKey = nodeInfo.Key
}
if nodeKey == "" {
return HandleError(errors.ErrorModelMissingRequiredData)
}
// find in db
node, err := svr.modelSvc.GetNodeByKey(nodeKey, nil)
if err == nil {
if node.IsMaster {
// error: cannot register master node
return HandleError(errors.ErrorGrpcNotAllowed)
} else {
// register existing
node.Status = constants.NodeStatusRegistered
node.Active = true
nodeD := delegate.NewModelNodeDelegate(node)
if err := nodeD.Save(); err != nil {
return HandleError(err)
}
var ok bool
node, ok = nodeD.GetModel().(*models.Node)
if !ok {
return HandleError(errors.ErrorGrpcInvalidType)
}
log.Infof("[NodeServer] updated worker[%s] in db. id: %s", nodeKey, nodeD.GetModel().GetId().Hex())
}
} else if err == mongo.ErrNoDocuments {
// register new
node = &models.Node{
Key: nodeKey,
Name: nodeInfo.Name,
Ip: nodeInfo.Ip,
Hostname: nodeInfo.Hostname,
Description: nodeInfo.Description,
MaxRunners: nodeInfo.MaxRunners,
Status: constants.NodeStatusRegistered,
Active: true,
Enabled: true,
}
if node.Name == "" {
node.Name = nodeKey
}
nodeD := delegate.NewModelDelegate(node)
if err := nodeD.Add(); err != nil {
return HandleError(err)
}
var ok bool
node, ok = nodeD.GetModel().(*models.Node)
if !ok {
return HandleError(errors.ErrorGrpcInvalidType)
}
log.Infof("[NodeServer] added worker[%s] in db. id: %s", nodeKey, nodeD.GetModel().GetId().Hex())
} else {
// error
return HandleError(err)
}
log.Infof("[NodeServer] master registered worker[%s]", req.GetNodeKey())
return HandleSuccessWithData(node)
}
// SendHeartbeat from worker to master
func (svr NodeServer) SendHeartbeat(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
// find in db
node, err := svr.modelSvc.GetNodeByKey(req.NodeKey, nil)
if err != nil {
if err == mongo.ErrNoDocuments {
return HandleError(errors.ErrorNodeNotExists)
}
return HandleError(err)
}
// validate status
if node.Status == constants.NodeStatusUnregistered {
return HandleError(errors.ErrorNodeUnregistered)
}
// update status
nodeD := delegate.NewModelNodeDelegate(node)
if err := nodeD.UpdateStatusOnline(); err != nil {
return HandleError(err)
}
return HandleSuccessWithData(node)
}
// Ping from worker to master
func (svr NodeServer) Ping(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
return HandleSuccess()
}
func (svr NodeServer) Subscribe(request *grpc.Request, stream grpc.NodeService_SubscribeServer) (err error) {
log.Infof("[NodeServer] master received subscribe request from node[%s]", request.NodeKey)
// finished channel
finished := make(chan bool)
// set subscribe
svr.server.SetSubscribe("node:"+request.NodeKey, &entity.GrpcSubscribe{
Stream: stream,
Finished: finished,
})
ctx := stream.Context()
log.Infof("[NodeServer] master subscribed node[%s]", request.NodeKey)
// Keep this scope alive because once this scope exits - the stream is closed
for {
select {
case <-finished:
log.Infof("[NodeServer] closing stream for node[%s]", request.NodeKey)
return nil
case <-ctx.Done():
log.Infof("[NodeServer] node[%s] has disconnected", request.NodeKey)
return nil
}
}
}
func (svr NodeServer) Unsubscribe(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) {
sub, err := svr.server.GetSubscribe("node:" + req.NodeKey)
if err != nil {
return nil, errors.ErrorGrpcSubscribeNotExists
}
select {
case sub.GetFinished() <- true:
log.Infof("unsubscribed node[%s]", req.NodeKey)
default:
// Default case is to avoid blocking in case client has already unsubscribed
}
svr.server.DeleteSubscribe(req.NodeKey)
return &grpc.Response{
Code: grpc.ResponseCode_OK,
Message: "unsubscribed successfully",
}, nil
}
func NewNodeServer() (res *NodeServer, err error) {
// node server
svr := &NodeServer{}
svr.modelSvc, err = service.GetService()
if err != nil {
return nil, err
}
svr.cfgSvc = nodeconfig.GetNodeConfigService()
return svr, nil
}

43
core/i18n/service.go
View File

@@ -1,43 +0,0 @@
package i18n
import "github.com/crawlab-team/crawlab/core/interfaces"
var translations []interfaces.Translation
var _svc interfaces.I18nService
type Service struct {
}
func (svc *Service) AddTranslations(t []interfaces.Translation) {
translations = append(translations, t...)
}
func (svc *Service) GetTranslations() (t []interfaces.Translation) {
return translations
}
func GetI18nService(cfgPath string) (svc2 interfaces.I18nService, err error) {
if _svc != nil {
return _svc, nil
}
_svc, err = NewI18nService()
if err != nil {
return nil, err
}
return _svc, nil
}
func ProvideGetI18nService(cfgPath string) func() (svc interfaces.I18nService, err error) {
return func() (svc interfaces.I18nService, err error) {
return GetI18nService(cfgPath)
}
}
func NewI18nService() (svc2 interfaces.I18nService, err error) {
svc := &Service{}
return svc, nil
}

50
core/middlewares/auth.go
View File

@@ -1,50 +0,0 @@
package middlewares
import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/user"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/gin-gonic/gin"
"github.com/spf13/viper"
)
func AuthorizationMiddleware() gin.HandlerFunc {
userSvc, _ := user.GetUserService()
return func(c *gin.Context) {
// disable auth for test
if viper.GetBool("auth.disabled") {
modelSvc, err := service.GetService()
if err != nil {
utils.HandleErrorInternalServerError(c, err)
return
}
u, err := modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil)
if err != nil {
utils.HandleErrorInternalServerError(c, err)
return
}
c.Set(constants.UserContextKey, u)
c.Next()
return
}
// token string
tokenStr := c.GetHeader("Authorization")
// validate token
u, err := userSvc.CheckToken(tokenStr)
if err != nil {
// validation failed, return error response
utils.HandleErrorUnauthorized(c, errors.ErrorHttpUnauthorized)
return
}
// set user in context
c.Set(constants.UserContextKey, u)
// validation success
c.Next()
}
}

9
core/middlewares/auth_v2.go
View File

@@ -3,11 +3,13 @@ package middlewares
import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/user"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/gin-gonic/gin"
"github.com/spf13/viper"
"go.mongodb.org/mongo-driver/bson"
)
func AuthorizationMiddlewareV2() gin.HandlerFunc {
@@ -15,12 +17,7 @@ func AuthorizationMiddlewareV2() gin.HandlerFunc {
return func(c *gin.Context) {
// disable auth for test
if viper.GetBool("auth.disabled") {
modelSvc, err := service.GetService()
if err != nil {
utils.HandleErrorInternalServerError(c, err)
return
}
u, err := modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil)
u, err := service.NewModelServiceV2[models.UserV2]().GetOne(bson.M{"username": constants.DefaultAdminUsername}, nil)
if err != nil {
utils.HandleErrorInternalServerError(c, err)
return

34
core/middlewares/filer_auth.go
View File

@@ -1,34 +0,0 @@
package middlewares
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/gin-gonic/gin"
"github.com/spf13/viper"
)
func FilerAuthorizationMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
// auth key
authKey := c.GetHeader("Authorization")
// server auth key
svrAuthKey := viper.GetString("fs.filer.authKey")
// skip to next if no server auth key is provided
if svrAuthKey == "" {
c.Next()
return
}
// validate
if authKey != svrAuthKey {
// validation failed, return error response
utils.HandleErrorUnauthorized(c, errors.ErrorHttpUnauthorized)
return
}
// validation success
c.Next()
}
}

92
core/models/client/binder_basic.go
View File

@@ -1,92 +0,0 @@
package client
import (
"encoding/json"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/crawlab/trace"
)
func NewBasicBinder(id interfaces.ModelId, res *grpc.Response) (b interfaces.GrpcModelBinder) {
return &BasicBinder{
id: id,
res: res,
}
}
type BasicBinder struct {
id interfaces.ModelId
res *grpc.Response
}
func (b *BasicBinder) Bind() (res interfaces.Model, err error) {
m := models.NewModelMap()
switch b.id {
case interfaces.ModelIdArtifact:
return b.Process(&m.Artifact)
case interfaces.ModelIdTag:
return b.Process(&m.Tag)
case interfaces.ModelIdNode:
return b.Process(&m.Node)
case interfaces.ModelIdProject:
return b.Process(&m.Project)
case interfaces.ModelIdSpider:
return b.Process(&m.Spider)
case interfaces.ModelIdTask:
return b.Process(&m.Task)
case interfaces.ModelIdJob:
return b.Process(&m.Job)
case interfaces.ModelIdSchedule:
return b.Process(&m.Schedule)
case interfaces.ModelIdUser:
return b.Process(&m.User)
case interfaces.ModelIdSetting:
return b.Process(&m.Setting)
case interfaces.ModelIdToken:
return b.Process(&m.Token)
case interfaces.ModelIdVariable:
return b.Process(&m.Variable)
case interfaces.ModelIdTaskQueue:
return b.Process(&m.TaskQueueItem)
case interfaces.ModelIdTaskStat:
return b.Process(&m.TaskStat)
case interfaces.ModelIdSpiderStat:
return b.Process(&m.SpiderStat)
case interfaces.ModelIdDataSource:
return b.Process(&m.DataSource)
case interfaces.ModelIdDataCollection:
return b.Process(&m.DataCollection)
case interfaces.ModelIdResult:
return b.Process(&m.Result)
case interfaces.ModelIdPassword:
return b.Process(&m.Password)
case interfaces.ModelIdExtraValue:
return b.Process(&m.ExtraValue)
case interfaces.ModelIdGit:
return b.Process(&m.Git)
case interfaces.ModelIdRole:
return b.Process(&m.Role)
case interfaces.ModelIdUserRole:
return b.Process(&m.UserRole)
case interfaces.ModelIdPermission:
return b.Process(&m.Permission)
case interfaces.ModelIdRolePermission:
return b.Process(&m.RolePermission)
case interfaces.ModelIdEnvironment:
return b.Process(&m.Environment)
case interfaces.ModelIdDependencySetting:
return b.Process(&m.DependencySetting)
default:
return nil, errors.ErrorModelInvalidModelId
}
}
func (b *BasicBinder) Process(d interfaces.Model) (res interfaces.Model, err error) {
if err := json.Unmarshal(b.res.Data, d); err != nil {
return nil, trace.TraceError(err)
}
return d, nil
}

100
core/models/client/binder_list.go
View File

@@ -1,100 +0,0 @@
package client
import (
"encoding/json"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/crawlab/trace"
)
func NewListBinder(id interfaces.ModelId, res *grpc.Response) (b interfaces.GrpcModelListBinder) {
return &ListBinder{
id: id,
res: res,
}
}
type ListBinder struct {
id interfaces.ModelId
res *grpc.Response
}
func (b *ListBinder) Bind() (l interfaces.List, err error) {
m := models.NewModelListMap()
switch b.id {
case interfaces.ModelIdArtifact:
return b.Process(&m.Artifacts)
case interfaces.ModelIdTag:
return b.Process(&m.Tags)
case interfaces.ModelIdNode:
return b.Process(&m.Nodes)
case interfaces.ModelIdProject:
return b.Process(&m.Projects)
case interfaces.ModelIdSpider:
return b.Process(&m.Spiders)
case interfaces.ModelIdTask:
return b.Process(&m.Tasks)
case interfaces.ModelIdJob:
return b.Process(&m.Jobs)
case interfaces.ModelIdSchedule:
return b.Process(&m.Schedules)
case interfaces.ModelIdUser:
return b.Process(&m.Users)
case interfaces.ModelIdSetting:
return b.Process(&m.Settings)
case interfaces.ModelIdToken:
return b.Process(&m.Tokens)
case interfaces.ModelIdVariable:
return b.Process(&m.Variables)
case interfaces.ModelIdTaskQueue:
return b.Process(&m.TaskQueueItems)
case interfaces.ModelIdTaskStat:
return b.Process(&m.TaskStats)
case interfaces.ModelIdSpiderStat:
return b.Process(&m.SpiderStats)
case interfaces.ModelIdDataSource:
return b.Process(&m.DataSources)
case interfaces.ModelIdDataCollection:
return b.Process(&m.DataCollections)
case interfaces.ModelIdResult:
return b.Process(&m.Results)
case interfaces.ModelIdPassword:
return b.Process(&m.Passwords)
case interfaces.ModelIdExtraValue:
return b.Process(&m.ExtraValues)
case interfaces.ModelIdGit:
return b.Process(&m.Gits)
case interfaces.ModelIdRole:
return b.Process(&m.Roles)
case interfaces.ModelIdUserRole:
return b.Process(&m.UserRoles)
case interfaces.ModelIdPermission:
return b.Process(&m.PermissionList)
case interfaces.ModelIdRolePermission:
return b.Process(&m.RolePermissionList)
case interfaces.ModelIdEnvironment:
return b.Process(&m.Environments)
case interfaces.ModelIdDependencySetting:
return b.Process(&m.DependencySettings)
default:
return l, errors.ErrorModelInvalidModelId
}
}
func (b *ListBinder) MustBind() (res interface{}) {
res, err := b.Bind()
if err != nil {
panic(err)
}
return res
}
func (b *ListBinder) Process(d interface{}) (l interfaces.List, err error) {
if err := json.Unmarshal(b.res.Data, d); err != nil {
return l, trace.TraceError(err)
}
return d.(interfaces.List), nil
}

251
core/models/client/model_base_service.go
View File

@@ -1,251 +0,0 @@
package client
import (
"encoding/json"
"github.com/apex/log"
"github.com/cenkalti/backoff/v4"
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/crawlab/trace"
errors2 "github.com/pkg/errors"
"github.com/spf13/viper"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"time"
)
type BaseServiceDelegate struct {
// settings
cfgPath string
// internals
id interfaces.ModelId
c interfaces.GrpcClient
}
func (d *BaseServiceDelegate) GetModelId() (id interfaces.ModelId) {
return d.id
}
func (d *BaseServiceDelegate) SetModelId(id interfaces.ModelId) {
d.id = id
}
func (d *BaseServiceDelegate) GetConfigPath() (path string) {
return d.cfgPath
}
func (d *BaseServiceDelegate) SetConfigPath(path string) {
d.cfgPath = path
}
func (d *BaseServiceDelegate) GetById(id primitive.ObjectID) (doc interfaces.Model, err error) {
log.Debugf("[BaseServiceDelegate] get by id[%s]", id.Hex())
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id})
c := d.getModelBaseServiceClient()
if c == nil {
return nil, trace.TraceError(errors.ErrorModelNilPointer)
}
log.Debugf("[BaseServiceDelegate] get by id[%s] req: %v", id.Hex(), req)
res, err := c.GetById(ctx, req)
if err != nil {
return nil, trace.TraceError(err)
}
log.Debugf("[BaseServiceDelegate] get by id[%s] res: %v", id.Hex(), res)
return NewBasicBinder(d.id, res).Bind()
}
func (d *BaseServiceDelegate) Get(query bson.M, opts *mongo.FindOptions) (doc interfaces.Model, err error) {
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, FindOptions: opts})
res, err := d.getModelBaseServiceClient().Get(ctx, req)
if err != nil {
return nil, err
}
return NewBasicBinder(d.id, res).Bind()
}
func (d *BaseServiceDelegate) GetList(query bson.M, opts *mongo.FindOptions) (l interfaces.List, err error) {
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, FindOptions: opts})
res, err := d.getModelBaseServiceClient().GetList(ctx, req)
if err != nil {
return l, err
}
return NewListBinder(d.id, res).Bind()
}
func (d *BaseServiceDelegate) DeleteById(id primitive.ObjectID, args ...interface{}) (err error) {
u := utils.GetUserFromArgs(args...)
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id, User: u})
_, err = d.getModelBaseServiceClient().DeleteById(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) Delete(query bson.M, args ...interface{}) (err error) {
u := utils.GetUserFromArgs(args...)
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u})
_, err = d.getModelBaseServiceClient().Delete(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) DeleteList(query bson.M, args ...interface{}) (err error) {
u := utils.GetUserFromArgs(args...)
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u})
_, err = d.getModelBaseServiceClient().DeleteList(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) ForceDeleteList(query bson.M, args ...interface{}) (err error) {
u := utils.GetUserFromArgs(args...)
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u})
_, err = d.getModelBaseServiceClient().ForceDeleteList(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) UpdateById(id primitive.ObjectID, update bson.M, args ...interface{}) (err error) {
u := utils.GetUserFromArgs(args...)
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id, Update: update, User: u})
_, err = d.getModelBaseServiceClient().UpdateById(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) Update(query bson.M, update bson.M, fields []string, args ...interface{}) (err error) {
u := utils.GetUserFromArgs(args...)
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, Update: update, Fields: fields, User: u})
_, err = d.getModelBaseServiceClient().Update(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) UpdateDoc(query bson.M, doc interfaces.Model, fields []string, args ...interface{}) (err error) {
u := utils.GetUserFromArgs(args...)
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, Doc: doc, Fields: fields, User: u})
_, err = d.getModelBaseServiceClient().UpdateDoc(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) Insert(u interfaces.User, docs ...interface{}) (err error) {
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Docs: docs, User: u})
_, err = d.getModelBaseServiceClient().Insert(ctx, req)
if err != nil {
return err
}
return nil
}
func (d *BaseServiceDelegate) Count(query bson.M) (total int, err error) {
ctx, cancel := d.c.Context()
defer cancel()
req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query})
res, err := d.getModelBaseServiceClient().Count(ctx, req)
if err != nil {
return total, err
}
if err := json.Unmarshal(res.Data, &total); err != nil {
return total, err
}
return total, nil
}
func (d *BaseServiceDelegate) newRequest(params interfaces.GrpcBaseServiceParams) (req *grpc.Request, err error) {
return d.c.NewModelBaseServiceRequest(d.id, params)
}
func (d *BaseServiceDelegate) mustNewRequest(params *entity.GrpcBaseServiceParams) (req *grpc.Request) {
req, err := d.newRequest(params)
if err != nil {
panic(err)
}
return req
}
func (d *BaseServiceDelegate) getModelBaseServiceClient() (c grpc.ModelBaseServiceClient) {
if err := backoff.Retry(func() (err error) {
c = d.c.GetModelBaseServiceClient()
if c == nil {
err = errors2.New("unable to get model base service client")
log.Debugf("[BaseServiceDelegate] err: %v", err)
return err
}
return nil
}, backoff.NewConstantBackOff(1*time.Second)); err != nil {
trace.PrintError(err)
}
return c
}
func NewBaseServiceDelegate(opts ...ModelBaseServiceDelegateOption) (svc2 interfaces.GrpcClientModelBaseService, err error) {
// base service
svc := &BaseServiceDelegate{}
// apply options
for _, opt := range opts {
opt(svc)
}
// config path
if viper.GetString("config.path") != "" {
svc.cfgPath = viper.GetString("config.path")
}
// dependency injection
if err := container.GetContainer().Invoke(func(client interfaces.GrpcClient) {
svc.c = client
}); err != nil {
return nil, err
}
return svc, nil
}
func ProvideBaseServiceDelegate(id interfaces.ModelId) func() (svc interfaces.GrpcClientModelBaseService, err error) {
return func() (svc interfaces.GrpcClientModelBaseService, err error) {
return NewBaseServiceDelegate(WithBaseServiceModelId(id))
}
}

329
core/models/client/model_delegate.go
View File

@@ -1,329 +0,0 @@
package client
import (
"encoding/json"
config2 "github.com/crawlab-team/crawlab/core/config"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/grpc/client"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/utils"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/crawlab/trace"
"github.com/spf13/viper"
)
func NewModelDelegate(doc interfaces.Model, opts ...ModelDelegateOption) interfaces.GrpcClientModelDelegate {
switch doc.(type) {
case *models.Artifact:
return newModelDelegate(interfaces.ModelIdArtifact, doc, opts...)
case *models.Tag:
return newModelDelegate(interfaces.ModelIdTag, doc, opts...)
case *models.Node:
return newModelDelegate(interfaces.ModelIdNode, doc, opts...)
case *models.Project:
return newModelDelegate(interfaces.ModelIdProject, doc, opts...)
case *models.Spider:
return newModelDelegate(interfaces.ModelIdSpider, doc, opts...)
case *models.Task:
return newModelDelegate(interfaces.ModelIdTask, doc, opts...)
case *models.Job:
return newModelDelegate(interfaces.ModelIdJob, doc, opts...)
case *models.Schedule:
return newModelDelegate(interfaces.ModelIdSchedule, doc, opts...)
case *models.User:
return newModelDelegate(interfaces.ModelIdUser, doc, opts...)
case *models.Setting:
return newModelDelegate(interfaces.ModelIdSetting, doc, opts...)
case *models.Token:
return newModelDelegate(interfaces.ModelIdToken, doc, opts...)
case *models.Variable:
return newModelDelegate(interfaces.ModelIdVariable, doc, opts...)
case *models.TaskQueueItem:
return newModelDelegate(interfaces.ModelIdTaskQueue, doc, opts...)
case *models.TaskStat:
return newModelDelegate(interfaces.ModelIdTaskStat, doc, opts...)
case *models.SpiderStat:
return newModelDelegate(interfaces.ModelIdSpiderStat, doc, opts...)
case *models.DataSource:
return newModelDelegate(interfaces.ModelIdDataSource, doc, opts...)
case *models.DataCollection:
return newModelDelegate(interfaces.ModelIdDataCollection, doc, opts...)
case *models.Result:
return newModelDelegate(interfaces.ModelIdResult, doc, opts...)
case *models.Password:
return newModelDelegate(interfaces.ModelIdPassword, doc, opts...)
case *models.ExtraValue:
return newModelDelegate(interfaces.ModelIdExtraValue, doc, opts...)
case *models.Git:
return newModelDelegate(interfaces.ModelIdGit, doc, opts...)
case *models.UserRole:
return newModelDelegate(interfaces.ModelIdUserRole, doc, opts...)
case *models.Permission:
return newModelDelegate(interfaces.ModelIdPermission, doc, opts...)
case *models.RolePermission:
return newModelDelegate(interfaces.ModelIdRolePermission, doc, opts...)
case *models.Environment:
return newModelDelegate(interfaces.ModelIdEnvironment, doc, opts...)
case *models.DependencySetting:
return newModelDelegate(interfaces.ModelIdDependencySetting, doc, opts...)
default:
_ = trace.TraceError(errors.ErrorModelInvalidType)
return nil
}
}
func newModelDelegate(id interfaces.ModelId, doc interfaces.Model, opts ...ModelDelegateOption) interfaces.GrpcClientModelDelegate {
var err error
// collection name
colName := models.GetModelColName(id)
// model delegate
d := &ModelDelegate{
id: id,
colName: colName,
doc: doc,
cfgPath: config2.GetConfigPath(),
a: &models.Artifact{
Col: colName,
},
}
// config path
if viper.GetString("config.path") != "" {
d.cfgPath = viper.GetString("config.path")
}
// apply options
for _, opt := range opts {
opt(d)
}
// grpc client
d.c, err = client.GetClient()
if err != nil {
trace.PrintError(errors.ErrorModelInvalidType)
return nil
}
if !d.c.IsStarted() {
if err := d.c.Start(); err != nil {
trace.PrintError(err)
return nil
}
} else if d.c.IsClosed() {
if err := d.c.Restart(); err != nil {
trace.PrintError(err)
return nil
}
}
return d
}
type ModelDelegate struct {
// settings
cfgPath string
// internals
id interfaces.ModelId
colName string
c interfaces.GrpcClient
doc interfaces.Model
a interfaces.ModelArtifact
}
func (d *ModelDelegate) Add() (err error) {
return d.do(interfaces.ModelDelegateMethodAdd)
}
func (d *ModelDelegate) Save() (err error) {
return d.do(interfaces.ModelDelegateMethodSave)
}
func (d *ModelDelegate) Delete() (err error) {
return d.do(interfaces.ModelDelegateMethodDelete)
}
func (d *ModelDelegate) GetArtifact() (res interfaces.ModelArtifact, err error) {
if err := d.do(interfaces.ModelDelegateMethodGetArtifact); err != nil {
return nil, err
}
return d.a, nil
}
func (d *ModelDelegate) GetModel() (res interfaces.Model) {
return d.doc
}
func (d *ModelDelegate) Refresh() (err error) {
return d.refresh()
}
func (d *ModelDelegate) GetConfigPath() (path string) {
return d.cfgPath
}
func (d *ModelDelegate) SetConfigPath(path string) {
d.cfgPath = path
}
func (d *ModelDelegate) Close() (err error) {
return d.c.Stop()
}
func (d *ModelDelegate) ToBytes(m interface{}) (bytes []byte, err error) {
if m != nil {
return utils.JsonToBytes(m)
}
return json.Marshal(d.doc)
}
func (d *ModelDelegate) do(method interfaces.ModelDelegateMethod) (err error) {
switch method {
case interfaces.ModelDelegateMethodAdd:
err = d.add()
case interfaces.ModelDelegateMethodSave:
err = d.save()
case interfaces.ModelDelegateMethodDelete:
err = d.delete()
case interfaces.ModelDelegateMethodGetArtifact, interfaces.ModelDelegateMethodRefresh:
return d.refresh()
default:
return trace.TraceError(errors.ErrorModelInvalidType)
}
if err != nil {
return err
}
return nil
}
func (d *ModelDelegate) add() (err error) {
ctx, cancel := d.c.Context()
defer cancel()
method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodAdd)
res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
ModelId: d.id,
Method: method,
Data: d.mustGetData(),
}))
if err != nil {
return trace.TraceError(err)
}
if err := d.deserialize(res, method); err != nil {
return err
}
return d.refreshArtifact()
}
func (d *ModelDelegate) save() (err error) {
ctx, cancel := d.c.Context()
defer cancel()
method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodSave)
res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
ModelId: d.id,
Method: method,
Data: d.mustGetData(),
}))
if err != nil {
return trace.TraceError(err)
}
if err := d.deserialize(res, method); err != nil {
return err
}
return d.refreshArtifact()
}
func (d *ModelDelegate) delete() (err error) {
ctx, cancel := d.c.Context()
defer cancel()
method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodDelete)
res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
ModelId: d.id,
Method: method,
Data: d.mustGetData(),
}))
if err != nil {
return trace.TraceError(err)
}
if err := d.deserialize(res, method); err != nil {
return err
}
return d.refreshArtifact()
}
func (d *ModelDelegate) refresh() (err error) {
ctx, cancel := d.c.Context()
defer cancel()
method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodRefresh)
res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
ModelId: d.id,
Method: method,
Data: d.mustGetData(),
}))
if err != nil {
return trace.TraceError(err)
}
if err := d.deserialize(res, method); err != nil {
return err
}
return nil
}
func (d *ModelDelegate) refreshArtifact() (err error) {
_, err = d.getArtifact()
return err
}
func (d *ModelDelegate) getArtifact() (res2 interfaces.ModelArtifact, err error) {
ctx, cancel := d.c.Context()
defer cancel()
method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodGetArtifact)
res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{
ModelId: d.id,
Method: method,
Data: d.mustGetData(),
}))
if err != nil {
return nil, err
}
if err := d.deserialize(res, method); err != nil {
return nil, err
}
return d.a, nil
}
func (d *ModelDelegate) mustGetData() (data []byte) {
data, err := d.getData()
if err != nil {
panic(err)
}
return data
}
func (d *ModelDelegate) getData() (data []byte, err error) {
return json.Marshal(d.doc)
}
func (d *ModelDelegate) deserialize(res *grpc.Response, method interfaces.ModelDelegateMethod) (err error) {
if method == interfaces.ModelDelegateMethodGetArtifact {
res, err := NewBasicBinder(interfaces.ModelIdArtifact, res).Bind()
if err != nil {
return err
}
a, ok := res.(interfaces.ModelArtifact)
if !ok {
return trace.TraceError(errors.ErrorModelInvalidType)
}
d.a = a
} else {
d.doc, err = NewBasicBinder(d.id, res).Bind()
if err != nil {
return err
}
}
return nil
}

70
core/models/client/model_environment_service.go
View File

@@ -1,70 +0,0 @@
package client
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type EnvironmentServiceDelegate struct {
interfaces.GrpcClientModelBaseService
}
func (svc *EnvironmentServiceDelegate) GetEnvironmentById(id primitive.ObjectID) (e interfaces.Environment, err error) {
res, err := svc.GetById(id)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Environment)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *EnvironmentServiceDelegate) GetEnvironment(query bson.M, opts *mongo.FindOptions) (e interfaces.Environment, err error) {
res, err := svc.Get(query, opts)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Environment)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *EnvironmentServiceDelegate) GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Environment, err error) {
list, err := svc.GetList(query, opts)
if err != nil {
return nil, err
}
for _, item := range list.GetModels() {
s, ok := item.(interfaces.Environment)
if !ok {
return nil, errors.ErrorModelInvalidType
}
res = append(res, s)
}
return res, nil
}
func NewEnvironmentServiceDelegate() (svc2 interfaces.GrpcClientModelEnvironmentService, err error) {
var opts []ModelBaseServiceDelegateOption
// apply options
opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdEnvironment))
// base service
baseSvc, err := NewBaseServiceDelegate(opts...)
if err != nil {
return nil, err
}
// service
svc := &EnvironmentServiceDelegate{baseSvc}
return svc, nil
}

37
core/models/client/model_node_delegate.go
View File

@@ -1,37 +0,0 @@
package client
import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/interfaces"
"time"
)
type ModelNodeDelegate struct {
n interfaces.Node
interfaces.GrpcClientModelDelegate
}
func (d *ModelNodeDelegate) UpdateStatus(active bool, activeTs *time.Time, status string) (err error) {
d.n.SetActive(active)
if activeTs != nil {
d.n.SetActiveTs(*activeTs)
}
d.n.SetStatus(status)
return d.Save()
}
func (d *ModelNodeDelegate) UpdateStatusOnline() (err error) {
now := time.Now()
return d.UpdateStatus(true, &now, constants.NodeStatusOnline)
}
func (d *ModelNodeDelegate) UpdateStatusOffline() (err error) {
return d.UpdateStatus(false, nil, constants.NodeStatusOffline)
}
func NewModelNodeDelegate(n interfaces.Node) interfaces.ModelNodeDelegate {
return &ModelNodeDelegate{
n: n,
GrpcClientModelDelegate: NewModelDelegate(n),
}
}

74
core/models/client/model_node_service.go
View File

@@ -1,74 +0,0 @@
package client
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type NodeServiceDelegate struct {
interfaces.GrpcClientModelBaseService
}
func (svc *NodeServiceDelegate) GetNodeById(id primitive.ObjectID) (n interfaces.Node, err error) {
res, err := svc.GetById(id)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Node)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *NodeServiceDelegate) GetNode(query bson.M, opts *mongo.FindOptions) (n interfaces.Node, err error) {
res, err := svc.Get(query, opts)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Node)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *NodeServiceDelegate) GetNodeByKey(key string) (n interfaces.Node, err error) {
return svc.GetNode(bson.M{"key": key}, nil)
}
func (svc *NodeServiceDelegate) GetNodeList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Node, err error) {
list, err := svc.GetList(query, opts)
if err != nil {
return nil, err
}
for _, item := range list.GetModels() {
s, ok := item.(interfaces.Node)
if !ok {
return nil, errors.ErrorModelInvalidType
}
res = append(res, s)
}
return res, nil
}
func NewNodeServiceDelegate() (svc2 interfaces.GrpcClientModelNodeService, err error) {
var opts []ModelBaseServiceDelegateOption
// apply options
opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdNode))
// base service
baseSvc, err := NewBaseServiceDelegate(opts...)
if err != nil {
return nil, err
}
// service
svc := &NodeServiceDelegate{baseSvc}
return svc, nil
}

48
core/models/client/model_service.go
View File

@@ -1,48 +0,0 @@
package client
import (
config2 "github.com/crawlab-team/crawlab/core/config"
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/interfaces"
)
type ServiceDelegate struct {
// settings
cfgPath string
// internals
c interfaces.GrpcClient
}
func (d *ServiceDelegate) GetConfigPath() string {
return d.cfgPath
}
func (d *ServiceDelegate) SetConfigPath(path string) {
d.cfgPath = path
}
func (d *ServiceDelegate) NewBaseServiceDelegate(id interfaces.ModelId) (svc interfaces.GrpcClientModelBaseService, err error) {
var opts []ModelBaseServiceDelegateOption
opts = append(opts, WithBaseServiceModelId(id))
if d.cfgPath != "" {
opts = append(opts, WithBaseServiceConfigPath(d.cfgPath))
}
return NewBaseServiceDelegate(opts...)
}
func NewServiceDelegate() (svc2 interfaces.GrpcClientModelService, err error) {
// service delegate
svc := &ServiceDelegate{
cfgPath: config2.GetConfigPath(),
}
// dependency injection
if err := container.GetContainer().Invoke(func(client interfaces.GrpcClient) {
svc.c = client
}); err != nil {
return nil, err
}
return svc, nil
}

70
core/models/client/model_spider_service.go
View File

@@ -1,70 +0,0 @@
package client
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type SpiderServiceDelegate struct {
interfaces.GrpcClientModelBaseService
}
func (svc *SpiderServiceDelegate) GetSpiderById(id primitive.ObjectID) (s interfaces.Spider, err error) {
res, err := svc.GetById(id)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Spider)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *SpiderServiceDelegate) GetSpider(query bson.M, opts *mongo.FindOptions) (s interfaces.Spider, err error) {
res, err := svc.Get(query, opts)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Spider)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *SpiderServiceDelegate) GetSpiderList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Spider, err error) {
list, err := svc.GetList(query, opts)
if err != nil {
return nil, err
}
for _, item := range list.GetModels() {
s, ok := item.(interfaces.Spider)
if !ok {
return nil, errors.ErrorModelInvalidType
}
res = append(res, s)
}
return res, nil
}
func NewSpiderServiceDelegate() (svc2 interfaces.GrpcClientModelSpiderService, err error) {
var opts []ModelBaseServiceDelegateOption
// apply options
opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdSpider))
// base service
baseSvc, err := NewBaseServiceDelegate(opts...)
if err != nil {
return nil, err
}
// service
svc := &SpiderServiceDelegate{baseSvc}
return svc, nil
}

70
core/models/client/model_task_service.go
View File

@@ -1,70 +0,0 @@
package client
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type TaskServiceDelegate struct {
interfaces.GrpcClientModelBaseService
}
func (svc *TaskServiceDelegate) GetTaskById(id primitive.ObjectID) (t interfaces.Task, err error) {
res, err := svc.GetById(id)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Task)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *TaskServiceDelegate) GetTask(query bson.M, opts *mongo.FindOptions) (t interfaces.Task, err error) {
res, err := svc.Get(query, opts)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.Task)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *TaskServiceDelegate) GetTaskList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Task, err error) {
list, err := svc.GetList(query, opts)
if err != nil {
return nil, err
}
for _, item := range list.GetModels() {
s, ok := item.(interfaces.Task)
if !ok {
return nil, errors.ErrorModelInvalidType
}
res = append(res, s)
}
return res, nil
}
func NewTaskServiceDelegate() (svc2 interfaces.GrpcClientModelTaskService, err error) {
var opts []ModelBaseServiceDelegateOption
// apply options
opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdTask))
// base service
baseSvc, err := NewBaseServiceDelegate(opts...)
if err != nil {
return nil, err
}
// service
svc := &TaskServiceDelegate{baseSvc}
return svc, nil
}

70
core/models/client/model_task_stat_service.go
View File

@@ -1,70 +0,0 @@
package client
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type TaskStatServiceDelegate struct {
interfaces.GrpcClientModelBaseService
}
func (svc *TaskStatServiceDelegate) GetTaskStatById(id primitive.ObjectID) (t interfaces.TaskStat, err error) {
res, err := svc.GetById(id)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.TaskStat)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *TaskStatServiceDelegate) GetTaskStat(query bson.M, opts *mongo.FindOptions) (t interfaces.TaskStat, err error) {
res, err := svc.Get(query, opts)
if err != nil {
return nil, err
}
s, ok := res.(interfaces.TaskStat)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return s, nil
}
func (svc *TaskStatServiceDelegate) GetTaskStatList(query bson.M, opts *mongo.FindOptions) (res []interfaces.TaskStat, err error) {
list, err := svc.GetList(query, opts)
if err != nil {
return nil, err
}
for _, item := range list.GetModels() {
s, ok := item.(interfaces.TaskStat)
if !ok {
return nil, errors.ErrorModelInvalidType
}
res = append(res, s)
}
return res, nil
}
func NewTaskStatServiceDelegate() (svc2 interfaces.GrpcClientModelTaskStatService, err error) {
var opts []ModelBaseServiceDelegateOption
// apply options
opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdTaskStat))
// base service
baseSvc, err := NewBaseServiceDelegate(opts...)
if err != nil {
return nil, err
}
// service
svc := &TaskStatServiceDelegate{baseSvc}
return svc, nil
}

33
core/models/client/options.go
View File

@@ -1,33 +0,0 @@
package client
import "github.com/crawlab-team/crawlab/core/interfaces"
type ModelDelegateOption func(delegate interfaces.GrpcClientModelDelegate)
func WithDelegateConfigPath(path string) ModelDelegateOption {
return func(d interfaces.GrpcClientModelDelegate) {
d.SetConfigPath(path)
}
}
type ModelServiceDelegateOption func(delegate interfaces.GrpcClientModelService)
func WithServiceConfigPath(path string) ModelServiceDelegateOption {
return func(d interfaces.GrpcClientModelService) {
d.SetConfigPath(path)
}
}
type ModelBaseServiceDelegateOption func(delegate interfaces.GrpcClientModelBaseService)
func WithBaseServiceModelId(id interfaces.ModelId) ModelBaseServiceDelegateOption {
return func(d interfaces.GrpcClientModelBaseService) {
d.SetModelId(id)
}
}
func WithBaseServiceConfigPath(path string) ModelBaseServiceDelegateOption {
return func(d interfaces.GrpcClientModelBaseService) {
d.SetConfigPath(path)
}
}

24
core/models/config_spider/common.go
View File

@@ -1,24 +0,0 @@
package config_spider
import "github.com/crawlab-team/crawlab/core/entity"
func GetAllFields(data entity.ConfigSpiderData) []entity.Field {
var fields []entity.Field
for _, stage := range data.Stages {
fields = append(fields, stage.Fields...)
}
return fields
}
func GetStartStageName(data entity.ConfigSpiderData) string {
// 如果 start_stage 设置了且在 stages 里，则返回
if data.StartStage != "" {
return data.StartStage
}
// 否则返回第一个 stage
for _, stage := range data.Stages {
return stage.Name
}
return ""
}

263
core/models/config_spider/scrapy.go
View File

@@ -1,263 +0,0 @@
package config_spider
//import (
// "errors"
// "fmt"
// "github.com/crawlab-team/crawlab/core/constants"
// "github.com/crawlab-team/crawlab/core/entity"
// "github.com/crawlab-team/crawlab/core/models"
// "github.com/crawlab-team/crawlab/core/utils"
// "path/filepath"
//)
//
//type ScrapyGenerator struct {
// Spider model.Spider
// ConfigData entity.ConfigSpiderData
//}
//
//// 生成爬虫文件
//func (g ScrapyGenerator) Generate() error {
// // 生成 items.py
// if err := g.ProcessItems(); err != nil {
// return err
// }
//
// // 生成 spider.py
// if err := g.ProcessSpider(); err != nil {
// return err
// }
// return nil
//}
//
//// 生成 items.py
//func (g ScrapyGenerator) ProcessItems() error {
// // 待处理文件名
// src := g.Spider.Src
// filePath := filepath.Join(src, "config_spider", "items.py")
//
// // 获取所有字段
// fields := g.GetAllFields()
//
// // 字段名列表（包含默认字段名）
// fieldNames := []string{
// "_id",
// "task_id",
// "ts",
// }
//
// // 加入字段
// for _, field := range fields {
// fieldNames = append(fieldNames, field.Name)
// }
//
// // 将字段名转化为python代码
// str := ""
// for _, fieldName := range fieldNames {
// line := g.PadCode(fmt.Sprintf("%s = scrapy.Field()", fieldName), 1)
// str += line
// }
//
// // 将占位符替换为代码
// if err := utils.SetFileVariable(filePath, constants.AnchorItems, str); err != nil {
// return err
// }
//
// return nil
//}
//
//// 生成 spider.py
//func (g ScrapyGenerator) ProcessSpider() error {
// // 待处理文件名
// src := g.Spider.Src
// filePath := filepath.Join(src, "config_spider", "spiders", "spider.py")
//
// // 替换 start_stage
// if err := utils.SetFileVariable(filePath, constants.AnchorStartStage, "parse_"+GetStartStageName(g.ConfigData)); err != nil {
// return err
// }
//
// // 替换 start_url
// if err := utils.SetFileVariable(filePath, constants.AnchorStartUrl, g.ConfigData.StartUrl); err != nil {
// return err
// }
//
// // 替换 parsers
// strParser := ""
// for _, stage := range g.ConfigData.Stages {
// stageName := stage.Name
// stageStr := g.GetParserString(stageName, stage)
// strParser += stageStr
// }
// if err := utils.SetFileVariable(filePath, constants.AnchorParsers, strParser); err != nil {
// return err
// }
//
// return nil
//}
//
//func (g ScrapyGenerator) GetParserString(stageName string, stage entity.Stage) string {
// // 构造函数定义行
// strDef := g.PadCode(fmt.Sprintf("def parse_%s(self, response):", stageName), 1)
//
// strParse := ""
// if stage.IsList {
// // 列表逻辑
// strParse = g.GetListParserString(stageName, stage)
// } else {
// // 非列表逻辑
// strParse = g.GetNonListParserString(stageName, stage)
// }
//
// // 构造
// str := fmt.Sprintf(`%s%s`, strDef, strParse)
//
// return str
//}
//
//func (g ScrapyGenerator) PadCode(str string, num int) string {
// res := ""
// for i := 0; i < num; i++ {
// res += " "
// }
// res += str
// res += "\n"
// return res
//}
//
//func (g ScrapyGenerator) GetNonListParserString(stageName string, stage entity.Stage) string {
// str := ""
//
// // 获取或构造item
// str += g.PadCode("item = Item() if response.meta.get('item') is None else response.meta.get('item')", 2)
//
// // 遍历字段列表
// for _, f := range stage.Fields {
// line := fmt.Sprintf(`item['%s'] = response.%s.extract_first()`, f.Name, g.GetExtractStringFromField(f))
// line = g.PadCode(line, 2)
// str += line
// }
//
// // next stage 字段
// if f, err := g.GetNextStageField(stage); err == nil {
// // 如果找到 next stage 字段，进行下一个回调
// str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url="get_real_url(response, item['%s'])", callback=self.parse_%s, meta={'item': item})`, f.Name, f.NextStage), 2)
// } else {
// // 如果没找到 next stage 字段，返回 item
// str += g.PadCode(fmt.Sprintf(`yield item`), 2)
// }
//
// // 加入末尾换行
// str += g.PadCode("", 0)
//
// return str
//}
//
//func (g ScrapyGenerator) GetListParserString(stageName string, stage entity.Stage) string {
// str := ""
//
// // 获取前一个 stage 的 item
// str += g.PadCode(`prev_item = response.meta.get('item')`, 2)
//
// // for 循环遍历列表
// str += g.PadCode(fmt.Sprintf(`for elem in response.%s:`, g.GetListString(stage)), 2)
//
// // 构造item
// str += g.PadCode(`item = Item()`, 3)
//
// // 遍历字段列表
// for _, f := range stage.Fields {
// line := fmt.Sprintf(`item['%s'] = elem.%s.extract_first()`, f.Name, g.GetExtractStringFromField(f))
// line = g.PadCode(line, 3)
// str += line
// }
//
// // 把前一个 stage 的 item 值赋给当前 item
// str += g.PadCode(`if prev_item is not None:`, 3)
// str += g.PadCode(`for key, value in prev_item.items():`, 4)
// str += g.PadCode(`item[key] = value`, 5)
//
// // next stage 字段
// if f, err := g.GetNextStageField(stage); err == nil {
// // 如果 url 为空，则不进入下一个 stage
// str += g.PadCode(fmt.Sprintf(`if not item['%s']:`, f.Name), 3)
// str += g.PadCode(`continue`, 4)
//
// // 如果找到 next stage 字段，进行下一个回调
// str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, item['%s']), callback=self.parse_%s, meta={'item': item})`, f.Name, f.NextStage), 3)
// } else {
// // 如果没找到 next stage 字段，返回 item
// str += g.PadCode(fmt.Sprintf(`yield item`), 3)
// }
//
// // 分页
// if stage.PageCss != "" || stage.PageXpath != "" {
// str += g.PadCode(fmt.Sprintf(`next_url = response.%s.extract_first()`, g.GetExtractStringFromStage(stage)), 2)
// str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, next_url), callback=self.parse_%s, meta={'item': prev_item})`, stageName), 2)
// }
//
// // 加入末尾换行
// str += g.PadCode("", 0)
//
// return str
//}
//
//// 获取所有字段
//func (g ScrapyGenerator) GetAllFields() []entity.Field {
// return GetAllFields(g.ConfigData)
//}
//
//// 获取包含 next stage 的字段
//func (g ScrapyGenerator) GetNextStageField(stage entity.Stage) (entity.Field, error) {
// for _, field := range stage.Fields {
// if field.NextStage != "" {
// return field, nil
// }
// }
// return entity.Field{}, errors.New("cannot find next stage field")
//}
//
//func (g ScrapyGenerator) GetExtractStringFromField(f entity.Field) string {
// if f.Css != "" {
// // 如果为CSS
// if f.Attr == "" {
// // 文本
// return fmt.Sprintf(`css('%s::text')`, f.Css)
// } else {
// // 属性
// return fmt.Sprintf(`css('%s::attr("%s")')`, f.Css, f.Attr)
// }
// } else {
// // 如果为XPath
// if f.Attr == "" {
// // 文本
// return fmt.Sprintf(`xpath('string(%s)')`, f.Xpath)
// } else {
// // 属性
// return fmt.Sprintf(`xpath('%s/@%s')`, f.Xpath, f.Attr)
// }
// }
//}
//
//func (g ScrapyGenerator) GetExtractStringFromStage(stage entity.Stage) string {
// // 分页元素属性，默认为 href
// pageAttr := "href"
// if stage.PageAttr != "" {
// pageAttr = stage.PageAttr
// }
//
// if stage.PageCss != "" {
// // 如果为CSS
// return fmt.Sprintf(`css('%s::attr("%s")')`, stage.PageCss, pageAttr)
// } else {
// // 如果为XPath
// return fmt.Sprintf(`xpath('%s/@%s')`, stage.PageXpath, pageAttr)
// }
//}
//
//func (g ScrapyGenerator) GetListString(stage entity.Stage) string {
// if stage.ListCss != "" {
// return fmt.Sprintf(`css('%s')`, stage.ListCss)
// } else {
// return fmt.Sprintf(`xpath('%s')`, stage.ListXpath)
// }
//}

25
core/models/delegate/base_test.go
View File

@@ -1,25 +0,0 @@
package delegate_test
import (
"context"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"testing"
"time"
)
func SetupTest(t *testing.T) {
CleanupTest()
t.Cleanup(CleanupTest)
}
func CleanupTest() {
db := mongo.GetMongoDb("")
names, _ := db.ListCollectionNames(context.Background(), bson.M{})
for _, n := range names {
_, _ = db.Collection(n).DeleteMany(context.Background(), bson.M{})
}
// avoid caching
time.Sleep(200 * time.Millisecond)
}

362
core/models/delegate/model.go
View File

@@ -1,362 +0,0 @@
package delegate
import (
"encoding/json"
errors2 "github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/event"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/errors"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/trace"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
mongo2 "go.mongodb.org/mongo-driver/mongo"
"reflect"
"time"
)
func NewModelDelegate(doc interfaces.Model, args ...interface{}) interfaces.ModelDelegate {
switch doc.(type) {
case *models.Artifact:
return newModelDelegate(interfaces.ModelIdArtifact, doc, args...)
case *models.Tag:
return newModelDelegate(interfaces.ModelIdTag, doc, args...)
case *models.Node:
return newModelDelegate(interfaces.ModelIdNode, doc, args...)
case *models.Project:
return newModelDelegate(interfaces.ModelIdProject, doc, args...)
case *models.Spider:
return newModelDelegate(interfaces.ModelIdSpider, doc, args...)
case *models.Task:
return newModelDelegate(interfaces.ModelIdTask, doc, args...)
case *models.Job:
return newModelDelegate(interfaces.ModelIdJob, doc, args...)
case *models.Schedule:
return newModelDelegate(interfaces.ModelIdSchedule, doc, args...)
case *models.User:
return newModelDelegate(interfaces.ModelIdUser, doc, args...)
case *models.Setting:
return newModelDelegate(interfaces.ModelIdSetting, doc, args...)
case *models.Token:
return newModelDelegate(interfaces.ModelIdToken, doc, args...)
case *models.Variable:
return newModelDelegate(interfaces.ModelIdVariable, doc, args...)
case *models.TaskQueueItem:
return newModelDelegate(interfaces.ModelIdTaskQueue, doc, args...)
case *models.TaskStat:
return newModelDelegate(interfaces.ModelIdTaskStat, doc, args...)
case *models.SpiderStat:
return newModelDelegate(interfaces.ModelIdSpiderStat, doc, args...)
case *models.DataSource:
return newModelDelegate(interfaces.ModelIdDataSource, doc, args...)
case *models.DataCollection:
return newModelDelegate(interfaces.ModelIdDataCollection, doc, args...)
case *models.Result:
return newModelDelegate(interfaces.ModelIdResult, doc, args...)
case *models.Password:
return newModelDelegate(interfaces.ModelIdPassword, doc, args...)
case *models.ExtraValue:
return newModelDelegate(interfaces.ModelIdExtraValue, doc, args...)
case *models.Git:
return newModelDelegate(interfaces.ModelIdGit, doc, args...)
case *models.Role:
return newModelDelegate(interfaces.ModelIdRole, doc, args...)
case *models.UserRole:
return newModelDelegate(interfaces.ModelIdUserRole, doc, args...)
case *models.Permission:
return newModelDelegate(interfaces.ModelIdPermission, doc, args...)
case *models.RolePermission:
return newModelDelegate(interfaces.ModelIdRolePermission, doc, args...)
case *models.Environment:
return newModelDelegate(interfaces.ModelIdEnvironment, doc, args...)
case *models.DependencySetting:
return newModelDelegate(interfaces.ModelIdDependencySetting, doc, args...)
default:
_ = trace.TraceError(errors2.ErrorModelInvalidType)
return nil
}
}
func newModelDelegate(id interfaces.ModelId, doc interfaces.Model, args ...interface{}) interfaces.ModelDelegate {
// user
u := utils.GetUserFromArgs(args...)
// collection name
colName := models.GetModelColName(id)
// model delegate
d := &ModelDelegate{
id: id,
colName: colName,
doc: doc,
a: &models.Artifact{
Col: colName,
},
u: u,
}
return d
}
type ModelDelegate struct {
id interfaces.ModelId
colName string
doc interfaces.Model // doc to delegate
cd bson.M // current doc
od bson.M // original doc
a interfaces.ModelArtifact // artifact
u interfaces.User // user
}
// Add model
func (d *ModelDelegate) Add() (err error) {
return d.do(interfaces.ModelDelegateMethodAdd)
}
// Save model
func (d *ModelDelegate) Save() (err error) {
return d.do(interfaces.ModelDelegateMethodSave)
}
// Delete model
func (d *ModelDelegate) Delete() (err error) {
return d.do(interfaces.ModelDelegateMethodDelete)
}
// GetArtifact refresh artifact and return it
func (d *ModelDelegate) GetArtifact() (res interfaces.ModelArtifact, err error) {
if err := d.do(interfaces.ModelDelegateMethodGetArtifact); err != nil {
return nil, err
}
return d.a, nil
}
// Refresh model
func (d *ModelDelegate) Refresh() (err error) {
return d.refresh()
}
// GetModel return model
func (d *ModelDelegate) GetModel() (res interfaces.Model) {
return d.doc
}
func (d *ModelDelegate) ToBytes(m interface{}) (bytes []byte, err error) {
if m != nil {
return utils.JsonToBytes(m)
}
return json.Marshal(d.doc)
}
// do action given the model delegate method
func (d *ModelDelegate) do(method interfaces.ModelDelegateMethod) (err error) {
switch method {
case interfaces.ModelDelegateMethodAdd:
err = d.add()
case interfaces.ModelDelegateMethodSave:
err = d.save()
case interfaces.ModelDelegateMethodDelete:
err = d.delete()
case interfaces.ModelDelegateMethodGetArtifact, interfaces.ModelDelegateMethodRefresh:
err = d.refresh()
default:
return trace.TraceError(errors2.ErrorModelInvalidType)
}
if err != nil {
return err
}
// trigger event
eventName := GetEventName(d, method)
go event.SendEvent(eventName, d.doc)
return nil
}
// add model
func (d *ModelDelegate) add() (err error) {
if d.doc == nil {
return trace.TraceError(errors.ErrMissingValue)
}
if d.doc.GetId().IsZero() {
d.doc.SetId(primitive.NewObjectID())
}
col := mongo.GetMongoCol(d.colName)
if _, err = col.Insert(d.doc); err != nil {
return trace.TraceError(err)
}
if err := d.upsertArtifact(); err != nil {
return trace.TraceError(err)
}
return d.refresh()
}
// save model
func (d *ModelDelegate) save() (err error) {
// validate
if d.doc == nil || d.doc.GetId().IsZero() {
return trace.TraceError(errors.ErrMissingValue)
}
// collection
col := mongo.GetMongoCol(d.colName)
// current doc
docData, err := bson.Marshal(d.doc)
if err != nil {
trace.PrintError(err)
} else {
if err := bson.Unmarshal(docData, &d.cd); err != nil {
trace.PrintError(err)
}
}
// original doc
if err := col.FindId(d.doc.GetId()).One(&d.od); err != nil {
trace.PrintError(err)
}
// replace
if err := col.ReplaceId(d.doc.GetId(), d.doc); err != nil {
return trace.TraceError(err)
}
// upsert artifact
if err := d.upsertArtifact(); err != nil {
return trace.TraceError(err)
}
return d.refresh()
}
// delete model
func (d *ModelDelegate) delete() (err error) {
if d.doc.GetId().IsZero() {
return trace.TraceError(errors2.ErrorModelMissingId)
}
col := mongo.GetMongoCol(d.colName)
if err := col.FindId(d.doc.GetId()).One(d.doc); err != nil {
return trace.TraceError(err)
}
if err := col.DeleteId(d.doc.GetId()); err != nil {
return trace.TraceError(err)
}
return d.deleteArtifact()
}
// refresh model and artifact
func (d *ModelDelegate) refresh() (err error) {
if d.doc.GetId().IsZero() {
return trace.TraceError(errors2.ErrorModelMissingId)
}
col := mongo.GetMongoCol(d.colName)
fr := col.FindId(d.doc.GetId())
if err := fr.One(d.doc); err != nil {
return trace.TraceError(err)
}
return d.refreshArtifact()
}
// refresh artifact
func (d *ModelDelegate) refreshArtifact() (err error) {
if d.doc.GetId().IsZero() {
return trace.TraceError(errors2.ErrorModelMissingId)
}
col := mongo.GetMongoCol(interfaces.ModelColNameArtifact)
if err := col.FindId(d.doc.GetId()).One(d.a); err != nil {
return trace.TraceError(err)
}
return nil
}
// upsertArtifact
func (d *ModelDelegate) upsertArtifact() (err error) {
// skip
if d._skip() {
return nil
}
// validate id
if d.doc.GetId().IsZero() {
return trace.TraceError(errors.ErrMissingValue)
}
// mongo collection
col := mongo.GetMongoCol(interfaces.ModelColNameArtifact)
// assign id to artifact
d.a.SetId(d.doc.GetId())
// attempt to find artifact
if err := col.FindId(d.doc.GetId()).One(d.a); err != nil {
if err == mongo2.ErrNoDocuments {
// new artifact
d.a.GetSys().SetCreateTs(time.Now())
d.a.GetSys().SetUpdateTs(time.Now())
if d.u != nil && !reflect.ValueOf(d.u).IsZero() {
d.a.GetSys().SetCreateUid(d.u.GetId())
d.a.GetSys().SetUpdateUid(d.u.GetId())
}
_, err = col.Insert(d.a)
if err != nil {
return trace.TraceError(err)
}
return nil
} else {
// error
return trace.TraceError(err)
}
}
// existing artifact
d.a.GetSys().SetUpdateTs(time.Now())
if d.u != nil {
d.a.GetSys().SetUpdateUid(d.u.GetId())
}
// save new artifact
return col.ReplaceId(d.a.GetId(), d.a)
}
// deleteArtifact
func (d *ModelDelegate) deleteArtifact() (err error) {
// skip
if d._skip() {
return nil
}
if d.doc.GetId().IsZero() {
return trace.TraceError(errors.ErrMissingValue)
}
col := mongo.GetMongoCol(interfaces.ModelColNameArtifact)
d.a.SetId(d.doc.GetId())
d.a.SetObj(d.doc)
d.a.SetDel(true)
d.a.GetSys().SetDeleteTs(time.Now())
if d.u != nil {
d.a.GetSys().SetDeleteUid(d.u.GetId())
}
return col.ReplaceId(d.doc.GetId(), d.a)
}
func (d *ModelDelegate) hasChange() (ok bool) {
return !utils.BsonMEqual(d.cd, d.od)
}
func (d *ModelDelegate) _skip() (ok bool) {
switch d.id {
case
interfaces.ModelIdArtifact,
interfaces.ModelIdTaskQueue,
interfaces.ModelIdTaskStat,
interfaces.ModelIdSpiderStat,
interfaces.ModelIdResult,
interfaces.ModelIdPassword:
return true
default:
return false
}
}

37
core/models/delegate/model_node.go
View File

@@ -1,37 +0,0 @@
package delegate
import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/interfaces"
"time"
)
type ModelNodeDelegate struct {
n interfaces.Node
interfaces.ModelDelegate
}
func (d *ModelNodeDelegate) UpdateStatus(active bool, activeTs *time.Time, status string) (err error) {
d.n.SetActive(active)
if activeTs != nil {
d.n.SetActiveTs(*activeTs)
}
d.n.SetStatus(status)
return d.Save()
}
func (d *ModelNodeDelegate) UpdateStatusOnline() (err error) {
now := time.Now()
return d.UpdateStatus(true, &now, constants.NodeStatusOnline)
}
func (d *ModelNodeDelegate) UpdateStatusOffline() (err error) {
return d.UpdateStatus(false, nil, constants.NodeStatusOffline)
}
func NewModelNodeDelegate(n interfaces.Node) interfaces.ModelNodeDelegate {
return &ModelNodeDelegate{
n: n,
ModelDelegate: NewModelDelegate(n),
}
}

65
core/models/delegate/model_node_test.go
View File

@@ -1,65 +0,0 @@
package delegate_test
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/delegate"
models2 "github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/stretchr/testify/require"
"testing"
)
func TestNode_Add(t *testing.T) {
SetupTest(t)
n := &models2.Node{}
err := delegate.NewModelDelegate(n).Add()
require.Nil(t, err)
require.NotNil(t, n.Id)
// validate artifact
a, err := delegate.NewModelDelegate(n).GetArtifact()
require.Nil(t, err)
require.Equal(t, n.Id, a.GetId())
require.NotNil(t, a.GetSys().GetCreateTs())
require.NotNil(t, a.GetSys().GetUpdateTs())
}
func TestNode_Save(t *testing.T) {
SetupTest(t)
n := &models2.Node{}
err := delegate.NewModelDelegate(n).Add()
name := "test_node"
n.Name = name
err = delegate.NewModelDelegate(n).Save()
require.Nil(t, err)
err = mongo.GetMongoCol(interfaces.ModelColNameNode).FindId(n.Id).One(&n)
require.Nil(t, err)
require.Equal(t, name, n.Name)
}
func TestNode_Delete(t *testing.T) {
SetupTest(t)
n := &models2.Node{
Name: "test_node",
}
err := delegate.NewModelDelegate(n).Add()
require.Nil(t, err)
err = delegate.NewModelDelegate(n).Delete()
require.Nil(t, err)
var a models2.Artifact
col := mongo.GetMongoCol(interfaces.ModelColNameArtifact)
err = col.FindId(n.Id).One(&a)
require.Nil(t, err)
require.NotNil(t, a.Obj)
require.True(t, a.Del)
}

65
core/models/delegate/model_role_test.go
View File

@@ -1,65 +0,0 @@
package delegate_test
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/delegate"
models2 "github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/stretchr/testify/require"
"testing"
)
func TestRole_Add(t *testing.T) {
SetupTest(t)
p := &models2.Role{}
err := delegate.NewModelDelegate(p).Add()
require.Nil(t, err)
require.NotNil(t, p.Id)
a, err := delegate.NewModelDelegate(p).GetArtifact()
require.Nil(t, err)
require.Equal(t, p.Id, a.GetId())
require.NotNil(t, a.GetSys().GetCreateTs())
require.NotNil(t, a.GetSys().GetUpdateTs())
}
func TestRole_Save(t *testing.T) {
SetupTest(t)
p := &models2.Role{}
err := delegate.NewModelDelegate(p).Add()
require.Nil(t, err)
name := "test_role"
p.Name = name
err = delegate.NewModelDelegate(p).Save()
require.Nil(t, err)
err = mongo.GetMongoCol(interfaces.ModelColNameRole).FindId(p.Id).One(&p)
require.Nil(t, err)
require.Equal(t, name, p.Name)
}
func TestRole_Delete(t *testing.T) {
SetupTest(t)
p := &models2.Role{
Name: "test_role",
}
err := delegate.NewModelDelegate(p).Add()
require.Nil(t, err)
err = delegate.NewModelDelegate(p).Delete()
require.Nil(t, err)
var a models2.Artifact
col := mongo.GetMongoCol(interfaces.ModelColNameArtifact)
err = col.FindId(p.Id).One(&a)
require.Nil(t, err)
require.NotNil(t, a.Obj)
require.True(t, a.Del)
}

65
core/models/delegate/model_test.go
View File

@@ -1,65 +0,0 @@
package delegate_test
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/delegate"
models2 "github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/stretchr/testify/require"
"testing"
)
func TestProject_Add(t *testing.T) {
SetupTest(t)
p := &models2.Project{}
err := delegate.NewModelDelegate(p).Add()
require.Nil(t, err)
require.NotNil(t, p.Id)
a, err := delegate.NewModelDelegate(p).GetArtifact()
require.Nil(t, err)
require.Equal(t, p.Id, a.GetId())
require.NotNil(t, a.GetSys().GetCreateTs())
require.NotNil(t, a.GetSys().GetUpdateTs())
}
func TestProject_Save(t *testing.T) {
SetupTest(t)
p := &models2.Project{}
err := delegate.NewModelDelegate(p).Add()
require.Nil(t, err)
name := "test_project"
p.Name = name
err = delegate.NewModelDelegate(p).Save()
require.Nil(t, err)
err = mongo.GetMongoCol(interfaces.ModelColNameProject).FindId(p.Id).One(&p)
require.Nil(t, err)
require.Equal(t, name, p.Name)
}
func TestProject_Delete(t *testing.T) {
SetupTest(t)
p := &models2.Project{
Name: "test_project",
}
err := delegate.NewModelDelegate(p).Add()
require.Nil(t, err)
err = delegate.NewModelDelegate(p).Delete()
require.Nil(t, err)
var a models2.Artifact
col := mongo.GetMongoCol(interfaces.ModelColNameArtifact)
err = col.FindId(p.Id).One(&a)
require.Nil(t, err)
require.NotNil(t, a.Obj)
require.True(t, a.Del)
}

20
core/models/delegate/utils_event.go
View File

@@ -1,20 +0,0 @@
package delegate
import (
"fmt"
"github.com/crawlab-team/crawlab/core/interfaces"
)
func GetEventName(d *ModelDelegate, method interfaces.ModelDelegateMethod) (eventName string) {
return getEventName(d, method)
}
func getEventName(d *ModelDelegate, method interfaces.ModelDelegateMethod) (eventName string) {
if method == interfaces.ModelDelegateMethodSave {
hasChange := d.hasChange()
if hasChange {
method = interfaces.ModelDelegateMethodChange
}
}
return fmt.Sprintf("model:%s:%s", d.colName, method)
}

56
core/models/models/artifact.go
View File

@@ -1,56 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Artifact struct {
Id primitive.ObjectID `bson:"_id" json:"_id"`
Col string `bson:"_col" json:"_col"`
Del bool `bson:"_del" json:"_del"`
TagIds []primitive.ObjectID `bson:"_tid" json:"_tid"`
Sys *ArtifactSys `bson:"_sys" json:"_sys"`
Obj interface{} `bson:"_obj" json:"_obj"`
}
func (a *Artifact) GetId() (id primitive.ObjectID) {
return a.Id
}
func (a *Artifact) SetId(id primitive.ObjectID) {
a.Id = id
}
func (a *Artifact) GetSys() (sys interfaces.ModelArtifactSys) {
if a.Sys == nil {
a.Sys = &ArtifactSys{}
}
return a.Sys
}
func (a *Artifact) GetTagIds() (ids []primitive.ObjectID) {
return a.TagIds
}
func (a *Artifact) SetTagIds(ids []primitive.ObjectID) {
a.TagIds = ids
}
func (a *Artifact) SetObj(obj interfaces.Model) {
a.Obj = obj
}
func (a *Artifact) SetDel(del bool) {
a.Del = del
}
type ArtifactList []Artifact
func (l *ArtifactList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

63
core/models/models/artifact_sys.go
View File

@@ -1,63 +0,0 @@
package models
import (
"go.mongodb.org/mongo-driver/bson/primitive"
"time"
)
type ArtifactSys struct {
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
CreateUid primitive.ObjectID `json:"create_uid" bson:"create_uid"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
UpdateUid primitive.ObjectID `json:"update_uid" bson:"update_uid"`
DeleteTs time.Time `json:"delete_ts" bson:"delete_ts"`
DeleteUid primitive.ObjectID `json:"delete_uid" bson:"delete_uid"`
}
func (sys *ArtifactSys) GetCreateTs() time.Time {
return sys.CreateTs
}
func (sys *ArtifactSys) SetCreateTs(ts time.Time) {
sys.CreateTs = ts
}
func (sys *ArtifactSys) GetUpdateTs() time.Time {
return sys.UpdateTs
}
func (sys *ArtifactSys) SetUpdateTs(ts time.Time) {
sys.UpdateTs = ts
}
func (sys *ArtifactSys) GetDeleteTs() time.Time {
return sys.DeleteTs
}
func (sys *ArtifactSys) SetDeleteTs(ts time.Time) {
sys.DeleteTs = ts
}
func (sys *ArtifactSys) GetCreateUid() primitive.ObjectID {
return sys.CreateUid
}
func (sys *ArtifactSys) SetCreateUid(id primitive.ObjectID) {
sys.CreateUid = id
}
func (sys *ArtifactSys) GetUpdateUid() primitive.ObjectID {
return sys.UpdateUid
}
func (sys *ArtifactSys) SetUpdateUid(id primitive.ObjectID) {
sys.UpdateUid = id
}
func (sys *ArtifactSys) GetDeleteUid() primitive.ObjectID {
return sys.DeleteUid
}
func (sys *ArtifactSys) SetDeleteUid(id primitive.ObjectID) {
sys.DeleteUid = id
}

13
core/models/models/base.go
View File

@@ -1,13 +0,0 @@
package models
import (
"go.mongodb.org/mongo-driver/bson/primitive"
)
type BaseModel struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
}
func (d *BaseModel) GetId() (id primitive.ObjectID) {
return d.Id
}

36
core/models/models/data_collection.go
View File

@@ -1,36 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type DataCollection struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Name string `json:"name" bson:"name"`
Fields []entity.DataField `json:"fields" bson:"fields"`
Dedup struct {
Enabled bool `json:"enabled" bson:"enabled"`
Keys []string `json:"keys" bson:"keys"`
Type string `json:"type" bson:"type"`
} `json:"dedup" bson:"dedup"`
}
func (dc *DataCollection) GetId() (id primitive.ObjectID) {
return dc.Id
}
func (dc *DataCollection) SetId(id primitive.ObjectID) {
dc.Id = id
}
type DataCollectionList []DataCollection
func (l *DataCollectionList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

42
core/models/models/data_source.go
View File

@@ -1,42 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type DataSource struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Name string `json:"name" bson:"name"`
Type string `json:"type" bson:"type"`
Description string `json:"description" bson:"description"`
Host string `json:"host" bson:"host"`
Port int `json:"port" bson:"port"`
Url string `json:"url" bson:"url"`
Hosts []string `json:"hosts" bson:"hosts"`
Database string `json:"database" bson:"database"`
Username string `json:"username" bson:"username"`
Password string `json:"password,omitempty" bson:"-"`
ConnectType string `json:"connect_type" bson:"connect_type"`
Status string `json:"status" bson:"status"`
Error string `json:"error" bson:"error"`
Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"`
}
func (ds *DataSource) GetId() (id primitive.ObjectID) {
return ds.Id
}
func (ds *DataSource) SetId(id primitive.ObjectID) {
ds.Id = id
}
type DataSourceList []DataSource
func (l *DataSourceList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

36
core/models/models/dependency_setting.go
View File

@@ -1,36 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
"time"
)
type DependencySetting struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Enabled bool `json:"enabled" bson:"enabled"`
Cmd string `json:"cmd" bson:"cmd"`
Proxy string `json:"proxy" bson:"proxy"`
LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"`
}
func (j *DependencySetting) GetId() (id primitive.ObjectID) {
return j.Id
}
func (j *DependencySetting) SetId(id primitive.ObjectID) {
j.Id = id
}
type DependencySettingList []DependencySetting
func (l *DependencySettingList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

46
core/models/models/environment.go
View File

@@ -1,46 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Environment struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Key string `json:"key" bson:"key"`
Value string `json:"value" bson:"value"`
}
func (e *Environment) GetId() (id primitive.ObjectID) {
return e.Id
}
func (e *Environment) SetId(id primitive.ObjectID) {
e.Id = id
}
func (e *Environment) GetKey() (key string) {
return e.Key
}
func (e *Environment) SetKey(key string) {
e.Key = key
}
func (e *Environment) GetValue() (value string) {
return e.Value
}
func (e *Environment) SetValue(value string) {
e.Value = value
}
type EnvironmentList []Environment
func (l *EnvironmentList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

64
core/models/models/extra_value.go
View File

@@ -1,64 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type ExtraValue struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
ObjectId primitive.ObjectID `json:"oid" bson:"oid"`
Model string `json:"model" bson:"m"`
Type string `json:"type" bson:"t"`
Value interface{} `json:"value" bson:"v"`
}
func (ev *ExtraValue) GetId() (id primitive.ObjectID) {
return ev.Id
}
func (ev *ExtraValue) SetId(id primitive.ObjectID) {
ev.Id = id
}
func (ev *ExtraValue) GetValue() (v interface{}) {
return ev.Value
}
func (ev *ExtraValue) SetValue(v interface{}) {
ev.Value = v
}
func (ev *ExtraValue) GetObjectId() (oid primitive.ObjectID) {
return ev.ObjectId
}
func (ev *ExtraValue) SetObjectId(oid primitive.ObjectID) {
ev.ObjectId = oid
}
func (ev *ExtraValue) GetModel() (m string) {
return ev.Model
}
func (ev *ExtraValue) SetModel(m string) {
ev.Model = m
}
func (ev *ExtraValue) GetType() (t string) {
return ev.Type
}
func (ev *ExtraValue) SetType(t string) {
ev.Type = t
}
type ExtraValueList []ExtraValue
func (l *ExtraValueList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

82
core/models/models/git.go
View File

@@ -1,82 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Git struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Url string `json:"url" bson:"url"`
AuthType string `json:"auth_type" bson:"auth_type"`
Username string `json:"username" bson:"username"`
Password string `json:"password" bson:"password"`
CurrentBranch string `json:"current_branch" bson:"current_branch"`
AutoPull bool `json:"auto_pull" bson:"auto_pull"`
}
func (g *Git) GetId() (id primitive.ObjectID) {
return g.Id
}
func (g *Git) SetId(id primitive.ObjectID) {
g.Id = id
}
func (g *Git) GetUrl() (url string) {
return g.Url
}
func (g *Git) SetUrl(url string) {
g.Url = url
}
func (g *Git) GetAuthType() (authType string) {
return g.AuthType
}
func (g *Git) SetAuthType(authType string) {
g.AuthType = authType
}
func (g *Git) GetUsername() (username string) {
return g.Username
}
func (g *Git) SetUsername(username string) {
g.Username = username
}
func (g *Git) GetPassword() (password string) {
return g.Password
}
func (g *Git) SetPassword(password string) {
g.Password = password
}
func (g *Git) GetCurrentBranch() (currentBranch string) {
return g.CurrentBranch
}
func (g *Git) SetCurrentBranch(currentBranch string) {
g.CurrentBranch = currentBranch
}
func (g *Git) GetAutoPull() (autoPull bool) {
return g.AutoPull
}
func (g *Git) SetAutoPull(autoPull bool) {
g.AutoPull = autoPull
}
type GitList []Git
func (l *GitList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

29
core/models/models/job.go
View File

@@ -1,29 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Job struct {
Id primitive.ObjectID `bson:"_id" json:"_id"`
TaskId primitive.ObjectID `bson:"task_id" json:"task_id"`
}
func (j *Job) GetId() (id primitive.ObjectID) {
return j.Id
}
func (j *Job) SetId(id primitive.ObjectID) {
j.Id = id
}
type JobList []Job
func (l *JobList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

119
core/models/models/node.go
View File

@@ -1,119 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
"time"
)
type Node struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Ip string `json:"ip" bson:"ip"`
Port string `json:"port" bson:"port"`
Mac string `json:"mac" bson:"mac"`
Hostname string `json:"hostname" bson:"hostname"`
Description string `json:"description" bson:"description"`
IsMaster bool `json:"is_master" bson:"is_master"`
Status string `json:"status" bson:"status"`
Enabled bool `json:"enabled" bson:"enabled"`
Active bool `json:"active" bson:"active"`
ActiveTs time.Time `json:"active_ts" bson:"active_ts"`
AvailableRunners int `json:"available_runners" bson:"available_runners"`
MaxRunners int `json:"max_runners" bson:"max_runners"`
}
func (n *Node) GetId() (id primitive.ObjectID) {
return n.Id
}
func (n *Node) SetId(id primitive.ObjectID) {
n.Id = id
}
func (n *Node) GetName() (name string) {
return n.Name
}
func (n *Node) SetName(name string) {
n.Name = name
}
func (n *Node) GetDescription() (description string) {
return n.Description
}
func (n *Node) SetDescription(description string) {
n.Description = description
}
func (n *Node) GetKey() (key string) {
return n.Key
}
func (n *Node) GetIsMaster() (ok bool) {
return n.IsMaster
}
func (n *Node) GetActive() (active bool) {
return n.Active
}
func (n *Node) SetActive(active bool) {
n.Active = active
}
func (n *Node) SetActiveTs(activeTs time.Time) {
n.ActiveTs = activeTs
}
func (n *Node) GetStatus() (status string) {
return n.Status
}
func (n *Node) SetStatus(status string) {
n.Status = status
}
func (n *Node) GetEnabled() (enabled bool) {
return n.Enabled
}
func (n *Node) SetEnabled(enabled bool) {
n.Enabled = enabled
}
func (n *Node) GetAvailableRunners() (runners int) {
return n.AvailableRunners
}
func (n *Node) SetAvailableRunners(runners int) {
n.AvailableRunners = runners
}
func (n *Node) GetMaxRunners() (runners int) {
return n.MaxRunners
}
func (n *Node) SetMaxRunners(runners int) {
n.MaxRunners = runners
}
func (n *Node) IncrementAvailableRunners() {
n.AvailableRunners++
}
func (n *Node) DecrementAvailableRunners() {
n.AvailableRunners--
}
type NodeList []Node
func (l *NodeList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

29
core/models/models/password.go
View File

@@ -1,29 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Password struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Password string `json:"password" bson:"p"`
}
func (p *Password) GetId() (id primitive.ObjectID) {
return p.Id
}
func (p *Password) SetId(id primitive.ObjectID) {
p.Id = id
}
type PasswordList []Password
func (l *PasswordList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

91
core/models/models/permission.go
View File

@@ -1,91 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Permission struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Type string `json:"type" bson:"type"`
Target []string `json:"target" bson:"target"`
Allow []string `json:"allow" bson:"allow"`
Deny []string `json:"deny" bson:"deny"`
}
func (p *Permission) GetId() (id primitive.ObjectID) {
return p.Id
}
func (p *Permission) SetId(id primitive.ObjectID) {
p.Id = id
}
func (p *Permission) GetKey() (key string) {
return p.Key
}
func (p *Permission) SetKey(key string) {
p.Key = key
}
func (p *Permission) GetName() (name string) {
return p.Name
}
func (p *Permission) SetName(name string) {
p.Name = name
}
func (p *Permission) GetDescription() (description string) {
return p.Description
}
func (p *Permission) SetDescription(description string) {
p.Description = description
}
func (p *Permission) GetType() (t string) {
return p.Type
}
func (p *Permission) SetType(t string) {
p.Type = t
}
func (p *Permission) GetTarget() (target []string) {
return p.Target
}
func (p *Permission) SetTarget(target []string) {
p.Target = target
}
func (p *Permission) GetAllow() (include []string) {
return p.Allow
}
func (p *Permission) SetAllow(include []string) {
p.Allow = include
}
func (p *Permission) GetDeny() (exclude []string) {
return p.Deny
}
func (p *Permission) SetDeny(exclude []string) {
p.Deny = exclude
}
type PermissionList []Permission
func (l *PermissionList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

47
core/models/models/project.go
View File

@@ -1,47 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Project struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Spiders int `json:"spiders" bson:"-"`
}
func (p *Project) GetId() (id primitive.ObjectID) {
return p.Id
}
func (p *Project) SetId(id primitive.ObjectID) {
p.Id = id
}
func (p *Project) GetName() (name string) {
return p.Name
}
func (p *Project) SetName(name string) {
p.Name = name
}
func (p *Project) GetDescription() (description string) {
return p.Description
}
func (p *Project) SetDescription(description string) {
p.Description = description
}
type ProjectList []Project
func (l *ProjectList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

60
core/models/models/result.go
View File

@@ -1,60 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Result bson.M
func (r *Result) GetId() (id primitive.ObjectID) {
res, ok := r.Value()["_id"]
if ok {
id, ok = res.(primitive.ObjectID)
if ok {
return id
}
}
return id
}
func (r *Result) SetId(id primitive.ObjectID) {
(*r)["_id"] = id
}
func (r *Result) Value() map[string]interface{} {
return *r
}
func (r *Result) SetValue(key string, value interface{}) {
(*r)[key] = value
}
func (r *Result) GetValue(key string) (value interface{}) {
return (*r)[key]
}
func (r *Result) GetTaskId() (id primitive.ObjectID) {
res := r.GetValue(constants.TaskKey)
if res == nil {
return id
}
id, _ = res.(primitive.ObjectID)
return id
}
func (r *Result) SetTaskId(id primitive.ObjectID) {
r.SetValue(constants.TaskKey, id)
}
type ResultList []Result
func (l *ResultList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

55
core/models/models/role.go
View File

@@ -1,55 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Role struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
}
func (r *Role) GetId() (id primitive.ObjectID) {
return r.Id
}
func (r *Role) SetId(id primitive.ObjectID) {
r.Id = id
}
func (r *Role) GetKey() (key string) {
return r.Key
}
func (r *Role) SetKey(key string) {
r.Key = key
}
func (r *Role) GetName() (name string) {
return r.Name
}
func (r *Role) SetName(name string) {
r.Name = name
}
func (r *Role) GetDescription() (description string) {
return r.Description
}
func (r *Role) SetDescription(description string) {
r.Description = description
}
type RoleList []Role
func (l *RoleList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

30
core/models/models/role_permission.go
View File

@@ -1,30 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type RolePermission struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"`
}
func (ur *RolePermission) GetId() (id primitive.ObjectID) {
return ur.Id
}
func (ur *RolePermission) SetId(id primitive.ObjectID) {
ur.Id = id
}
type RolePermissionList []RolePermission
func (l *RolePermissionList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

113
core/models/models/schedule.go
View File

@@ -1,113 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/robfig/cron/v3"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Schedule struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Cron string `json:"cron" bson:"cron"`
EntryId cron.EntryID `json:"entry_id" bson:"entry_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
Priority int `json:"priority" bson:"priority"`
Enabled bool `json:"enabled" bson:"enabled"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
}
func (s *Schedule) GetId() (id primitive.ObjectID) {
return s.Id
}
func (s *Schedule) SetId(id primitive.ObjectID) {
s.Id = id
}
func (s *Schedule) GetEnabled() (enabled bool) {
return s.Enabled
}
func (s *Schedule) SetEnabled(enabled bool) {
s.Enabled = enabled
}
func (s *Schedule) GetEntryId() (id cron.EntryID) {
return s.EntryId
}
func (s *Schedule) SetEntryId(id cron.EntryID) {
s.EntryId = id
}
func (s *Schedule) GetCron() (c string) {
return s.Cron
}
func (s *Schedule) SetCron(c string) {
s.Cron = c
}
func (s *Schedule) GetSpiderId() (id primitive.ObjectID) {
return s.SpiderId
}
func (s *Schedule) SetSpiderId(id primitive.ObjectID) {
s.SpiderId = id
}
func (s *Schedule) GetMode() (mode string) {
return s.Mode
}
func (s *Schedule) SetMode(mode string) {
s.Mode = mode
}
func (s *Schedule) GetNodeIds() (ids []primitive.ObjectID) {
return s.NodeIds
}
func (s *Schedule) SetNodeIds(ids []primitive.ObjectID) {
s.NodeIds = ids
}
func (s *Schedule) GetCmd() (cmd string) {
return s.Cmd
}
func (s *Schedule) SetCmd(cmd string) {
s.Cmd = cmd
}
func (s *Schedule) GetParam() (param string) {
return s.Param
}
func (s *Schedule) SetParam(param string) {
s.Param = param
}
func (s *Schedule) GetPriority() (p int) {
return s.Priority
}
func (s *Schedule) SetPriority(p int) {
s.Priority = p
}
type ScheduleList []Schedule
func (l *ScheduleList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

31
core/models/models/setting.go
View File

@@ -1,31 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Setting struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Key string `json:"key" bson:"key"`
Value bson.M `json:"value" bson:"value"`
}
func (s *Setting) GetId() (id primitive.ObjectID) {
return s.Id
}
func (s *Setting) SetId(id primitive.ObjectID) {
s.Id = id
}
type SettingList []Setting
func (l *SettingList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

137
core/models/models/spider.go
View File

@@ -1,137 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Env struct {
Name string `json:"name" bson:"name"`
Value string `json:"value" bson:"value"`
}
type Spider struct {
Id primitive.ObjectID `json:"_id" bson:"_id"` // spider id
Name string `json:"name" bson:"name"` // spider name
Type string `json:"type" bson:"type"` // spider type
ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id
ColName string `json:"col_name,omitempty" bson:"-"` // data collection name
DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id
DataSource *DataSource `json:"data_source,omitempty" bson:"-"` // data source
Description string `json:"description" bson:"description"` // description
ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id
Mode string `json:"mode" bson:"mode"` // default Task.Mode
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds
Stat *SpiderStat `json:"stat,omitempty" bson:"-"`
// execution
Cmd string `json:"cmd" bson:"cmd"` // execute command
Param string `json:"param" bson:"param"` // default task param
Priority int `json:"priority" bson:"priority"`
AutoInstall bool `json:"auto_install" bson:"auto_install"`
// settings
IncrementalSync bool `json:"incremental_sync" bson:"incremental_sync"` // whether to incrementally sync files
}
func (s *Spider) GetId() (id primitive.ObjectID) {
return s.Id
}
func (s *Spider) SetId(id primitive.ObjectID) {
s.Id = id
}
func (s *Spider) GetName() (name string) {
return s.Name
}
func (s *Spider) SetName(name string) {
s.Name = name
}
func (s *Spider) GetDescription() (description string) {
return s.Description
}
func (s *Spider) SetDescription(description string) {
s.Description = description
}
func (s *Spider) GetType() (ty string) {
return s.Type
}
func (s *Spider) GetMode() (mode string) {
return s.Mode
}
func (s *Spider) SetMode(mode string) {
s.Mode = mode
}
func (s *Spider) GetNodeIds() (ids []primitive.ObjectID) {
return s.NodeIds
}
func (s *Spider) SetNodeIds(ids []primitive.ObjectID) {
s.NodeIds = ids
}
func (s *Spider) GetCmd() (cmd string) {
return s.Cmd
}
func (s *Spider) SetCmd(cmd string) {
s.Cmd = cmd
}
func (s *Spider) GetParam() (param string) {
return s.Param
}
func (s *Spider) SetParam(param string) {
s.Param = param
}
func (s *Spider) GetPriority() (p int) {
return s.Priority
}
func (s *Spider) SetPriority(p int) {
s.Priority = p
}
func (s *Spider) GetColId() (id primitive.ObjectID) {
return s.ColId
}
func (s *Spider) SetColId(id primitive.ObjectID) {
s.ColId = id
}
func (s *Spider) GetIncrementalSync() (incrementalSync bool) {
return s.IncrementalSync
}
func (s *Spider) SetIncrementalSync(incrementalSync bool) {
s.IncrementalSync = incrementalSync
}
func (s *Spider) GetAutoInstall() (autoInstall bool) {
return s.AutoInstall
}
func (s *Spider) SetAutoInstall(autoInstall bool) {
s.AutoInstall = autoInstall
}
type SpiderList []Spider
func (l *SpiderList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

38
core/models/models/spider_stat.go
View File

@@ -1,38 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type SpiderStat struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"`
LastTask *Task `json:"last_task,omitempty" bson:"-"`
Tasks int `json:"tasks" bson:"tasks"`
Results int `json:"results" bson:"results"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second
AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second
AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second
AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second
}
func (s *SpiderStat) GetId() (id primitive.ObjectID) {
return s.Id
}
func (s *SpiderStat) SetId(id primitive.ObjectID) {
s.Id = id
}
type SpiderStatList []SpiderStat
func (l *SpiderStatList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

44
core/models/models/tag.go
View File

@@ -1,44 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Tag struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Name string `json:"name" bson:"name"`
Color string `json:"color" bson:"color"`
Description string `json:"description" bson:"description"`
Col string `json:"col" bson:"col"`
}
func (t *Tag) GetId() (id primitive.ObjectID) {
return t.Id
}
func (t *Tag) SetId(id primitive.ObjectID) {
t.Id = id
}
func (t *Tag) GetName() (res string) {
return t.Name
}
func (t *Tag) GetColor() (res string) {
return t.Color
}
func (t *Tag) SetCol(col string) {
t.Col = col
}
type TagList []Tag
func (l *TagList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

118
core/models/models/task.go
View File

@@ -1,118 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
"time"
)
type Task struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Status string `json:"status" bson:"status"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Error string `json:"error" bson:"error"`
Pid int `json:"pid" bson:"pid"`
ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` // Schedule.Id
Type string `json:"type" bson:"type"`
Mode string `json:"mode" bson:"mode"` // running mode of Task
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // list of Node.Id
ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"` // parent Task.Id if it'Spider a sub-task
Priority int `json:"priority" bson:"priority"`
Stat *TaskStat `json:"stat,omitempty" bson:"-"`
HasSub bool `json:"has_sub" json:"has_sub"` // whether to have sub-tasks
SubTasks []Task `json:"sub_tasks,omitempty" bson:"-"`
Spider *Spider `json:"spider,omitempty" bson:"-"`
UserId primitive.ObjectID `json:"-" bson:"-"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
}
func (t *Task) GetId() (id primitive.ObjectID) {
return t.Id
}
func (t *Task) SetId(id primitive.ObjectID) {
t.Id = id
}
func (t *Task) GetNodeId() (id primitive.ObjectID) {
return t.NodeId
}
func (t *Task) SetNodeId(id primitive.ObjectID) {
t.NodeId = id
}
func (t *Task) GetNodeIds() (ids []primitive.ObjectID) {
return t.NodeIds
}
func (t *Task) GetStatus() (status string) {
return t.Status
}
func (t *Task) SetStatus(status string) {
t.Status = status
}
func (t *Task) GetError() (error string) {
return t.Error
}
func (t *Task) SetError(error string) {
t.Error = error
}
func (t *Task) GetPid() (pid int) {
return t.Pid
}
func (t *Task) SetPid(pid int) {
t.Pid = pid
}
func (t *Task) GetSpiderId() (id primitive.ObjectID) {
return t.SpiderId
}
func (t *Task) GetType() (ty string) {
return t.Type
}
func (t *Task) GetCmd() (cmd string) {
return t.Cmd
}
func (t *Task) GetParam() (param string) {
return t.Param
}
func (t *Task) GetPriority() (p int) {
return t.Priority
}
func (t *Task) GetUserId() (id primitive.ObjectID) {
return t.UserId
}
func (t *Task) SetUserId(id primitive.ObjectID) {
t.UserId = id
}
type TaskList []Task
func (l *TaskList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}
type TaskDailyItem struct {
Date string `json:"date" bson:"_id"`
TaskCount int `json:"task_count" bson:"task_count"`
AvgRuntimeDuration float64 `json:"avg_runtime_duration" bson:"avg_runtime_duration"`
}

30
core/models/models/task_queue_item.go
View File

@@ -1,30 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type TaskQueueItem struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Priority int `json:"p" bson:"p"`
NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"`
}
func (t *TaskQueueItem) GetId() (id primitive.ObjectID) {
return t.Id
}
func (t *TaskQueueItem) SetId(id primitive.ObjectID) {
t.Id = id
}
type TaskQueueItemList []TaskQueueItem
func (l *TaskQueueItemList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

101
core/models/models/task_stat.go
View File

@@ -1,101 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
"time"
)
type TaskStat struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"`
StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"`
EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond
ResultCount int64 `json:"result_count" bson:"result_count"`
ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"`
}
func (s *TaskStat) GetId() (id primitive.ObjectID) {
return s.Id
}
func (s *TaskStat) SetId(id primitive.ObjectID) {
s.Id = id
}
func (s *TaskStat) GetCreateTs() (ts time.Time) {
return s.CreateTs
}
func (s *TaskStat) SetCreateTs(ts time.Time) {
s.CreateTs = ts
}
func (s *TaskStat) GetStartTs() (ts time.Time) {
return s.StartTs
}
func (s *TaskStat) SetStartTs(ts time.Time) {
s.StartTs = ts
}
func (s *TaskStat) GetEndTs() (ts time.Time) {
return s.EndTs
}
func (s *TaskStat) SetEndTs(ts time.Time) {
s.EndTs = ts
}
func (s *TaskStat) GetWaitDuration() (d int64) {
return s.WaitDuration
}
func (s *TaskStat) SetWaitDuration(d int64) {
s.WaitDuration = d
}
func (s *TaskStat) GetRuntimeDuration() (d int64) {
return s.RuntimeDuration
}
func (s *TaskStat) SetRuntimeDuration(d int64) {
s.RuntimeDuration = d
}
func (s *TaskStat) GetTotalDuration() (d int64) {
return s.WaitDuration + s.RuntimeDuration
}
func (s *TaskStat) SetTotalDuration(d int64) {
s.TotalDuration = d
}
func (s *TaskStat) GetResultCount() (c int64) {
return s.ResultCount
}
func (s *TaskStat) SetResultCount(c int64) {
s.ResultCount = c
}
func (s *TaskStat) GetErrorLogCount() (c int64) {
return s.ErrorLogCount
}
func (s *TaskStat) SetErrorLogCount(c int64) {
s.ErrorLogCount = c
}
type TaskStatList []TaskStat
func (l *TaskStatList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

30
core/models/models/token.go
View File

@@ -1,30 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Token struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Name string `json:"name" bson:"name"`
Token string `json:"token" bson:"token"`
}
func (t *Token) GetId() (id primitive.ObjectID) {
return t.Id
}
func (t *Token) SetId(id primitive.ObjectID) {
t.Id = id
}
type TokenList []Token
func (l *TokenList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

59
core/models/models/user.go
View File

@@ -1,59 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type User struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Username string `json:"username" bson:"username"`
Password string `json:"password,omitempty" bson:"-"`
Role string `json:"role" bson:"role"`
Email string `json:"email" bson:"email"`
//Setting UserSetting `json:"setting" bson:"setting"`
}
func (u *User) GetId() (id primitive.ObjectID) {
return u.Id
}
func (u *User) SetId(id primitive.ObjectID) {
u.Id = id
}
func (u *User) GetUsername() (name string) {
return u.Username
}
func (u *User) GetPassword() (p string) {
return u.Password
}
func (u *User) GetRole() (r string) {
return u.Role
}
func (u *User) GetEmail() (email string) {
return u.Email
}
//type UserSetting struct {
// NotificationTrigger string `json:"notification_trigger" bson:"notification_trigger"`
// DingTalkRobotWebhook string `json:"ding_talk_robot_webhook" bson:"ding_talk_robot_webhook"`
// WechatRobotWebhook string `json:"wechat_robot_webhook" bson:"wechat_robot_webhook"`
// EnabledNotifications []string `json:"enabled_notifications" bson:"enabled_notifications"`
// ErrorRegexPattern string `json:"error_regex_pattern" bson:"error_regex_pattern"`
// MaxErrorLog int `json:"max_error_log" bson:"max_error_log"`
// LogExpireDuration int64 `json:"log_expire_duration" bson:"log_expire_duration"`
//}
type UserList []User
func (l *UserList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

30
core/models/models/user_role.go
View File

@@ -1,30 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type UserRole struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
}
func (ur *UserRole) GetId() (id primitive.ObjectID) {
return ur.Id
}
func (ur *UserRole) SetId(id primitive.ObjectID) {
ur.Id = id
}
type UserRoleList []UserRole
func (l *UserRoleList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

96
core/models/models/utils_binder_legacy.go
View File

@@ -1,96 +0,0 @@
package models
//func AssignFields(d interface{}, fieldIds ...interfaces.ModelId) (res interface{}, err error) {
// return assignFields(d, fieldIds...)
//}
//
//func AssignListFields(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
// return assignListFields(list, fieldIds...)
//}
//
//func AssignListFieldsAsPtr(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
// return assignListFieldsAsPtr(list, fieldIds...)
//}
//
//func assignFields(d interface{}, fieldIds ...interfaces.ModelId) (res interface{}, err error) {
// doc, ok := d.(interfaces.Model)
// if !ok {
// return nil, errors.ErrorModelInvalidType
// }
// if len(fieldIds) == 0 {
// return doc, nil
// }
// for _, fid := range fieldIds {
// switch fid {
// case interfaces.ModelIdTag:
// // convert interface
// d, ok := doc.(interfaces.ModelWithTags)
// if !ok {
// return nil, errors.ErrorModelInvalidType
// }
//
// // attempt to get artifact
// a, err := doc.GetArtifact()
// if err != nil {
// return nil, err
// }
//
// // skip if no artifact found
// if a == nil {
// return d, nil
// }
//
// // assign tags
// tags, err := a.GetTags()
// if err != nil {
// return nil, err
// }
// d.SetTags(tags)
//
// return d, nil
// }
// }
// return doc, nil
//}
//
//func _assignListFields(asPtr bool, list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
// vList := reflect.ValueOf(list)
// if vList.Kind() != reflect.Array &&
// vList.Kind() != reflect.Slice {
// return res, errors.ErrorModelInvalidType
// }
// for i := 0; i < vList.Len(); i++ {
// vItem := vList.Index(i)
// var item interface{}
// if vItem.CanAddr() {
// item = vItem.Addr().Interface()
// } else {
// item = vItem.Interface()
// }
// doc, ok := item.(interfaces.Model)
// if !ok {
// return res, errors.ErrorModelInvalidType
// }
// ptr, err := assignFields(doc, fieldIds...)
// if err != nil {
// return res, err
// }
// v := reflect.ValueOf(ptr)
// if !asPtr {
// // non-pointer item
// res.Add(v.Elem().Interface())
// } else {
// // pointer item
// res.Add(v.Interface())
// }
// }
// return res, nil
//}
//
//func assignListFields(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
// return _assignListFields(false, list, fieldIds...)
//}
//
//func assignListFieldsAsPtr(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) {
// return _assignListFields(true, list, fieldIds...)
//}

10
core/models/models/utils_col.go
View File

@@ -1,10 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/utils/binders"
)
func GetModelColName(id interfaces.ModelId) (colName string) {
return binders.NewColNameBinder(id).MustBindString()
}

95
core/models/models/utils_model_map.go
View File

@@ -1,95 +0,0 @@
package models
type ModelMap struct {
Artifact Artifact
Tag Tag
Node Node
Project Project
Spider Spider
Task Task
Job Job
Schedule Schedule
User User
Setting Setting
Token Token
Variable Variable
TaskQueueItem TaskQueueItem
TaskStat TaskStat
SpiderStat SpiderStat
DataSource DataSource
DataCollection DataCollection
Result Result
Password Password
ExtraValue ExtraValue
Git Git
Role Role
UserRole UserRole
Permission Permission
RolePermission RolePermission
Environment Environment
DependencySetting DependencySetting
}
type ModelListMap struct {
Artifacts ArtifactList
Tags TagList
Nodes NodeList
Projects ProjectList
Spiders SpiderList
Tasks TaskList
Jobs JobList
Schedules ScheduleList
Users UserList
Settings SettingList
Tokens TokenList
Variables VariableList
TaskQueueItems TaskQueueItemList
TaskStats TaskStatList
SpiderStats SpiderStatList
DataSources DataSourceList
DataCollections DataCollectionList
Results ResultList
Passwords PasswordList
ExtraValues ExtraValueList
Gits GitList
Roles RoleList
UserRoles UserRoleList
PermissionList PermissionList
RolePermissionList RolePermissionList
Environments EnvironmentList
DependencySettings DependencySettingList
}
func NewModelMap() (m *ModelMap) {
return &ModelMap{}
}
func NewModelListMap() (m *ModelListMap) {
return &ModelListMap{
Artifacts: ArtifactList{},
Tags: TagList{},
Nodes: NodeList{},
Projects: ProjectList{},
Spiders: SpiderList{},
Tasks: TaskList{},
Jobs: JobList{},
Schedules: ScheduleList{},
Users: UserList{},
Settings: SettingList{},
Tokens: TokenList{},
Variables: VariableList{},
TaskQueueItems: TaskQueueItemList{},
TaskStats: TaskStatList{},
SpiderStats: SpiderStatList{},
DataSources: DataSourceList{},
DataCollections: DataCollectionList{},
Results: ResultList{},
Passwords: PasswordList{},
ExtraValues: ExtraValueList{},
Gits: GitList{},
Roles: RoleList{},
PermissionList: PermissionList{},
RolePermissionList: RolePermissionList{},
Environments: EnvironmentList{},
}
}

34
core/models/models/utils_tag.go
View File

@@ -1,34 +0,0 @@
package models
import (
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/trace"
)
func convertInterfacesToTags(tags []interfaces.Tag) (res []Tag) {
if tags == nil {
return nil
}
for _, t := range tags {
tag, ok := t.(*Tag)
if !ok {
log.Warnf("%v: cannot convert tag", trace.TraceError(errors.ErrorModelInvalidType))
return nil
}
if tag == nil {
log.Warnf("%v: cannot convert tag", trace.TraceError(errors.ErrorModelInvalidType))
return nil
}
res = append(res, *tag)
}
return res
}
func convertTagsToInterfaces(tags []Tag) (res []interfaces.Tag) {
for _, t := range tags {
res = append(res, &t)
}
return res
}

31
core/models/models/variable.go
View File

@@ -1,31 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type Variable struct {
Id primitive.ObjectID `json:"_id" bson:"_id"`
Key string `json:"key" bson:"key"`
Value string `json:"value" bson:"value"`
Remark string `json:"remark" bson:"remark"`
}
func (v *Variable) GetId() (id primitive.ObjectID) {
return v.Id
}
func (v *Variable) SetId(id primitive.ObjectID) {
v.Id = id
}
type VariableList []Variable
func (l *VariableList) GetModels() (res []interfaces.Model) {
for i := range *l {
d := (*l)[i]
res = append(res, &d)
}
return res
}

40
core/models/service/artifact_service.go
View File

@@ -1,40 +0,0 @@
package service
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
models2 "github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
func convertTypeArtifact(d interface{}, err error) (res *models2.Artifact, err2 error) {
if err != nil {
return nil, err
}
res, ok := d.(*models2.Artifact)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return res, nil
}
func (svc *Service) GetArtifactById(id primitive.ObjectID) (res *models2.Artifact, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdArtifact).GetById(id)
return convertTypeArtifact(d, err)
}
func (svc *Service) GetArtifact(query bson.M, opts *mongo.FindOptions) (res *models2.Artifact, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdArtifact).Get(query, opts)
return convertTypeArtifact(d, err)
}
func (svc *Service) GetArtifactList(query bson.M, opts *mongo.FindOptions) (res []models2.Artifact, err error) {
l, err := svc.GetBaseService(interfaces.ModelIdArtifact).GetList(query, opts)
for _, doc := range l.GetModels() {
d := doc.(*models2.Artifact)
res = append(res, *d)
}
return res, nil
}

420
core/models/service/base_service.go
View File

@@ -1,420 +0,0 @@
package service
import (
"encoding/json"
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/delegate"
models2 "github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/trace"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"reflect"
"strings"
"sync"
"time"
)
type BaseService struct {
id interfaces.ModelId
col *mongo.Col
}
func (svc *BaseService) GetModelId() (id interfaces.ModelId) {
return svc.id
}
func (svc *BaseService) SetModelId(id interfaces.ModelId) {
svc.id = id
}
func (svc *BaseService) GetCol() (col *mongo.Col) {
return svc.col
}
func (svc *BaseService) SetCol(col *mongo.Col) {
svc.col = col
}
func (svc *BaseService) GetById(id primitive.ObjectID) (res interfaces.Model, err error) {
// find result
fr := svc.findId(id)
// bind
return NewBasicBinder(svc.id, fr).Bind()
}
func (svc *BaseService) Get(query bson.M, opts *mongo.FindOptions) (res interfaces.Model, err error) {
// find result
fr := svc.find(query, opts)
// bind
return NewBasicBinder(svc.id, fr).Bind()
}
func (svc *BaseService) GetList(query bson.M, opts *mongo.FindOptions) (l interfaces.List, err error) {
// find result
tic := time.Now()
log.Debugf("baseService.GetMany -> svc.find:start")
log.Debugf("baseService.GetMany -> svc.id: %v", svc.id)
log.Debugf("baseService.GetMany -> svc.col.GetName(): %v", svc.col.GetName())
log.Debugf("baseService.GetMany -> query: %v", query)
log.Debugf("baseService.GetMany -> opts: %v", opts)
fr := svc.find(query, opts)
log.Debugf("baseService.GetMany -> svc.find:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds())
// bind
return NewListBinder(svc.id, fr).Bind()
}
func (svc *BaseService) DeleteById(id primitive.ObjectID, args ...interface{}) (err error) {
return svc.deleteId(id, args...)
}
func (svc *BaseService) Delete(query bson.M, args ...interface{}) (err error) {
return svc.delete(query)
}
func (svc *BaseService) DeleteList(query bson.M, args ...interface{}) (err error) {
return svc.deleteList(query)
}
func (svc *BaseService) ForceDeleteList(query bson.M, args ...interface{}) (err error) {
return svc.forceDeleteList(query)
}
func (svc *BaseService) UpdateById(id primitive.ObjectID, update bson.M, args ...interface{}) (err error) {
return svc.updateId(id, update)
}
func (svc *BaseService) Update(query bson.M, update bson.M, fields []string, args ...interface{}) (err error) {
return svc.update(query, update, fields)
}
func (svc *BaseService) UpdateDoc(query bson.M, doc interfaces.Model, fields []string, args ...interface{}) (err error) {
return svc.update(query, doc, fields)
}
func (svc *BaseService) Insert(u interfaces.User, docs ...interface{}) (err error) {
log.Debugf("baseService.Insert -> svc.col.GetName(): %v", svc.col.GetName())
log.Debugf("baseService.Insert -> docs: %v", docs)
return svc.insert(u, docs...)
}
func (svc *BaseService) Count(query bson.M) (total int, err error) {
return svc.count(query)
}
func (svc *BaseService) findId(id primitive.ObjectID) (fr *mongo.FindResult) {
if svc.col == nil {
return mongo.NewFindResultWithError(constants.ErrMissingCol)
}
return svc.col.FindId(id)
}
func (svc *BaseService) find(query bson.M, opts *mongo.FindOptions) (fr *mongo.FindResult) {
if svc.col == nil {
return mongo.NewFindResultWithError(constants.ErrMissingCol)
}
return svc.col.Find(query, opts)
}
func (svc *BaseService) deleteId(id primitive.ObjectID, args ...interface{}) (err error) {
if svc.col == nil {
return trace.TraceError(constants.ErrMissingCol)
}
fr := svc.findId(id)
doc, err := NewBasicBinder(svc.id, fr).Bind()
if err != nil {
return err
}
return delegate.NewModelDelegate(doc, svc._getUserFromArgs(args...)).Delete()
}
func (svc *BaseService) delete(query bson.M, args ...interface{}) (err error) {
if svc.col == nil {
return trace.TraceError(constants.ErrMissingCol)
}
var doc models2.BaseModel
if err := svc.find(query, nil).One(&doc); err != nil {
return err
}
return svc.deleteId(doc.Id, svc._getUserFromArgs(args...))
}
func (svc *BaseService) deleteList(query bson.M, args ...interface{}) (err error) {
if svc.col == nil {
return trace.TraceError(constants.ErrMissingCol)
}
fr := svc.find(query, nil)
list, err := NewListBinder(svc.id, fr).Bind()
if err != nil {
return err
}
for _, doc := range list.GetModels() {
if err := delegate.NewModelDelegate(doc, svc._getUserFromArgs(args...)).Delete(); err != nil {
return err
}
}
return nil
}
func (svc *BaseService) forceDeleteList(query bson.M, args ...interface{}) (err error) {
return svc.col.Delete(query)
}
func (svc *BaseService) count(query bson.M) (total int, err error) {
if svc.col == nil {
return total, trace.TraceError(constants.ErrMissingCol)
}
return svc.col.Count(query)
}
func (svc *BaseService) update(query bson.M, update interface{}, fields []string, args ...interface{}) (err error) {
update, err = svc._getUpdateBsonM(update, fields)
if err != nil {
return err
}
return svc._update(query, update, svc._getUserFromArgs(args...))
}
func (svc *BaseService) updateId(id primitive.ObjectID, update interface{}, args ...interface{}) (err error) {
update, err = svc._getUpdateBsonM(update, nil)
if err != nil {
return err
}
return svc._updateById(id, update, svc._getUserFromArgs(args...))
}
func (svc *BaseService) insert(u interfaces.User, docs ...interface{}) (err error) {
// validate col
if svc.col == nil {
return trace.TraceError(constants.ErrMissingCol)
}
// iterate docs
for i, doc := range docs {
switch doc.(type) {
case map[string]interface{}:
// doc type: map[string]interface{}, need to handle _id
d := doc.(map[string]interface{})
vId, ok := d["_id"]
if !ok {
// _id not exists
d["_id"] = primitive.NewObjectID()
} else {
// _id exists
switch vId.(type) {
case string:
// _id type: string
sId, ok := vId.(string)
if ok {
d["_id"], err = primitive.ObjectIDFromHex(sId)
if err != nil {
return trace.TraceError(err)
}
}
case primitive.ObjectID:
// _id type: primitive.ObjectID
// do nothing
default:
return trace.TraceError(errors.ErrorModelInvalidType)
}
}
}
docs[i] = doc
}
// perform insert
ids, err := svc.col.InsertMany(docs)
if err != nil {
return err
}
// upsert artifacts
query := bson.M{
"_id": bson.M{
"$in": ids,
},
}
fr := svc.col.Find(query, nil)
list, err := NewListBinder(svc.id, fr).Bind()
for _, doc := range list.GetModels() {
// upsert artifact when performing model delegate save
if err := delegate.NewModelDelegate(doc, u).Save(); err != nil {
return err
}
}
return nil
}
func (svc *BaseService) _update(query bson.M, update interface{}, args ...interface{}) (err error) {
// ids of query
var ids []primitive.ObjectID
list, err := NewListBinder(svc.id, svc.find(query, nil)).Bind()
if err != nil {
return err
}
for _, doc := range list.GetModels() {
ids = append(ids, doc.GetId())
}
// update model objects
if err := svc.col.Update(query, update); err != nil {
return err
}
// update artifacts
u := svc._getUserFromArgs(args...)
return mongo.GetMongoCol(interfaces.ModelColNameArtifact).Update(query, svc._getUpdateArtifactUpdate(u))
}
func (svc *BaseService) _updateById(id primitive.ObjectID, update interface{}, args ...interface{}) (err error) {
// update model object
if err := svc.col.UpdateId(id, update); err != nil {
return err
}
// update artifact
u := svc._getUserFromArgs(args...)
return mongo.GetMongoCol(interfaces.ModelColNameArtifact).UpdateId(id, svc._getUpdateArtifactUpdate(u))
}
func (svc *BaseService) _getUpdateBsonM(update interface{}, fields []string) (res bson.M, err error) {
switch update.(type) {
case interfaces.Model:
// convert to bson.M
var updateBsonM bson.M
bytes, err := json.Marshal(&update)
if err != nil {
return nil, err
}
if err := json.Unmarshal(bytes, &updateBsonM); err != nil {
return nil, err
}
return svc._getUpdateBsonM(updateBsonM, fields)
case bson.M:
// convert to bson.M
updateBsonM := update.(bson.M)
// filter fields if not nil
if fields != nil {
// fields map
fieldsMap := map[string]bool{}
for _, f := range fields {
fieldsMap[f] = true
}
// remove unselected fields
for k := range updateBsonM {
if _, ok := fieldsMap[k]; !ok {
delete(updateBsonM, k)
}
}
}
// normalize update bson.M
if !svc._containsDollar(updateBsonM) {
if _, ok := updateBsonM["$set"]; !ok {
updateBsonM = bson.M{
"$set": updateBsonM,
}
}
}
return updateBsonM, nil
}
v := reflect.ValueOf(update)
switch v.Kind() {
case reflect.Struct:
if v.CanAddr() {
update = v.Addr().Interface()
return svc._getUpdateBsonM(update, fields)
}
return nil, errors.ErrorModelInvalidType
default:
return nil, errors.ErrorModelInvalidType
}
}
func (svc *BaseService) _getUpdateArtifactUpdate(u interfaces.User) (res bson.M) {
var uid primitive.ObjectID
if u != nil {
uid = u.GetId()
}
return bson.M{
"$set": bson.M{
"_sys.update_ts": time.Now(),
"_sys.update_uid": uid,
},
}
}
func (svc *BaseService) _getUserFromArgs(args ...interface{}) (u interfaces.User) {
return utils.GetUserFromArgs(args...)
}
func (svc *BaseService) _containsDollar(updateBsonM bson.M) (ok bool) {
for k := range updateBsonM {
if strings.HasPrefix(k, "$") {
return true
}
}
return false
}
func NewBaseService(id interfaces.ModelId, opts ...BaseServiceOption) (svc2 interfaces.ModelBaseService) {
// service
svc := &BaseService{
id: id,
}
// apply options
for _, opt := range opts {
opt(svc)
}
// get collection name if not set
if svc.GetCol() == nil {
colName := models2.GetModelColName(id)
svc.SetCol(mongo.GetMongoCol(colName))
}
return svc
}
var store = sync.Map{}
func GetBaseService(id interfaces.ModelId) (svc interfaces.ModelBaseService) {
res, ok := store.Load(id)
if ok {
svc, ok = res.(interfaces.ModelBaseService)
if ok {
return svc
}
}
svc = NewBaseService(id)
store.Store(id, svc)
return svc
}
func GetBaseServiceByColName(id interfaces.ModelId, colName string) (svc interfaces.ModelBaseService) {
res, ok := store.Load(colName)
if ok {
svc, ok = res.(interfaces.ModelBaseService)
if ok {
return svc
}
}
col := mongo.GetMongoCol(colName)
svc = NewBaseService(id, WithBaseServiceCol(col))
store.Store(colName, svc)
return svc
}

92
core/models/service/binder_basic.go
View File

@@ -1,92 +0,0 @@
package service
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
)
func NewBasicBinder(id interfaces.ModelId, fr *mongo.FindResult) (b interfaces.ModelBinder) {
return &BasicBinder{
id: id,
fr: fr,
m: models.NewModelMap(),
}
}
type BasicBinder struct {
id interfaces.ModelId
fr *mongo.FindResult
m *models.ModelMap
}
func (b *BasicBinder) Bind() (res interfaces.Model, err error) {
m := b.m
switch b.id {
case interfaces.ModelIdArtifact:
return b.Process(&m.Artifact)
case interfaces.ModelIdTag:
return b.Process(&m.Tag)
case interfaces.ModelIdNode:
return b.Process(&m.Node)
case interfaces.ModelIdProject:
return b.Process(&m.Project)
case interfaces.ModelIdSpider:
return b.Process(&m.Spider)
case interfaces.ModelIdTask:
return b.Process(&m.Task)
case interfaces.ModelIdJob:
return b.Process(&m.Job)
case interfaces.ModelIdSchedule:
return b.Process(&m.Schedule)
case interfaces.ModelIdUser:
return b.Process(&m.User)
case interfaces.ModelIdSetting:
return b.Process(&m.Setting)
case interfaces.ModelIdToken:
return b.Process(&m.Token)
case interfaces.ModelIdVariable:
return b.Process(&m.Variable)
case interfaces.ModelIdTaskQueue:
return b.Process(&m.TaskQueueItem)
case interfaces.ModelIdTaskStat:
return b.Process(&m.TaskStat)
case interfaces.ModelIdSpiderStat:
return b.Process(&m.SpiderStat)
case interfaces.ModelIdDataSource:
return b.Process(&m.DataSource)
case interfaces.ModelIdDataCollection:
return b.Process(&m.DataCollection)
case interfaces.ModelIdResult:
return b.Process(&m.Result)
case interfaces.ModelIdPassword:
return b.Process(&m.Password)
case interfaces.ModelIdExtraValue:
return b.Process(&m.ExtraValue)
case interfaces.ModelIdGit:
return b.Process(&m.Git)
case interfaces.ModelIdRole:
return b.Process(&m.Role)
case interfaces.ModelIdUserRole:
return b.Process(&m.UserRole)
case interfaces.ModelIdPermission:
return b.Process(&m.Permission)
case interfaces.ModelIdRolePermission:
return b.Process(&m.RolePermission)
case interfaces.ModelIdEnvironment:
return b.Process(&m.Environment)
case interfaces.ModelIdDependencySetting:
return b.Process(&m.DependencySetting)
default:
return nil, errors.ErrorModelInvalidModelId
}
}
func (b *BasicBinder) Process(d interfaces.Model) (res interfaces.Model, err error) {
if err := b.fr.One(d); err != nil {
return nil, err
}
return d, nil
}

93
core/models/service/binder_list.go
View File

@@ -1,93 +0,0 @@
package service
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/trace"
)
func NewListBinder(id interfaces.ModelId, fr *mongo.FindResult) (b interfaces.ModelListBinder) {
return &ListBinder{
id: id,
m: models.NewModelListMap(),
fr: fr,
b: NewBasicBinder(id, fr),
}
}
type ListBinder struct {
id interfaces.ModelId
m *models.ModelListMap
fr *mongo.FindResult
b interfaces.ModelBinder
}
func (b *ListBinder) Bind() (l interfaces.List, err error) {
m := b.m
switch b.id {
case interfaces.ModelIdArtifact:
return b.Process(&m.Artifacts)
case interfaces.ModelIdTag:
return b.Process(&m.Tags)
case interfaces.ModelIdNode:
return b.Process(&m.Nodes)
case interfaces.ModelIdProject:
return b.Process(&m.Projects)
case interfaces.ModelIdSpider:
return b.Process(&m.Spiders)
case interfaces.ModelIdTask:
return b.Process(&m.Tasks)
case interfaces.ModelIdSchedule:
return b.Process(&m.Schedules)
case interfaces.ModelIdUser:
return b.Process(&m.Users)
case interfaces.ModelIdSetting:
return b.Process(&m.Settings)
case interfaces.ModelIdToken:
return b.Process(&m.Tokens)
case interfaces.ModelIdVariable:
return b.Process(&m.Variables)
case interfaces.ModelIdTaskQueue:
return b.Process(&m.TaskQueueItems)
case interfaces.ModelIdTaskStat:
return b.Process(&m.TaskStats)
case interfaces.ModelIdSpiderStat:
return b.Process(&m.SpiderStats)
case interfaces.ModelIdDataSource:
return b.Process(&m.DataSources)
case interfaces.ModelIdDataCollection:
return b.Process(&m.DataCollections)
case interfaces.ModelIdResult:
return b.Process(&m.Results)
case interfaces.ModelIdPassword:
return b.Process(&m.Passwords)
case interfaces.ModelIdExtraValue:
return b.Process(&m.ExtraValues)
case interfaces.ModelIdGit:
return b.Process(&m.Gits)
case interfaces.ModelIdRole:
return b.Process(&m.Roles)
case interfaces.ModelIdUserRole:
return b.Process(&m.UserRoles)
case interfaces.ModelIdPermission:
return b.Process(&m.PermissionList)
case interfaces.ModelIdRolePermission:
return b.Process(&m.RolePermissionList)
case interfaces.ModelIdEnvironment:
return b.Process(&m.Environments)
case interfaces.ModelIdDependencySetting:
return b.Process(&m.DependencySettings)
default:
return l, errors.ErrorModelInvalidModelId
}
}
func (b *ListBinder) Process(d interface{}) (l interfaces.List, err error) {
if err := b.fr.All(d); err != nil {
return l, trace.TraceError(err)
}
return d.(interfaces.List), nil
}

45
core/models/service/data_collection_service.go
View File

@@ -1,45 +0,0 @@
package service
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
models2 "github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
func convertTypeDataCollection(d interface{}, err error) (res *models2.DataCollection, err2 error) {
if err != nil {
return nil, err
}
res, ok := d.(*models2.DataCollection)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return res, nil
}
func (svc *Service) GetDataCollectionById(id primitive.ObjectID) (res *models2.DataCollection, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdDataCollection).GetById(id)
return convertTypeDataCollection(d, err)
}
func (svc *Service) GetDataCollection(query bson.M, opts *mongo.FindOptions) (res *models2.DataCollection, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdDataCollection).Get(query, opts)
return convertTypeDataCollection(d, err)
}
func (svc *Service) GetDataCollectionList(query bson.M, opts *mongo.FindOptions) (res []models2.DataCollection, err error) {
l, err := svc.GetBaseService(interfaces.ModelIdDataCollection).GetList(query, opts)
for _, doc := range l.GetModels() {
d := doc.(*models2.DataCollection)
res = append(res, *d)
}
return res, nil
}
func (svc *Service) GetDataCollectionByName(name string, opts *mongo.FindOptions) (res *models2.DataCollection, err error) {
query := bson.M{"name": name}
return svc.GetDataCollection(query, opts)
}

40
core/models/service/data_source_service.go
View File

@@ -1,40 +0,0 @@
package service
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
models2 "github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
func convertTypeDataSource(d interface{}, err error) (res *models2.DataSource, err2 error) {
if err != nil {
return nil, err
}
res, ok := d.(*models2.DataSource)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return res, nil
}
func (svc *Service) GetDataSourceById(id primitive.ObjectID) (res *models2.DataSource, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdDataSource).GetById(id)
return convertTypeDataSource(d, err)
}
func (svc *Service) GetDataSource(query bson.M, opts *mongo.FindOptions) (res *models2.DataSource, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdDataSource).Get(query, opts)
return convertTypeDataSource(d, err)
}
func (svc *Service) GetDataSourceList(query bson.M, opts *mongo.FindOptions) (res []models2.DataSource, err error) {
l, err := svc.GetBaseService(interfaces.ModelIdDataSource).GetList(query, opts)
for _, doc := range l.GetModels() {
d := doc.(*models2.DataSource)
res = append(res, *d)
}
return res, nil
}

40
core/models/service/dependency_setting_service.go
View File

@@ -1,40 +0,0 @@
package service
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
models2 "github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
func convertTypeDependencySetting(d interface{}, err error) (res *models2.DependencySetting, err2 error) {
if err != nil {
return nil, err
}
res, ok := d.(*models2.DependencySetting)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return res, nil
}
func (svc *Service) GetDependencySettingById(id primitive.ObjectID) (res *models2.DependencySetting, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdDependencySetting).GetById(id)
return convertTypeDependencySetting(d, err)
}
func (svc *Service) GetDependencySetting(query bson.M, opts *mongo.FindOptions) (res *models2.DependencySetting, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdDependencySetting).Get(query, opts)
return convertTypeDependencySetting(d, err)
}
func (svc *Service) GetDependencySettingList(query bson.M, opts *mongo.FindOptions) (res []models2.DependencySetting, err error) {
l, err := svc.GetBaseService(interfaces.ModelIdDependencySetting).GetList(query, opts)
for _, doc := range l.GetModels() {
d := doc.(*models2.DependencySetting)
res = append(res, *d)
}
return res, nil
}

40
core/models/service/environment_service.go
View File

@@ -1,40 +0,0 @@
package service
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
models2 "github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
func convertTypeEnvironment(d interface{}, err error) (res *models2.Environment, err2 error) {
if err != nil {
return nil, err
}
res, ok := d.(*models2.Environment)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return res, nil
}
func (svc *Service) GetEnvironmentById(id primitive.ObjectID) (res *models2.Environment, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdEnvironment).GetById(id)
return convertTypeEnvironment(d, err)
}
func (svc *Service) GetEnvironment(query bson.M, opts *mongo.FindOptions) (res *models2.Environment, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdEnvironment).Get(query, opts)
return convertTypeEnvironment(d, err)
}
func (svc *Service) GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []models2.Environment, err error) {
l, err := svc.GetBaseService(interfaces.ModelIdEnvironment).GetList(query, opts)
for _, doc := range l.GetModels() {
d := doc.(*models2.Environment)
res = append(res, *d)
}
return res, nil
}

45
core/models/service/extra_value_service.go
View File

@@ -1,45 +0,0 @@
package service
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
func convertTypeExtraValue(d interface{}, err error) (res *models.ExtraValue, err2 error) {
if err != nil {
return nil, err
}
res, ok := d.(*models.ExtraValue)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return res, nil
}
func (svc *Service) GetExtraValueById(id primitive.ObjectID) (res *models.ExtraValue, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdExtraValue).GetById(id)
return convertTypeExtraValue(d, err)
}
func (svc *Service) GetExtraValue(query bson.M, opts *mongo.FindOptions) (res *models.ExtraValue, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdExtraValue).Get(query, opts)
return convertTypeExtraValue(d, err)
}
func (svc *Service) GetExtraValueList(query bson.M, opts *mongo.FindOptions) (res []models.ExtraValue, err error) {
l, err := svc.GetBaseService(interfaces.ModelIdExtraValue).GetList(query, opts)
for _, doc := range l.GetModels() {
d := doc.(*models.ExtraValue)
res = append(res, *d)
}
return res, nil
}
func (svc *Service) GetExtraValueByObjectIdModel(oid primitive.ObjectID, m string, opts *mongo.FindOptions) (res *models.ExtraValue, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdExtraValue).Get(bson.M{"oid": oid, "m": m}, opts)
return convertTypeExtraValue(d, err)
}

43
core/models/service/git_service.go
View File

@@ -1,43 +0,0 @@
package service
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
models2 "github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
func convertTypeGit(d interface{}, err error) (res *models2.Git, err2 error) {
if err != nil {
return nil, err
}
res, ok := d.(*models2.Git)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return res, nil
}
func (svc *Service) GetGitById(id primitive.ObjectID) (res *models2.Git, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdGit).GetById(id)
return convertTypeGit(d, err)
}
func (svc *Service) GetGit(query bson.M, opts *mongo.FindOptions) (res *models2.Git, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdGit).Get(query, opts)
return convertTypeGit(d, err)
}
func (svc *Service) GetGitList(query bson.M, opts *mongo.FindOptions) (res []models2.Git, err error) {
l, err := svc.GetBaseService(interfaces.ModelIdGit).GetList(query, opts)
if l == nil {
return nil, nil
}
for _, doc := range l.GetModels() {
d := doc.(*models2.Git)
res = append(res, *d)
}
return res, nil
}

105
core/models/service/interface.go
View File

@@ -1,105 +0,0 @@
package service
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type ModelService interface {
interfaces.ModelService
DropAll() (err error)
GetNodeById(id primitive.ObjectID) (res *models.Node, err error)
GetNode(query bson.M, opts *mongo.FindOptions) (res *models.Node, err error)
GetNodeList(query bson.M, opts *mongo.FindOptions) (res []models.Node, err error)
GetNodeByKey(key string, opts *mongo.FindOptions) (res *models.Node, err error)
GetProjectById(id primitive.ObjectID) (res *models.Project, err error)
GetProject(query bson.M, opts *mongo.FindOptions) (res *models.Project, err error)
GetProjectList(query bson.M, opts *mongo.FindOptions) (res []models.Project, err error)
GetArtifactById(id primitive.ObjectID) (res *models.Artifact, err error)
GetArtifact(query bson.M, opts *mongo.FindOptions) (res *models.Artifact, err error)
GetArtifactList(query bson.M, opts *mongo.FindOptions) (res []models.Artifact, err error)
GetTagById(id primitive.ObjectID) (res *models.Tag, err error)
GetTag(query bson.M, opts *mongo.FindOptions) (res *models.Tag, err error)
GetTagList(query bson.M, opts *mongo.FindOptions) (res []models.Tag, err error)
GetTagIds(colName string, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error)
UpdateTagsById(colName string, id primitive.ObjectID, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error)
UpdateTags(colName string, query bson.M, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error)
GetJobById(id primitive.ObjectID) (res *models.Job, err error)
GetJob(query bson.M, opts *mongo.FindOptions) (res *models.Job, err error)
GetJobList(query bson.M, opts *mongo.FindOptions) (res []models.Job, err error)
GetScheduleById(id primitive.ObjectID) (res *models.Schedule, err error)
GetSchedule(query bson.M, opts *mongo.FindOptions) (res *models.Schedule, err error)
GetScheduleList(query bson.M, opts *mongo.FindOptions) (res []models.Schedule, err error)
GetUserById(id primitive.ObjectID) (res *models.User, err error)
GetUser(query bson.M, opts *mongo.FindOptions) (res *models.User, err error)
GetUserList(query bson.M, opts *mongo.FindOptions) (res []models.User, err error)
GetUserByUsername(username string, opts *mongo.FindOptions) (res *models.User, err error)
GetUserByUsernameWithPassword(username string, opts *mongo.FindOptions) (res *models.User, err error)
GetSettingById(id primitive.ObjectID) (res *models.Setting, err error)
GetSetting(query bson.M, opts *mongo.FindOptions) (res *models.Setting, err error)
GetSettingList(query bson.M, opts *mongo.FindOptions) (res []models.Setting, err error)
GetSettingByKey(key string, opts *mongo.FindOptions) (res *models.Setting, err error)
GetSpiderById(id primitive.ObjectID) (res *models.Spider, err error)
GetSpider(query bson.M, opts *mongo.FindOptions) (res *models.Spider, err error)
GetSpiderList(query bson.M, opts *mongo.FindOptions) (res []models.Spider, err error)
GetTaskById(id primitive.ObjectID) (res *models.Task, err error)
GetTask(query bson.M, opts *mongo.FindOptions) (res *models.Task, err error)
GetTaskList(query bson.M, opts *mongo.FindOptions) (res []models.Task, err error)
GetTokenById(id primitive.ObjectID) (res *models.Token, err error)
GetToken(query bson.M, opts *mongo.FindOptions) (res *models.Token, err error)
GetTokenList(query bson.M, opts *mongo.FindOptions) (res []models.Token, err error)
GetVariableById(id primitive.ObjectID) (res *models.Variable, err error)
GetVariable(query bson.M, opts *mongo.FindOptions) (res *models.Variable, err error)
GetVariableList(query bson.M, opts *mongo.FindOptions) (res []models.Variable, err error)
GetVariableByKey(key string, opts *mongo.FindOptions) (res *models.Variable, err error)
GetTaskQueueItemById(id primitive.ObjectID) (res *models.TaskQueueItem, err error)
GetTaskQueueItem(query bson.M, opts *mongo.FindOptions) (res *models.TaskQueueItem, err error)
GetTaskQueueItemList(query bson.M, opts *mongo.FindOptions) (res []models.TaskQueueItem, err error)
GetTaskStatById(id primitive.ObjectID) (res *models.TaskStat, err error)
GetTaskStat(query bson.M, opts *mongo.FindOptions) (res *models.TaskStat, err error)
GetTaskStatList(query bson.M, opts *mongo.FindOptions) (res []models.TaskStat, err error)
GetSpiderStatById(id primitive.ObjectID) (res *models.SpiderStat, err error)
GetSpiderStat(query bson.M, opts *mongo.FindOptions) (res *models.SpiderStat, err error)
GetSpiderStatList(query bson.M, opts *mongo.FindOptions) (res []models.SpiderStat, err error)
GetDataSourceById(id primitive.ObjectID) (res *models.DataSource, err error)
GetDataSource(query bson.M, opts *mongo.FindOptions) (res *models.DataSource, err error)
GetDataSourceList(query bson.M, opts *mongo.FindOptions) (res []models.DataSource, err error)
GetDataCollectionById(id primitive.ObjectID) (res *models.DataCollection, err error)
GetDataCollection(query bson.M, opts *mongo.FindOptions) (res *models.DataCollection, err error)
GetDataCollectionList(query bson.M, opts *mongo.FindOptions) (res []models.DataCollection, err error)
GetDataCollectionByName(name string, opts *mongo.FindOptions) (res *models.DataCollection, err error)
GetPasswordById(id primitive.ObjectID) (res *models.Password, err error)
GetPassword(query bson.M, opts *mongo.FindOptions) (res *models.Password, err error)
GetPasswordList(query bson.M, opts *mongo.FindOptions) (res []models.Password, err error)
GetExtraValueById(id primitive.ObjectID) (res *models.ExtraValue, err error)
GetExtraValue(query bson.M, opts *mongo.FindOptions) (res *models.ExtraValue, err error)
GetExtraValueList(query bson.M, opts *mongo.FindOptions) (res []models.ExtraValue, err error)
GetExtraValueByObjectIdModel(oid primitive.ObjectID, m string, opts *mongo.FindOptions) (res *models.ExtraValue, err error)
GetGitById(id primitive.ObjectID) (res *models.Git, err error)
GetGit(query bson.M, opts *mongo.FindOptions) (res *models.Git, err error)
GetGitList(query bson.M, opts *mongo.FindOptions) (res []models.Git, err error)
GetRoleById(id primitive.ObjectID) (res *models.Role, err error)
GetRole(query bson.M, opts *mongo.FindOptions) (res *models.Role, err error)
GetRoleList(query bson.M, opts *mongo.FindOptions) (res []models.Role, err error)
GetRoleByName(name string, opts *mongo.FindOptions) (res *models.Role, err error)
GetRoleByKey(key string, opts *mongo.FindOptions) (res *models.Role, err error)
GetUserRoleById(id primitive.ObjectID) (res *models.UserRole, err error)
GetUserRole(query bson.M, opts *mongo.FindOptions) (res *models.UserRole, err error)
GetUserRoleList(query bson.M, opts *mongo.FindOptions) (res []models.UserRole, err error)
GetUserRoleListByUserId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.UserRole, err error)
GetUserRoleListByRoleId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.UserRole, err error)
GetPermissionById(id primitive.ObjectID) (res *models.Permission, err error)
GetPermission(query bson.M, opts *mongo.FindOptions) (res *models.Permission, err error)
GetPermissionList(query bson.M, opts *mongo.FindOptions) (res []models.Permission, err error)
GetPermissionByKey(key string, opts *mongo.FindOptions) (res *models.Permission, err error)
GetRolePermission(query bson.M, opts *mongo.FindOptions) (res *models.RolePermission, err error)
GetRolePermissionList(query bson.M, opts *mongo.FindOptions) (res []models.RolePermission, err error)
GetRolePermissionListByRoleId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.RolePermission, err error)
GetRolePermissionListByPermissionId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.RolePermission, err error)
GetEnvironmentById(id primitive.ObjectID) (res *models.Environment, err error)
GetEnvironment(query bson.M, opts *mongo.FindOptions) (res *models.Environment, err error)
GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []models.Environment, err error)
}

40
core/models/service/job_service.go
View File

@@ -1,40 +0,0 @@
package service
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
models2 "github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
func convertTypeJob(d interface{}, err error) (res *models2.Job, err2 error) {
if err != nil {
return nil, err
}
res, ok := d.(*models2.Job)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return res, nil
}
func (svc *Service) GetJobById(id primitive.ObjectID) (res *models2.Job, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdJob).GetById(id)
return convertTypeJob(d, err)
}
func (svc *Service) GetJob(query bson.M, opts *mongo.FindOptions) (res *models2.Job, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdJob).Get(query, opts)
return convertTypeJob(d, err)
}
func (svc *Service) GetJobList(query bson.M, opts *mongo.FindOptions) (res []models2.Job, err error) {
l, err := svc.GetBaseService(interfaces.ModelIdJob).GetList(query, opts)
for _, doc := range l.GetModels() {
d := doc.(*models2.Job)
res = append(res, *d)
}
return res, nil
}

45
core/models/service/node_service.go
View File

@@ -1,45 +0,0 @@
package service
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
models2 "github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
func convertTypeNode(d interface{}, err error) (res *models2.Node, err2 error) {
if err != nil {
return nil, err
}
res, ok := d.(*models2.Node)
if !ok {
return nil, errors.ErrorModelInvalidType
}
return res, nil
}
func (svc *Service) GetNodeById(id primitive.ObjectID) (res *models2.Node, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdNode).GetById(id)
return convertTypeNode(d, err)
}
func (svc *Service) GetNode(query bson.M, opts *mongo.FindOptions) (res *models2.Node, err error) {
d, err := svc.GetBaseService(interfaces.ModelIdNode).Get(query, opts)
return convertTypeNode(d, err)
}
func (svc *Service) GetNodeList(query bson.M, opts *mongo.FindOptions) (res []models2.Node, err error) {
l, err := svc.GetBaseService(interfaces.ModelIdNode).GetList(query, opts)
for _, doc := range l.GetModels() {
d := doc.(*models2.Node)
res = append(res, *d)
}
return res, nil
}
func (svc *Service) GetNodeByKey(key string, opts *mongo.FindOptions) (res *models2.Node, err error) {
query := bson.M{"key": key}
return svc.GetNode(query, opts)
}

25
core/models/service/options.go
View File

@@ -1,25 +0,0 @@
package service
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/db/mongo"
)
type Option func(ModelService)
type BaseServiceOption func(svc interfaces.ModelBaseService)
func WithBaseServiceModelId(id interfaces.ModelId) BaseServiceOption {
return func(svc interfaces.ModelBaseService) {
svc.SetModelId(id)
}
}
func WithBaseServiceCol(col *mongo.Col) BaseServiceOption {
return func(svc interfaces.ModelBaseService) {
_svc, ok := svc.(*BaseService)
if ok {
_svc.SetCol(col)
}
}
}

Some files were not shown because too many files have changed in this diff Show More