git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: removed unnecessary code

Browse Source
This commit is contained in:
Marvin Zhang
2024-07-11 12:45:20 +08:00
parent 13596436cd
commit f2f298d8f8
69 changed files with 469 additions and 1142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
core/controllers/base.go
View File

@@ -1,69 +0,0 @@
package controllers
import "github.com/gin-gonic/gin"
const (
ControllerIdNode = iota << 1
ControllerIdProject
ControllerIdSpider
ControllerIdTask
ControllerIdJob
ControllerIdSchedule
ControllerIdUser
ControllerIdSetting
ControllerIdToken
ControllerIdVariable
ControllerIdTag
ControllerIdLogin
ControllerIdColor
ControllerIdDataSource
ControllerIdDataCollection
ControllerIdResult
ControllerIdStats
ControllerIdFiler
ControllerIdGit
ControllerIdRole
ControllerIdPermission
ControllerIdExport
ControllerIdNotification
ControllerIdFilter
ControllerIdEnvironment
ControllerIdSync
ControllerIdVersion
ControllerIdI18n
ControllerIdSystemInfo
ControllerIdDemo
)
type ControllerId int
type BasicController interface {
Get(c *gin.Context)
Post(c *gin.Context)
Put(c *gin.Context)
Delete(c *gin.Context)
}
type ListController interface {
BasicController
GetList(c *gin.Context)
PutList(c *gin.Context)
PostList(c *gin.Context)
DeleteList(c *gin.Context)
}
type Action struct {
Method string
Path string
HandlerFunc gin.HandlerFunc
}
type ActionController interface {
Actions() (actions []Action)
}
type ListActionController interface {
ListController
ActionController
}

14
core/controllers/binder.go
View File

@@ -1,14 +0,0 @@
package controllers
import (
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/gin-gonic/gin"
)
type BinderInterface interface {
Bind(c *gin.Context) (res interfaces.Model, err error)
BindList(c *gin.Context) (res []interfaces.Model, err error)
BindBatchRequestPayload(c *gin.Context) (payload entity.BatchRequestPayload, err error)
BindBatchRequestPayloadWithStringData(c *gin.Context) (payload entity.BatchRequestPayloadWithStringData, res interfaces.Model, err error)
}

208
core/controllers/binder_json.go
View File

@@ -1,208 +0,0 @@
package controllers
import (
"encoding/json"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/gin-gonic/gin"
)
func NewJsonBinder(id ControllerId) (b *JsonBinder) {
return &JsonBinder{
id: id,
}
}
type JsonBinder struct {
id ControllerId
}
func (b *JsonBinder) Bind(c *gin.Context) (res interfaces.Model, err error) {
// declare
m := models.NewModelMap()
switch b.id {
case ControllerIdNode:
err = c.ShouldBindJSON(&m.Node)
return &m.Node, err
case ControllerIdProject:
err = c.ShouldBindJSON(&m.Project)
return &m.Project, err
case ControllerIdSpider:
err = c.ShouldBindJSON(&m.Spider)
return &m.Spider, err
case ControllerIdTask:
err = c.ShouldBindJSON(&m.Task)
return &m.Task, err
case ControllerIdJob:
err = c.ShouldBindJSON(&m.Job)
return &m.Job, err
case ControllerIdSchedule:
err = c.ShouldBindJSON(&m.Schedule)
return &m.Schedule, err
case ControllerIdUser:
err = c.ShouldBindJSON(&m.User)
return &m.User, nil
case ControllerIdSetting:
err = c.ShouldBindJSON(&m.Setting)
return &m.Setting, nil
case ControllerIdToken:
err = c.ShouldBindJSON(&m.Token)
return &m.Token, nil
case ControllerIdVariable:
err = c.ShouldBindJSON(&m.Variable)
return &m.Variable, nil
case ControllerIdTag:
err = c.ShouldBindJSON(&m.Tag)
return &m.Tag, nil
case ControllerIdDataSource:
err = c.ShouldBindJSON(&m.DataSource)
return &m.DataSource, nil
case ControllerIdDataCollection:
err = c.ShouldBindJSON(&m.DataCollection)
return &m.DataCollection, nil
case ControllerIdGit:
err = c.ShouldBindJSON(&m.Git)
return &m.Git, nil
case ControllerIdRole:
err = c.ShouldBindJSON(&m.Role)
return &m.Role, nil
case ControllerIdPermission:
err = c.ShouldBindJSON(&m.Permission)
return &m.Permission, nil
case ControllerIdEnvironment:
err = c.ShouldBindJSON(&m.Environment)
return &m.Environment, nil
default:
return nil, errors.ErrorControllerInvalidControllerId
}
}
func (b *JsonBinder) BindList(c *gin.Context) (res interface{}, err error) {
// declare
m := models.NewModelListMap()
// bind
switch b.id {
case ControllerIdNode:
err = c.ShouldBindJSON(&m.Nodes)
return m.Nodes, err
case ControllerIdProject:
err = c.ShouldBindJSON(&m.Projects)
return m.Projects, err
case ControllerIdSpider:
err = c.ShouldBindJSON(&m.Spiders)
return m.Spiders, err
case ControllerIdTask:
err = c.ShouldBindJSON(&m.Tasks)
return m.Tasks, err
case ControllerIdJob:
err = c.ShouldBindJSON(&m.Jobs)
return m.Jobs, err
case ControllerIdSchedule:
err = c.ShouldBindJSON(&m.Schedules)
return m.Schedules, err
case ControllerIdUser:
err = c.ShouldBindJSON(&m.Users)
return m.Users, nil
case ControllerIdSetting:
err = c.ShouldBindJSON(&m.Settings)
return m.Settings, nil
case ControllerIdToken:
err = c.ShouldBindJSON(&m.Tokens)
return m.Tokens, nil
case ControllerIdVariable:
err = c.ShouldBindJSON(&m.Variables)
return m.Variables, nil
case ControllerIdTag:
err = c.ShouldBindJSON(&m.Tags)
return m.Tags, nil
case ControllerIdDataSource:
err = c.ShouldBindJSON(&m.DataSources)
return m.DataSources, nil
case ControllerIdDataCollection:
err = c.ShouldBindJSON(&m.DataCollections)
return m.DataCollections, nil
case ControllerIdGit:
err = c.ShouldBindJSON(&m.Gits)
return m.Gits, nil
case ControllerIdRole:
err = c.ShouldBindJSON(&m.Roles)
return m.Roles, nil
case ControllerIdEnvironment:
err = c.ShouldBindJSON(&m.Environments)
return m.Environments, nil
default:
return nil, errors.ErrorControllerInvalidControllerId
}
}
func (b *JsonBinder) BindBatchRequestPayload(c *gin.Context) (payload entity.BatchRequestPayload, err error) {
if err := c.ShouldBindJSON(&payload); err != nil {
return payload, err
}
return payload, nil
}
func (b *JsonBinder) BindBatchRequestPayloadWithStringData(c *gin.Context) (payload entity.BatchRequestPayloadWithStringData, res interfaces.Model, err error) {
// declare
m := models.NewModelMap()
// bind
if err := c.ShouldBindJSON(&payload); err != nil {
return payload, nil, err
}
// validate
if len(payload.Ids) == 0 ||
len(payload.Fields) == 0 {
return payload, nil, errors.ErrorControllerRequestPayloadInvalid
}
// unmarshall
switch b.id {
case ControllerIdNode:
err = json.Unmarshal([]byte(payload.Data), &m.Node)
return payload, &m.Node, err
case ControllerIdProject:
err = json.Unmarshal([]byte(payload.Data), &m.Project)
return payload, &m.Project, err
case ControllerIdSpider:
err = json.Unmarshal([]byte(payload.Data), &m.Spider)
return payload, &m.Spider, err
case ControllerIdTask:
err = json.Unmarshal([]byte(payload.Data), &m.Task)
return payload, &m.Task, err
case ControllerIdJob:
err = json.Unmarshal([]byte(payload.Data), &m.Job)
return payload, &m.Job, err
case ControllerIdSchedule:
err = json.Unmarshal([]byte(payload.Data), &m.Schedule)
return payload, &m.Schedule, err
case ControllerIdUser:
err = json.Unmarshal([]byte(payload.Data), &m.User)
return payload, &m.User, err
case ControllerIdSetting:
err = json.Unmarshal([]byte(payload.Data), &m.Setting)
return payload, &m.Setting, err
case ControllerIdToken:
err = json.Unmarshal([]byte(payload.Data), &m.Token)
return payload, &m.Token, err
case ControllerIdVariable:
err = json.Unmarshal([]byte(payload.Data), &m.Variable)
return payload, &m.Variable, err
case ControllerIdDataSource:
err = json.Unmarshal([]byte(payload.Data), &m.DataSource)
return payload, &m.DataSource, err
case ControllerIdDataCollection:
err = json.Unmarshal([]byte(payload.Data), &m.DataCollection)
return payload, &m.DataCollection, err
case ControllerIdEnvironment:
err = json.Unmarshal([]byte(payload.Data), &m.Environment)
return payload, &m.Environment, err
default:
return payload, nil, errors.ErrorControllerInvalidControllerId
}
}

2
core/controllers/data_source_v2.go
View File

@@ -3,7 +3,7 @@ package controllers
import (
"github.com/crawlab-team/crawlab/core/ds"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson/primitive"

17
core/controllers/delegate_action.go
View File

@@ -1,17 +0,0 @@
package controllers
func NewActionControllerDelegate(id ControllerId, actions []Action) (d *ActionControllerDelegate) {
return &ActionControllerDelegate{
id: id,
actions: actions,
}
}
type ActionControllerDelegate struct {
id ControllerId
actions []Action
}
func (ctr *ActionControllerDelegate) Actions() (actions []Action) {
return ctr.actions
}

99
core/controllers/delegate_basic.go
View File

@@ -1,99 +0,0 @@
package controllers
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
delegate2 "github.com/crawlab-team/crawlab/core/models/delegate"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson/primitive"
mongo2 "go.mongodb.org/mongo-driver/mongo"
)
func NewBasicControllerDelegate(id ControllerId, svc interfaces.ModelBaseService) (d *BasicControllerDelegate) {
return &BasicControllerDelegate{
id: id,
svc: svc,
}
}
type BasicControllerDelegate struct {
id ControllerId
svc interfaces.ModelBaseService
}
func (d *BasicControllerDelegate) Get(c *gin.Context) {
id, err := primitive.ObjectIDFromHex(c.Param("id"))
if err != nil {
HandleErrorBadRequest(c, err)
return
}
doc, err := d.svc.GetById(id)
if err == mongo2.ErrNoDocuments {
HandleErrorNotFound(c, err)
return
}
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
HandleSuccessWithData(c, doc)
}
func (d *BasicControllerDelegate) Post(c *gin.Context) {
doc, err := NewJsonBinder(d.id).Bind(c)
if err != nil {
HandleErrorBadRequest(c, err)
return
}
if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Add(); err != nil {
HandleErrorInternalServerError(c, err)
return
}
HandleSuccessWithData(c, doc)
}
func (d *BasicControllerDelegate) Put(c *gin.Context) {
id, err := primitive.ObjectIDFromHex(c.Param("id"))
if err != nil {
HandleErrorBadRequest(c, err)
return
}
doc, err := NewJsonBinder(d.id).Bind(c)
if err != nil {
HandleErrorBadRequest(c, err)
return
}
if doc.GetId() != id {
HandleErrorBadRequest(c, errors.ErrorHttpBadRequest)
return
}
_, err = d.svc.GetById(id)
if err != nil {
HandleErrorNotFound(c, err)
return
}
if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Save(); err != nil {
HandleErrorInternalServerError(c, err)
return
}
HandleSuccessWithData(c, doc)
}
func (d *BasicControllerDelegate) Delete(c *gin.Context) {
id := c.Param("id")
oid, err := primitive.ObjectIDFromHex(id)
if err != nil {
HandleErrorBadRequest(c, err)
return
}
doc, err := d.svc.GetById(oid)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Delete(); err != nil {
HandleErrorInternalServerError(c, err)
return
}
HandleSuccess(c)
}

222
core/controllers/delegate_list.go
View File

@@ -1,222 +0,0 @@
package controllers
import (
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/delegate"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/crawlab/trace"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
mongo2 "go.mongodb.org/mongo-driver/mongo"
"reflect"
"time"
)
func NewListControllerDelegate(id ControllerId, svc interfaces.ModelBaseService) (d *ListControllerDelegate) {
if svc == nil {
panic(errors.ErrorControllerNoModelService)
}
return &ListControllerDelegate{
id: id,
svc: svc,
bc: NewBasicControllerDelegate(id, svc),
}
}
type ListControllerDelegate struct {
id ControllerId
svc interfaces.ModelBaseService
bc BasicController
}
func (d *ListControllerDelegate) Get(c *gin.Context) {
d.bc.Get(c)
}
func (d *ListControllerDelegate) Post(c *gin.Context) {
d.bc.Post(c)
}
func (d *ListControllerDelegate) Put(c *gin.Context) {
d.bc.Put(c)
}
func (d *ListControllerDelegate) Delete(c *gin.Context) {
d.bc.Delete(c)
}
func (d *ListControllerDelegate) GetList(c *gin.Context) {
// get all if query field "all" is set true
all := MustGetFilterAll(c)
if all {
d.getAll(c)
return
}
// get list and total
l, total, err := d.getList(c)
if err != nil {
return
}
// response
HandleSuccessWithListData(c, l, total)
}
func (d *ListControllerDelegate) PostList(c *gin.Context) {
// bind
docs, err := NewJsonBinder(d.id).BindList(c)
if err != nil {
HandleErrorBadRequest(c, err)
return
}
// success ids
var ids []primitive.ObjectID
// reflect
switch reflect.TypeOf(docs).Kind() {
case reflect.Slice, reflect.Array:
s := reflect.ValueOf(docs)
for i := 0; i < s.Len(); i++ {
item := s.Index(i)
if !item.CanAddr() {
HandleErrorInternalServerError(c, errors.ErrorModelInvalidType)
return
}
ptr := item.Addr()
doc, ok := ptr.Interface().(interfaces.Model)
if !ok {
HandleErrorInternalServerError(c, errors.ErrorModelInvalidType)
return
}
if err := delegate.NewModelDelegate(doc, GetUserFromContext(c)).Add(); err != nil {
_ = trace.TraceError(err)
continue
}
ids = append(ids, doc.GetId())
}
}
// check
items, err := utils.GetArrayItems(docs)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
if len(ids) < len(items) {
HandleErrorInternalServerError(c, errors.ErrorControllerAddError)
return
}
// success
HandleSuccessWithData(c, docs)
}
func (d *ListControllerDelegate) PutList(c *gin.Context) {
payload, doc, err := NewJsonBinder(d.id).BindBatchRequestPayloadWithStringData(c)
if err != nil {
HandleErrorBadRequest(c, err)
return
}
// query
query := bson.M{
"_id": bson.M{
"$in": payload.Ids,
},
}
// update
if err := d.svc.UpdateDoc(query, doc, payload.Fields); err != nil {
HandleErrorInternalServerError(c, err)
return
}
HandleSuccess(c)
}
func (d *ListControllerDelegate) DeleteList(c *gin.Context) {
payload, err := NewJsonBinder(d.id).BindBatchRequestPayload(c)
if err != nil {
HandleErrorBadRequest(c, err)
return
}
if err := d.svc.DeleteList(bson.M{
"_id": bson.M{
"$in": payload.Ids,
},
}); err != nil {
HandleErrorInternalServerError(c, err)
return
}
HandleSuccess(c)
}
func (d *ListControllerDelegate) getAll(c *gin.Context) {
// get list
tic := time.Now()
log.Debugf("getAll -> d.svc.GetMany:start")
list, err := d.svc.GetList(nil, &mongo.FindOptions{
Sort: bson.D{{"_id", -1}},
})
if err != nil {
if err == mongo2.ErrNoDocuments {
HandleErrorNotFound(c, err)
} else {
HandleErrorInternalServerError(c, err)
}
return
}
log.Debugf("getAll -> d.svc.GetMany:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds())
tic = time.Now()
// total count
tic = time.Now()
log.Debugf("getAll -> d.svc.Count:start")
total, err := d.svc.Count(nil)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
log.Debugf("getAll -> d.svc.Count:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds())
// response
HandleSuccessWithListData(c, list, total)
}
func (d *ListControllerDelegate) getList(c *gin.Context) (l interfaces.List, total int, err error) {
// params
pagination := MustGetPagination(c)
query := MustGetFilterQuery(c)
sort := MustGetSortOption(c)
// get list
l, err = d.svc.GetList(query, &mongo.FindOptions{
Sort: sort,
Skip: pagination.Size * (pagination.Page - 1),
Limit: pagination.Size,
})
if err != nil {
if err.Error() == mongo2.ErrNoDocuments.Error() {
HandleSuccessWithListData(c, nil, 0)
} else {
HandleErrorInternalServerError(c, err)
}
return
}
// total count
total, err = d.svc.Count(query)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
return l, total, nil
}

17
core/controllers/delegate_list_action.go
View File

@@ -1,17 +0,0 @@
package controllers
import (
"github.com/crawlab-team/crawlab/core/interfaces"
)
func NewListPostActionControllerDelegate(id ControllerId, svc interfaces.ModelBaseService, actions []Action) (d *ListActionControllerDelegate) {
return &ListActionControllerDelegate{
NewListControllerDelegate(id, svc),
NewActionControllerDelegate(id, actions),
}
}
type ListActionControllerDelegate struct {
ListController
ActionController
}

57
core/controllers/environment.go
View File

@@ -1,57 +0,0 @@
package controllers
import (
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/service"
)
var EnvironmentController *environmentController
var EnvironmentActions []Action
type environmentController struct {
ListActionControllerDelegate
d ListActionControllerDelegate
ctx *environmentContext
}
type environmentContext struct {
modelSvc service.ModelService
userSvc interfaces.UserService
}
func newEnvironmentContext() *environmentContext {
// context
ctx := &environmentContext{}
// dependency injection
if err := container.GetContainer().Invoke(func(
modelSvc service.ModelService,
userSvc interfaces.UserService,
) {
ctx.modelSvc = modelSvc
ctx.userSvc = userSvc
}); err != nil {
panic(err)
}
return ctx
}
func newEnvironmentController() *environmentController {
modelSvc, err := service.GetService()
if err != nil {
panic(err)
}
ctr := NewListPostActionControllerDelegate(ControllerIdEnvironment, modelSvc.GetBaseService(interfaces.ModelIdEnvironment), EnvironmentActions)
d := NewListPostActionControllerDelegate(ControllerIdEnvironment, modelSvc.GetBaseService(interfaces.ModelIdEnvironment), EnvironmentActions)
ctx := newEnvironmentContext()
return &environmentController{
ListActionControllerDelegate: *ctr,
d: *d,
ctx: ctx,
}
}

14
core/controllers/project_v2.go
View File

@@ -2,7 +2,7 @@ package controllers
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/gin-gonic/gin"
@@ -15,7 +15,7 @@ func GetProjectList(c *gin.Context) {
// get all list
all := MustGetFilterAll(c)
if all {
NewControllerV2[models.ProjectV2]().getAll(c)
NewControllerV2[models2.ProjectV2]().getAll(c)
return
}
@@ -25,7 +25,7 @@ func GetProjectList(c *gin.Context) {
sort := MustGetSortOption(c)
// get list
projects, err := service.NewModelServiceV2[models.ProjectV2]().GetMany(query, &mongo.FindOptions{
projects, err := service.NewModelServiceV2[models2.ProjectV2]().GetMany(query, &mongo.FindOptions{
Sort: sort,
Skip: pagination.Size * (pagination.Page - 1),
Limit: pagination.Size,
@@ -37,12 +37,12 @@ func GetProjectList(c *gin.Context) {
return
}
if len(projects) == 0 {
HandleSuccessWithListData(c, []models.ProjectV2{}, 0)
HandleSuccessWithListData(c, []models2.ProjectV2{}, 0)
return
}
// total count
total, err := service.NewModelServiceV2[models.ProjectV2]().Count(query)
total, err := service.NewModelServiceV2[models2.ProjectV2]().Count(query)
if err != nil {
HandleErrorInternalServerError(c, err)
return
@@ -61,7 +61,7 @@ func GetProjectList(c *gin.Context) {
}
// spiders
spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(bson.M{
spiders, err := service.NewModelServiceV2[models2.SpiderV2]().GetMany(bson.M{
"project_id": bson.M{
"$in": ids,
},
@@ -80,7 +80,7 @@ func GetProjectList(c *gin.Context) {
}
// assign
var data []models.ProjectV2
var data []models2.ProjectV2
for _, p := range projects {
p.Spiders = cache[p.Id]
data = append(data, p)

10
core/controllers/result_v2.go
View File

@@ -1,7 +1,7 @@
package controllers
import (
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/result"
"github.com/crawlab-team/crawlab/core/utils"
@@ -32,17 +32,17 @@ func GetResultList(c *gin.Context) {
}
// data collection
dc, err := service.NewModelServiceV2[models.DataCollectionV2]().GetById(dcId)
dc, err := service.NewModelServiceV2[models2.DataCollectionV2]().GetById(dcId)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// data source
ds, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(dsId)
ds, err := service.NewModelServiceV2[models2.DataSourceV2]().GetById(dsId)
if err != nil {
if err.Error() == mongo2.ErrNoDocuments.Error() {
ds = &models.DataSourceV2{}
ds = &models2.DataSourceV2{}
} else {
HandleErrorInternalServerError(c, err)
return
@@ -54,7 +54,7 @@ func GetResultList(c *gin.Context) {
"col_id": dc.Id,
"data_source_id": ds.Id,
}
s, err := service.NewModelServiceV2[models.SpiderV2]().GetOne(sq, nil)
s, err := service.NewModelServiceV2[models2.SpiderV2]().GetOne(sq, nil)
if err != nil {
HandleErrorInternalServerError(c, err)
return

22
core/controllers/router_v2.go
View File

@@ -2,7 +2,7 @@ package controllers
import (
"github.com/crawlab-team/crawlab/core/middlewares"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/gin-gonic/gin"
"net/http"
)
@@ -55,8 +55,8 @@ func InitRoutes(app *gin.Engine) (err error) {
// routes groups
groups := NewRouterGroups(app)
RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models.DataCollectionV2]())
RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models.DataSourceV2]([]Action{
RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models2.DataCollectionV2]())
RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models2.DataSourceV2]([]Action{
{
Method: http.MethodPost,
Path: "",
@@ -73,16 +73,16 @@ func InitRoutes(app *gin.Engine) (err error) {
HandlerFunc: PostDataSourceChangePassword,
},
}...))
RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models.EnvironmentV2]())
RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]())
RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models.ProjectV2]([]Action{
RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models2.EnvironmentV2]())
RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models2.NodeV2]())
RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models2.ProjectV2]([]Action{
{
Method: http.MethodGet,
Path: "",
HandlerFunc: GetProjectList,
},
}...))
RegisterController(groups.AuthGroup, "/schedules", NewControllerV2[models.ScheduleV2]([]Action{
RegisterController(groups.AuthGroup, "/schedules", NewControllerV2[models2.ScheduleV2]([]Action{
{
Method: http.MethodPost,
Path: "",
@@ -104,7 +104,7 @@ func InitRoutes(app *gin.Engine) (err error) {
HandlerFunc: PostScheduleDisable,
},
}...))
RegisterController(groups.AuthGroup, "/spiders", NewControllerV2[models.SpiderV2]([]Action{
RegisterController(groups.AuthGroup, "/spiders", NewControllerV2[models2.SpiderV2]([]Action{
{
Method: http.MethodGet,
Path: "/:id",
@@ -202,7 +202,7 @@ func InitRoutes(app *gin.Engine) (err error) {
HandlerFunc: PostSpiderDataSource,
},
}...))
RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models.TaskV2]([]Action{
RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models2.TaskV2]([]Action{
{
Method: http.MethodGet,
Path: "/:id",
@@ -249,14 +249,14 @@ func InitRoutes(app *gin.Engine) (err error) {
HandlerFunc: GetTaskData,
},
}...))
RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models.TokenV2]([]Action{
RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models2.TokenV2]([]Action{
{
Method: http.MethodPost,
Path: "",
HandlerFunc: PostToken,
},
}...))
RegisterController(groups.AuthGroup, "/users", NewControllerV2[models.UserV2]([]Action{
RegisterController(groups.AuthGroup, "/users", NewControllerV2[models2.UserV2]([]Action{
{
Method: http.MethodPost,
Path: "",

4
core/controllers/router_v2_test.go
View File

@@ -2,7 +2,7 @@ package controllers_test
import (
"github.com/crawlab-team/crawlab/core/controllers"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
"testing"
@@ -28,7 +28,7 @@ func TestRouterGroups(t *testing.T) {
func TestRegisterController_Routes(t *testing.T) {
router := gin.Default()
groups := controllers.NewRouterGroups(router)
ctr := controllers.NewControllerV2[models.TestModel]()
ctr := controllers.NewControllerV2[models.TestModelV2]()
basePath := "/testmodels"
controllers.RegisterController(groups.AuthGroup, basePath, ctr)

2
core/controllers/schedule_v2.go
View File

@@ -2,7 +2,7 @@ package controllers
import (
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/schedule"
"github.com/gin-gonic/gin"

5
core/controllers/setting_v2.go
View File

@@ -2,6 +2,7 @@ package controllers
import (
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
@@ -12,7 +13,7 @@ func GetSetting(c *gin.Context) {
key := c.Param("id")
// setting
s, err := service.NewModelServiceV2[models.SettingV2]().GetOne(bson.M{"key": key}, nil)
s, err := service.NewModelServiceV2[models2.SettingV2]().GetOne(bson.M{"key": key}, nil)
if err != nil {
HandleErrorInternalServerError(c, err)
return
@@ -32,7 +33,7 @@ func PutSetting(c *gin.Context) {
return
}
modelSvc := service.NewModelServiceV2[models.SettingV2]()
modelSvc := service.NewModelServiceV2[models2.SettingV2]()
// setting
_s, err := modelSvc.GetOne(bson.M{"key": key}, nil)

102
core/controllers/spider_v2.go
View File

@@ -6,7 +6,7 @@ import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/fs"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/spider/admin"
"github.com/crawlab-team/crawlab/core/utils"
@@ -29,7 +29,7 @@ func GetSpiderById(c *gin.Context) {
HandleErrorBadRequest(c, err)
return
}
s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id)
s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id)
if errors.Is(err, mongo2.ErrNoDocuments) {
HandleErrorNotFound(c, err)
return
@@ -40,7 +40,7 @@ func GetSpiderById(c *gin.Context) {
}
// stat
s.Stat, err = service.NewModelServiceV2[models.SpiderStatV2]().GetById(s.Id)
s.Stat, err = service.NewModelServiceV2[models2.SpiderStatV2]().GetById(s.Id)
if err != nil {
if !errors.Is(err, mongo2.ErrNoDocuments) {
HandleErrorInternalServerError(c, err)
@@ -50,7 +50,7 @@ func GetSpiderById(c *gin.Context) {
// data collection
if !s.ColId.IsZero() {
col, err := service.NewModelServiceV2[models.DataCollectionV2]().GetById(s.ColId)
col, err := service.NewModelServiceV2[models2.DataCollectionV2]().GetById(s.ColId)
if err != nil {
if !errors.Is(err, mongo2.ErrNoDocuments) {
HandleErrorInternalServerError(c, err)
@@ -63,7 +63,7 @@ func GetSpiderById(c *gin.Context) {
// git
if utils.IsPro() && !s.GitId.IsZero() {
s.Git, err = service.NewModelServiceV2[models.GitV2]().GetById(s.GitId)
s.Git, err = service.NewModelServiceV2[models2.GitV2]().GetById(s.GitId)
if err != nil {
if !errors.Is(err, mongo2.ErrNoDocuments) {
HandleErrorInternalServerError(c, err)
@@ -79,14 +79,14 @@ func GetSpiderList(c *gin.Context) {
// get all list
all := MustGetFilterAll(c)
if all {
NewControllerV2[models.SpiderV2]().getAll(c)
NewControllerV2[models2.SpiderV2]().getAll(c)
return
}
// get list
withStats := c.Query("stats")
if withStats == "" {
NewControllerV2[models.SpiderV2]().GetList(c)
NewControllerV2[models2.SpiderV2]().GetList(c)
return
}
@@ -101,7 +101,7 @@ func getSpiderListWithStats(c *gin.Context) {
sort := MustGetSortOption(c)
// get list
spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(query, &mongo.FindOptions{
spiders, err := service.NewModelServiceV2[models2.SpiderV2]().GetMany(query, &mongo.FindOptions{
Sort: sort,
Skip: pagination.Size * (pagination.Page - 1),
Limit: pagination.Size,
@@ -113,7 +113,7 @@ func getSpiderListWithStats(c *gin.Context) {
return
}
if len(spiders) == 0 {
HandleSuccessWithListData(c, []models.SpiderV2{}, 0)
HandleSuccessWithListData(c, []models2.SpiderV2{}, 0)
return
}
@@ -128,21 +128,21 @@ func getSpiderListWithStats(c *gin.Context) {
}
// total count
total, err := service.NewModelServiceV2[models.SpiderV2]().Count(query)
total, err := service.NewModelServiceV2[models2.SpiderV2]().Count(query)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// stat list
spiderStats, err := service.NewModelServiceV2[models.SpiderStatV2]().GetMany(bson.M{"_id": bson.M{"$in": ids}}, nil)
spiderStats, err := service.NewModelServiceV2[models2.SpiderStatV2]().GetMany(bson.M{"_id": bson.M{"$in": ids}}, nil)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// cache stat list to dict
dict := map[primitive.ObjectID]models.SpiderStatV2{}
dict := map[primitive.ObjectID]models2.SpiderStatV2{}
var taskIds []primitive.ObjectID
for _, st := range spiderStats {
if st.Tasks > 0 {
@@ -159,9 +159,9 @@ func getSpiderListWithStats(c *gin.Context) {
}
// task list and stats
var tasks []models.TaskV2
dictTask := map[primitive.ObjectID]models.TaskV2{}
dictTaskStat := map[primitive.ObjectID]models.TaskStatV2{}
var tasks []models2.TaskV2
dictTask := map[primitive.ObjectID]models2.TaskV2{}
dictTaskStat := map[primitive.ObjectID]models2.TaskStatV2{}
if len(taskIds) > 0 {
// task list
queryTask := bson.M{
@@ -169,14 +169,14 @@ func getSpiderListWithStats(c *gin.Context) {
"$in": taskIds,
},
}
tasks, err = service.NewModelServiceV2[models.TaskV2]().GetMany(queryTask, nil)
tasks, err = service.NewModelServiceV2[models2.TaskV2]().GetMany(queryTask, nil)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// task stats list
taskStats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(queryTask, nil)
taskStats, err := service.NewModelServiceV2[models2.TaskStatV2]().GetMany(queryTask, nil)
if err != nil {
HandleErrorInternalServerError(c, err)
return
@@ -198,9 +198,9 @@ func getSpiderListWithStats(c *gin.Context) {
}
// git list
var gits []models.GitV2
var gits []models2.GitV2
if len(gitIds) > 0 && utils.IsPro() {
gits, err = service.NewModelServiceV2[models.GitV2]().GetMany(bson.M{"_id": bson.M{"$in": gitIds}}, nil)
gits, err = service.NewModelServiceV2[models2.GitV2]().GetMany(bson.M{"_id": bson.M{"$in": gitIds}}, nil)
if err != nil {
HandleErrorInternalServerError(c, err)
return
@@ -208,13 +208,13 @@ func getSpiderListWithStats(c *gin.Context) {
}
// cache git list to dict
dictGit := map[primitive.ObjectID]models.GitV2{}
dictGit := map[primitive.ObjectID]models2.GitV2{}
for _, g := range gits {
dictGit[g.Id] = g
}
// iterate list again
var data []models.SpiderV2
var data []models2.SpiderV2
for _, s := range spiders {
// spider stat
st, ok := dict[s.Id]
@@ -246,7 +246,7 @@ func getSpiderListWithStats(c *gin.Context) {
func PostSpider(c *gin.Context) {
// bind
var s models.SpiderV2
var s models2.SpiderV2
if err := c.ShouldBindJSON(&s); err != nil {
HandleErrorBadRequest(c, err)
return
@@ -264,7 +264,7 @@ func PostSpider(c *gin.Context) {
// add
s.SetCreated(u.Id)
s.SetUpdated(u.Id)
id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(s)
id, err := service.NewModelServiceV2[models2.SpiderV2]().InsertOne(s)
if err != nil {
HandleErrorInternalServerError(c, err)
return
@@ -272,11 +272,11 @@ func PostSpider(c *gin.Context) {
s.SetId(id)
// add stat
st := models.SpiderStatV2{}
st := models2.SpiderStatV2{}
st.SetId(id)
st.SetCreated(u.Id)
st.SetUpdated(u.Id)
_, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(st)
_, err = service.NewModelServiceV2[models2.SpiderStatV2]().InsertOne(st)
if err != nil {
HandleErrorInternalServerError(c, err)
return
@@ -305,7 +305,7 @@ func PutSpiderById(c *gin.Context) {
}
// bind
var s models.SpiderV2
var s models2.SpiderV2
if err := c.ShouldBindJSON(&s); err != nil {
HandleErrorBadRequest(c, err)
return
@@ -319,7 +319,7 @@ func PutSpiderById(c *gin.Context) {
u := GetUserFromContextV2(c)
modelSvc := service.NewModelServiceV2[models.SpiderV2]()
modelSvc := service.NewModelServiceV2[models2.SpiderV2]()
// save
s.SetUpdated(u.Id)
@@ -348,19 +348,19 @@ func DeleteSpiderById(c *gin.Context) {
if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) {
// delete spider
err = service.NewModelServiceV2[models.SpiderV2]().DeleteById(id)
err = service.NewModelServiceV2[models2.SpiderV2]().DeleteById(id)
if err != nil {
return err
}
// delete spider stat
err = service.NewModelServiceV2[models.SpiderStatV2]().DeleteById(id)
err = service.NewModelServiceV2[models2.SpiderStatV2]().DeleteById(id)
if err != nil {
return err
}
// related tasks
tasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{"spider_id": id}, nil)
tasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(bson.M{"spider_id": id}, nil)
if err != nil {
return err
}
@@ -376,13 +376,13 @@ func DeleteSpiderById(c *gin.Context) {
}
// delete related tasks
err = service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}})
err = service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}})
if err != nil {
return err
}
// delete related task stats
err = service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}})
err = service.NewModelServiceV2[models2.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}})
if err != nil {
return err
}
@@ -410,7 +410,7 @@ func DeleteSpiderById(c *gin.Context) {
go func() {
// spider
s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id)
s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id)
if err != nil {
log.Errorf("failed to get spider: %s", err.Error())
trace.PrintError(err)
@@ -451,7 +451,7 @@ func DeleteSpiderList(c *gin.Context) {
if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) {
// delete spiders
if err := service.NewModelServiceV2[models.SpiderV2]().DeleteMany(bson.M{
if err := service.NewModelServiceV2[models2.SpiderV2]().DeleteMany(bson.M{
"_id": bson.M{
"$in": payload.Ids,
},
@@ -460,7 +460,7 @@ func DeleteSpiderList(c *gin.Context) {
}
// delete spider stats
if err := service.NewModelServiceV2[models.SpiderStatV2]().DeleteMany(bson.M{
if err := service.NewModelServiceV2[models2.SpiderStatV2]().DeleteMany(bson.M{
"_id": bson.M{
"$in": payload.Ids,
},
@@ -469,7 +469,7 @@ func DeleteSpiderList(c *gin.Context) {
}
// related tasks
tasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{"spider_id": bson.M{"$in": payload.Ids}}, nil)
tasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(bson.M{"spider_id": bson.M{"$in": payload.Ids}}, nil)
if err != nil {
return err
}
@@ -485,12 +485,12 @@ func DeleteSpiderList(c *gin.Context) {
}
// delete related tasks
if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil {
if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil {
return err
}
// delete related task stats
if err := service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil {
if err := service.NewModelServiceV2[models2.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil {
return err
}
@@ -524,7 +524,7 @@ func DeleteSpiderList(c *gin.Context) {
defer wg.Done()
// spider
s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id)
s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id)
if err != nil {
log.Errorf("failed to get spider: %s", err.Error())
trace.PrintError(err)
@@ -692,14 +692,14 @@ func GetSpiderDataSource(c *gin.Context) {
}
// spider
s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id)
s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// data source
ds, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(s.DataSourceId)
ds, err := service.NewModelServiceV2[models2.DataSourceV2]().GetById(s.DataSourceId)
if err != nil {
if err.Error() == mongo2.ErrNoDocuments.Error() {
HandleSuccess(c)
@@ -728,7 +728,7 @@ func PostSpiderDataSource(c *gin.Context) {
}
// spider
s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id)
s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id)
if err != nil {
HandleErrorInternalServerError(c, err)
return
@@ -736,7 +736,7 @@ func PostSpiderDataSource(c *gin.Context) {
// data source
if !dsId.IsZero() {
_, err = service.NewModelServiceV2[models.DataSourceV2]().GetById(dsId)
_, err = service.NewModelServiceV2[models2.DataSourceV2]().GetById(dsId)
if err != nil {
HandleErrorInternalServerError(c, err)
return
@@ -747,7 +747,7 @@ func PostSpiderDataSource(c *gin.Context) {
u := GetUserFromContextV2(c)
s.DataSourceId = dsId
s.SetUpdatedBy(u.Id)
_, err = service.NewModelServiceV2[models.SpiderV2]().InsertOne(*s)
_, err = service.NewModelServiceV2[models2.SpiderV2]().InsertOne(*s)
if err != nil {
HandleErrorInternalServerError(c, err)
return
@@ -756,7 +756,7 @@ func PostSpiderDataSource(c *gin.Context) {
HandleSuccess(c)
}
func getSpiderFsSvc(s *models.SpiderV2) (svc interfaces.FsServiceV2, err error) {
func getSpiderFsSvc(s *models2.SpiderV2) (svc interfaces.FsServiceV2, err error) {
workspacePath := viper.GetString("workspace")
fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, s.Id.Hex()))
@@ -764,7 +764,7 @@ func getSpiderFsSvc(s *models.SpiderV2) (svc interfaces.FsServiceV2, err error)
}
func getSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsServiceV2, err error) {
s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id)
s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id)
if err != nil {
log.Errorf("failed to get spider: %s", err.Error())
trace.PrintError(err)
@@ -773,8 +773,8 @@ func getSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsServiceV2, err
return getSpiderFsSvc(s)
}
func upsertSpiderDataCollection(s *models.SpiderV2) (err error) {
modelSvc := service.NewModelServiceV2[models.DataCollectionV2]()
func upsertSpiderDataCollection(s *models2.SpiderV2) (err error) {
modelSvc := service.NewModelServiceV2[models2.DataCollectionV2]()
if s.ColId.IsZero() {
// validate
if s.ColName == "" {
@@ -785,7 +785,7 @@ func upsertSpiderDataCollection(s *models.SpiderV2) (err error) {
if err != nil {
if errors.Is(err, mongo2.ErrNoDocuments) {
// not exists, add new
dc = &models.DataCollectionV2{Name: s.ColName}
dc = &models2.DataCollectionV2{Name: s.ColName}
dcId, err := modelSvc.InsertOne(*dc)
if err != nil {
return err
@@ -812,7 +812,7 @@ func upsertSpiderDataCollection(s *models.SpiderV2) (err error) {
return nil
}
func UpsertSpiderDataCollection(s *models.SpiderV2) (err error) {
func UpsertSpiderDataCollection(s *models2.SpiderV2) (err error) {
return upsertSpiderDataCollection(s)
}
@@ -824,7 +824,7 @@ func getSpiderRootPath(c *gin.Context) (rootPath string, err error) {
}
// spider
s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id)
s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id)
if err != nil {
return "", err
}

44
core/controllers/task_v2.go
View File

@@ -5,7 +5,7 @@ import (
log2 "github.com/apex/log"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/result"
"github.com/crawlab-team/crawlab/core/spider/admin"
@@ -34,7 +34,7 @@ func GetTaskById(c *gin.Context) {
}
// task
t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id)
t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id)
if errors.Is(err, mongo2.ErrNoDocuments) {
HandleErrorNotFound(c, err)
return
@@ -45,7 +45,7 @@ func GetTaskById(c *gin.Context) {
}
// spider
t.Spider, _ = service.NewModelServiceV2[models.SpiderV2]().GetById(t.SpiderId)
t.Spider, _ = service.NewModelServiceV2[models2.SpiderV2]().GetById(t.SpiderId)
// skip if task status is pending
if t.Status == constants.TaskStatusPending {
@@ -54,7 +54,7 @@ func GetTaskById(c *gin.Context) {
}
// task stat
t.Stat, _ = service.NewModelServiceV2[models.TaskStatV2]().GetById(id)
t.Stat, _ = service.NewModelServiceV2[models2.TaskStatV2]().GetById(id)
HandleSuccessWithData(c, t)
}
@@ -62,7 +62,7 @@ func GetTaskById(c *gin.Context) {
func GetTaskList(c *gin.Context) {
withStats := c.Query("stats")
if withStats == "" {
NewControllerV2[models.TaskV2]().GetList(c)
NewControllerV2[models2.TaskV2]().GetList(c)
return
}
@@ -72,7 +72,7 @@ func GetTaskList(c *gin.Context) {
sort := MustGetSortOption(c)
// get tasks
tasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(query, &mongo.FindOptions{
tasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(query, &mongo.FindOptions{
Sort: sort,
Skip: pagination.Size * (pagination.Page - 1),
Limit: pagination.Size,
@@ -101,14 +101,14 @@ func GetTaskList(c *gin.Context) {
}
// total count
total, err := service.NewModelServiceV2[models.TaskV2]().Count(query)
total, err := service.NewModelServiceV2[models2.TaskV2]().Count(query)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}
// stat list
stats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(bson.M{
stats, err := service.NewModelServiceV2[models2.TaskStatV2]().GetMany(bson.M{
"_id": bson.M{
"$in": taskIds,
},
@@ -119,13 +119,13 @@ func GetTaskList(c *gin.Context) {
}
// cache stat list to dict
statsDict := map[primitive.ObjectID]models.TaskStatV2{}
statsDict := map[primitive.ObjectID]models2.TaskStatV2{}
for _, s := range stats {
statsDict[s.Id] = s
}
// spider list
spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(bson.M{
spiders, err := service.NewModelServiceV2[models2.SpiderV2]().GetMany(bson.M{
"_id": bson.M{
"$in": spiderIds,
},
@@ -136,7 +136,7 @@ func GetTaskList(c *gin.Context) {
}
// cache spider list to dict
spiderDict := map[primitive.ObjectID]models.SpiderV2{}
spiderDict := map[primitive.ObjectID]models2.SpiderV2{}
for _, s := range spiders {
spiderDict[s.Id] = s
}
@@ -170,22 +170,22 @@ func DeleteTaskById(c *gin.Context) {
// delete in db
if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) {
// delete task
_, err = service.NewModelServiceV2[models.TaskV2]().GetById(id)
_, err = service.NewModelServiceV2[models2.TaskV2]().GetById(id)
if err != nil {
return err
}
err = service.NewModelServiceV2[models.TaskV2]().DeleteById(id)
err = service.NewModelServiceV2[models2.TaskV2]().DeleteById(id)
if err != nil {
return err
}
// delete task stat
_, err = service.NewModelServiceV2[models.TaskStatV2]().GetById(id)
_, err = service.NewModelServiceV2[models2.TaskStatV2]().GetById(id)
if err != nil {
log2.Warnf("delete task stat error: %s", err.Error())
return nil
}
err = service.NewModelServiceV2[models.TaskStatV2]().DeleteById(id)
err = service.NewModelServiceV2[models2.TaskStatV2]().DeleteById(id)
if err != nil {
log2.Warnf("delete task stat error: %s", err.Error())
return nil
@@ -217,7 +217,7 @@ func DeleteList(c *gin.Context) {
if err := mongo.RunTransaction(func(context mongo2.SessionContext) error {
// delete tasks
if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{
if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{
"_id": bson.M{
"$in": payload.Ids,
},
@@ -226,7 +226,7 @@ func DeleteList(c *gin.Context) {
}
// delete task stats
if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{
if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{
"_id": bson.M{
"$in": payload.Ids,
},
@@ -261,7 +261,7 @@ func DeleteList(c *gin.Context) {
func PostTaskRun(c *gin.Context) {
// task
var t models.TaskV2
var t models2.TaskV2
if err := c.ShouldBindJSON(&t); err != nil {
HandleErrorBadRequest(c, err)
return
@@ -274,7 +274,7 @@ func PostTaskRun(c *gin.Context) {
}
// spider
s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(t.SpiderId)
s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(t.SpiderId)
if err != nil {
HandleErrorInternalServerError(c, err)
return
@@ -319,7 +319,7 @@ func PostTaskRestart(c *gin.Context) {
}
// task
t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id)
t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id)
if err != nil {
HandleErrorInternalServerError(c, err)
return
@@ -363,7 +363,7 @@ func PostTaskCancel(c *gin.Context) {
}
// task
t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id)
t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id)
if err != nil {
HandleErrorInternalServerError(c, err)
return
@@ -446,7 +446,7 @@ func GetTaskData(c *gin.Context) {
}
// task
t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id)
t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id)
if err != nil {
HandleErrorInternalServerError(c, err)
return

2
core/controllers/token_v2.go
View File

@@ -1,7 +1,7 @@
package controllers
import (
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/user"
"github.com/gin-gonic/gin"

2
core/controllers/user_v2.go
View File

@@ -1,7 +1,7 @@
package controllers
import (
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/gin-gonic/gin"

2
core/controllers/user_v2_test.go
View File

@@ -3,7 +3,7 @@ package controllers_test
import (
"github.com/crawlab-team/crawlab/core/controllers"
"github.com/crawlab-team/crawlab/core/middlewares"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"

2
core/controllers/utils_context.go
View File

@@ -3,7 +3,7 @@ package controllers
import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/gin-gonic/gin"
)

26
core/models/models/git_v2.go
View File

@@ -1,26 +0,0 @@
package models
import (
"github.com/crawlab-team/crawlab/vcs"
"time"
)
type GitV2 struct {
any `collection:"gits"`
BaseModelV2[GitV2] `bson:",inline"`
Url string `json:"url" bson:"url"`
Name string `json:"name" bson:"name"`
AuthType string `json:"auth_type" bson:"auth_type"`
Username string `json:"username" bson:"username"`
Password string `json:"password" bson:"password"`
CurrentBranch string `json:"current_branch" bson:"current_branch"`
Status string `json:"status" bson:"status"`
Error string `json:"error" bson:"error"`
Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"`
Refs []vcs.GitRef `json:"refs" bson:"refs"`
RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"`
CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"`
// settings
AutoPull bool `json:"auto_pull" bson:"auto_pull"`
}

23
core/models/models/metric_v2.go
View File

@@ -1,23 +0,0 @@
package models
import "go.mongodb.org/mongo-driver/bson/primitive"
type MetricV2 struct {
any `collection:"metrics"`
BaseModelV2[MetricV2] `bson:",inline"`
Type string `json:"type" bson:"type"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"`
TotalMemory uint64 `json:"total_memory" bson:"total_memory"`
AvailableMemory uint64 `json:"available_memory" bson:"available_memory"`
UsedMemory uint64 `json:"used_memory" bson:"used_memory"`
UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"`
TotalDisk uint64 `json:"total_disk" bson:"total_disk"`
AvailableDisk uint64 `json:"available_disk" bson:"available_disk"`
UsedDisk uint64 `json:"used_disk" bson:"used_disk"`
UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"`
DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"`
DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"`
NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"`
NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"`
}

24
core/models/models/node_v2.go
View File

@@ -1,24 +0,0 @@
package models
import (
"time"
)
type NodeV2 struct {
any `collection:"nodes"`
BaseModelV2[NodeV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Ip string `json:"ip" bson:"ip"`
Port string `json:"port" bson:"port"`
Mac string `json:"mac" bson:"mac"`
Hostname string `json:"hostname" bson:"hostname"`
Description string `json:"description" bson:"description"`
IsMaster bool `json:"is_master" bson:"is_master"`
Status string `json:"status" bson:"status"`
Enabled bool `json:"enabled" bson:"enabled"`
Active bool `json:"active" bson:"active"`
ActiveAt time.Time `json:"active_at" bson:"active_ts"`
AvailableRunners int `json:"available_runners" bson:"available_runners"`
MaxRunners int `json:"max_runners" bson:"max_runners"`
}

31
core/models/models/notification_setting_v2.go
View File

@@ -1,31 +0,0 @@
package models
type NotificationSettingV2 struct {
any `collection:"notification_settings"`
BaseModelV2[NotificationSettingV2] `bson:",inline"`
Type string `json:"type" bson:"type"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Enabled bool `json:"enabled" bson:"enabled"`
Global bool `json:"global" bson:"global"`
Title string `json:"title,omitempty" bson:"title,omitempty"`
Template string `json:"template,omitempty" bson:"template,omitempty"`
TaskTrigger string `json:"task_trigger" bson:"task_trigger"`
Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"`
Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"`
}
type NotificationSettingMail struct {
Server string `json:"server" bson:"server"`
Port string `json:"port,omitempty" bson:"port,omitempty"`
User string `json:"user,omitempty" bson:"user,omitempty"`
Password string `json:"password,omitempty" bson:"password,omitempty"`
SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"`
SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"`
To string `json:"to,omitempty" bson:"to,omitempty"`
Cc string `json:"cc,omitempty" bson:"cc,omitempty"`
}
type NotificationSettingMobile struct {
Webhook string `json:"webhook" bson:"webhook"`
}

13
core/models/models/permission_v2.go
View File

@@ -1,13 +0,0 @@
package models
type PermissionV2 struct {
any `collection:"permissions"`
BaseModelV2[PermissionV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Type string `json:"type" bson:"type"`
Target []string `json:"target" bson:"target"`
Allow []string `json:"allow" bson:"allow"`
Deny []string `json:"deny" bson:"deny"`
}

9
core/models/models/project_v2.go
View File

@@ -1,9 +0,0 @@
package models
type ProjectV2 struct {
any `collection:"projects"`
BaseModelV2[ProjectV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Spiders int `json:"spiders" bson:"-"`
}

12
core/models/models/role_permission_v2.go
View File

@@ -1,12 +0,0 @@
package models
import (
"go.mongodb.org/mongo-driver/bson/primitive"
)
type RolePermissionV2 struct {
any `collection:"role_permissions"`
BaseModelV2[RolePermissionV2] `bson:",inline"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"`
}

9
core/models/models/role_v2.go
View File

@@ -1,9 +0,0 @@
package models
type RoleV2 struct {
any `collection:"roles"`
BaseModelV2[RoleV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
}

23
core/models/models/schedule_v2.go
View File

@@ -1,23 +0,0 @@
package models
import (
"github.com/robfig/cron/v3"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type ScheduleV2 struct {
any `collection:"schedules"`
BaseModelV2[ScheduleV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Cron string `json:"cron" bson:"cron"`
EntryId cron.EntryID `json:"entry_id" bson:"entry_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
Priority int `json:"priority" bson:"priority"`
Enabled bool `json:"enabled" bson:"enabled"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
}

12
core/models/models/setting_v2.go
View File

@@ -1,12 +0,0 @@
package models
import (
"go.mongodb.org/mongo-driver/bson"
)
type SettingV2 struct {
any `collection:"settings"`
BaseModelV2[SettingV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Value bson.M `json:"value" bson:"value"`
}

20
core/models/models/spider_stat_v2.go
View File

@@ -1,20 +0,0 @@
package models
import (
"go.mongodb.org/mongo-driver/bson/primitive"
)
type SpiderStatV2 struct {
any `collection:"spider_stats"`
BaseModelV2[SpiderStatV2] `bson:",inline"`
LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"`
LastTask *TaskV2 `json:"last_task,omitempty" bson:"-"`
Tasks int `json:"tasks" bson:"tasks"`
Results int `json:"results" bson:"results"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second
AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second
AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second
AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second
}

32
core/models/models/spider_v2.go
View File

@@ -1,32 +0,0 @@
package models
import (
"go.mongodb.org/mongo-driver/bson/primitive"
)
type SpiderV2 struct {
any `collection:"spiders"`
BaseModelV2[SpiderV2] `bson:",inline"`
Name string `json:"name" bson:"name"` // spider name
Type string `json:"type" bson:"type"` // spider type
ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id
ColName string `json:"col_name,omitempty" bson:"-"` // data collection name
DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id
DataSource *DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source
Description string `json:"description" bson:"description"` // description
ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id
Mode string `json:"mode" bson:"mode"` // default Task.Mode
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds
GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id
GitRootPath string `json:"git_root_path" bson:"git_root_path"`
Git *GitV2 `json:"git,omitempty" bson:"-"`
// stats
Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"`
// execution
Cmd string `json:"cmd" bson:"cmd"` // execute command
Param string `json:"param" bson:"param"` // default task param
Priority int `json:"priority" bson:"priority"`
AutoInstall bool `json:"auto_install" bson:"auto_install"`
}

12
core/models/models/task_queue_item_v2.go
View File

@@ -1,12 +0,0 @@
package models
import (
"go.mongodb.org/mongo-driver/bson/primitive"
)
type TaskQueueItemV2 struct {
any `collection:"task_queue"`
BaseModelV2[TaskQueueItemV2] `bson:",inline"`
Priority int `json:"p" bson:"p"`
NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"`
}

18
core/models/models/task_stat_v2.go
View File

@@ -1,18 +0,0 @@
package models
import (
"time"
)
type TaskStatV2 struct {
any `collection:"task_stats"`
BaseModelV2[TaskStatV2] `bson:",inline"`
CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"`
StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"`
EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond
ResultCount int64 `json:"result_count" bson:"result_count"`
ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"`
}

30
core/models/models/task_v2.go
View File

@@ -1,30 +0,0 @@
package models
import (
"go.mongodb.org/mongo-driver/bson/primitive"
"time"
)
type TaskV2 struct {
any `collection:"tasks"`
BaseModelV2[TaskV2] `bson:",inline"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Status string `json:"status" bson:"status"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Error string `json:"error" bson:"error"`
Pid int `json:"pid" bson:"pid"`
ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"`
Type string `json:"type" bson:"type"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"`
Priority int `json:"priority" bson:"priority"`
Stat *TaskStatV2 `json:"stat,omitempty" bson:"-"`
HasSub bool `json:"has_sub" json:"has_sub"`
SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"`
Spider *SpiderV2 `json:"spider,omitempty" bson:"-"`
UserId primitive.ObjectID `json:"-" bson:"-"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
}

8
core/models/models/token_v2.go
View File

@@ -1,8 +0,0 @@
package models
type TokenV2 struct {
any `collection:"tokens"`
BaseModelV2[TokenV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Token string `json:"token" bson:"token"`
}

12
core/models/models/user_role_v2.go
View File

@@ -1,12 +0,0 @@
package models
import (
"go.mongodb.org/mongo-driver/bson/primitive"
)
type UserRoleV2 struct {
any `collection:"user_roles"`
BaseModelV2[UserRoleV2] `bson:",inline"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
}

10
core/models/models/user_v2.go
View File

@@ -1,10 +0,0 @@
package models
type UserV2 struct {
any `collection:"users"`
BaseModelV2[UserV2] `bson:",inline"`
Username string `json:"username" bson:"username"`
Password string `json:"-,omitempty" bson:"password"`
Role string `json:"role" bson:"role"`
Email string `json:"email" bson:"email"`
}

0
core/models/models/base_v2.go → core/models/models/v2/base_v2.go
View File

0
core/models/models/data_collection_v2.go → core/models/models/v2/data_collection_v2.go
View File

0
core/models/models/data_source_v2.go → core/models/models/v2/data_source_v2.go
View File

0
core/models/models/dependency_log_v2.go → core/models/models/v2/dependency_log_v2.go
View File

0
core/models/models/dependency_setting_v2.go → core/models/models/v2/dependency_setting_v2.go
View File

0
core/models/models/dependency_task_v2.go → core/models/models/v2/dependency_task_v2.go
View File

0
core/models/models/dependency_v2.go → core/models/models/v2/dependency_v2.go
View File

0
core/models/models/environment_v2.go → core/models/models/v2/environment_v2.go
View File

27
core/models/models/v2/git_v2.go Normal file
View File

@@ -0,0 +1,27 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/vcs"
"time"
)
type GitV2 struct {
any `collection:"gits"`
models.BaseModelV2[GitV2] `bson:",inline"`
Url string `json:"url" bson:"url"`
Name string `json:"name" bson:"name"`
AuthType string `json:"auth_type" bson:"auth_type"`
Username string `json:"username" bson:"username"`
Password string `json:"password" bson:"password"`
CurrentBranch string `json:"current_branch" bson:"current_branch"`
Status string `json:"status" bson:"status"`
Error string `json:"error" bson:"error"`
Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"`
Refs []vcs.GitRef `json:"refs" bson:"refs"`
RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"`
CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"`
// settings
AutoPull bool `json:"auto_pull" bson:"auto_pull"`
}

26
core/models/models/v2/metric_v2.go Normal file
View File

@@ -0,0 +1,26 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type MetricV2 struct {
any `collection:"metrics"`
models.BaseModelV2[MetricV2] `bson:",inline"`
Type string `json:"type" bson:"type"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"`
TotalMemory uint64 `json:"total_memory" bson:"total_memory"`
AvailableMemory uint64 `json:"available_memory" bson:"available_memory"`
UsedMemory uint64 `json:"used_memory" bson:"used_memory"`
UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"`
TotalDisk uint64 `json:"total_disk" bson:"total_disk"`
AvailableDisk uint64 `json:"available_disk" bson:"available_disk"`
UsedDisk uint64 `json:"used_disk" bson:"used_disk"`
UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"`
DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"`
DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"`
NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"`
NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"`
}

25
core/models/models/v2/node_v2.go Normal file
View File

@@ -0,0 +1,25 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"time"
)
type NodeV2 struct {
any `collection:"nodes"`
models.BaseModelV2[NodeV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Ip string `json:"ip" bson:"ip"`
Port string `json:"port" bson:"port"`
Mac string `json:"mac" bson:"mac"`
Hostname string `json:"hostname" bson:"hostname"`
Description string `json:"description" bson:"description"`
IsMaster bool `json:"is_master" bson:"is_master"`
Status string `json:"status" bson:"status"`
Enabled bool `json:"enabled" bson:"enabled"`
Active bool `json:"active" bson:"active"`
ActiveAt time.Time `json:"active_at" bson:"active_ts"`
AvailableRunners int `json:"available_runners" bson:"available_runners"`
MaxRunners int `json:"max_runners" bson:"max_runners"`
}

1
core/models/models/v2/notification_channel_v2.go Normal file
View File

@@ -0,0 +1 @@
package models

33
core/models/models/v2/notification_setting_v2.go Normal file
View File

@@ -0,0 +1,33 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type NotificationSettingV2 struct {
any `collection:"notification_settings"`
models.BaseModelV2[NotificationSettingV2] `bson:",inline"`
Type string `json:"type" bson:"type"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Enabled bool `json:"enabled" bson:"enabled"`
Global bool `json:"global" bson:"global"`
Title string `json:"title,omitempty" bson:"title,omitempty"`
Template string `json:"template,omitempty" bson:"template,omitempty"`
TaskTrigger string `json:"task_trigger" bson:"task_trigger"`
Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"`
Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"`
}
type NotificationSettingMail struct {
Server string `json:"server" bson:"server"`
Port string `json:"port,omitempty" bson:"port,omitempty"`
User string `json:"user,omitempty" bson:"user,omitempty"`
Password string `json:"password,omitempty" bson:"password,omitempty"`
SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"`
SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"`
To string `json:"to,omitempty" bson:"to,omitempty"`
Cc string `json:"cc,omitempty" bson:"cc,omitempty"`
}
type NotificationSettingMobile struct {
Webhook string `json:"webhook" bson:"webhook"`
}

15
core/models/models/v2/permission_v2.go Normal file
View File

@@ -0,0 +1,15 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type PermissionV2 struct {
any `collection:"permissions"`
models.BaseModelV2[PermissionV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Type string `json:"type" bson:"type"`
Target []string `json:"target" bson:"target"`
Allow []string `json:"allow" bson:"allow"`
Deny []string `json:"deny" bson:"deny"`
}

11
core/models/models/v2/project_v2.go Normal file
View File

@@ -0,0 +1,11 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type ProjectV2 struct {
any `collection:"projects"`
models.BaseModelV2[ProjectV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Spiders int `json:"spiders" bson:"-"`
}

13
core/models/models/v2/role_permission_v2.go Normal file
View File

@@ -0,0 +1,13 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type RolePermissionV2 struct {
any `collection:"role_permissions"`
models.BaseModelV2[RolePermissionV2] `bson:",inline"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"`
}

11
core/models/models/v2/role_v2.go Normal file
View File

@@ -0,0 +1,11 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type RoleV2 struct {
any `collection:"roles"`
models.BaseModelV2[RoleV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
}

24
core/models/models/v2/schedule_v2.go Normal file
View File

@@ -0,0 +1,24 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/robfig/cron/v3"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type ScheduleV2 struct {
any `collection:"schedules"`
models.BaseModelV2[ScheduleV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Cron string `json:"cron" bson:"cron"`
EntryId cron.EntryID `json:"entry_id" bson:"entry_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
Priority int `json:"priority" bson:"priority"`
Enabled bool `json:"enabled" bson:"enabled"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
}

13
core/models/models/v2/setting_v2.go Normal file
View File

@@ -0,0 +1,13 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson"
)
type SettingV2 struct {
any `collection:"settings"`
models.BaseModelV2[SettingV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Value bson.M `json:"value" bson:"value"`
}

21
core/models/models/v2/spider_stat_v2.go Normal file
View File

@@ -0,0 +1,21 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type SpiderStatV2 struct {
any `collection:"spider_stats"`
models.BaseModelV2[SpiderStatV2] `bson:",inline"`
LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"`
LastTask *TaskV2 `json:"last_task,omitempty" bson:"-"`
Tasks int `json:"tasks" bson:"tasks"`
Results int `json:"results" bson:"results"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second
AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second
AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second
AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second
}

33
core/models/models/v2/spider_v2.go Normal file
View File

@@ -0,0 +1,33 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type SpiderV2 struct {
any `collection:"spiders"`
models.BaseModelV2[SpiderV2] `bson:",inline"`
Name string `json:"name" bson:"name"` // spider name
Type string `json:"type" bson:"type"` // spider type
ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id
ColName string `json:"col_name,omitempty" bson:"-"` // data collection name
DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id
DataSource *models.DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source
Description string `json:"description" bson:"description"` // description
ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id
Mode string `json:"mode" bson:"mode"` // default Task.Mode
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds
GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id
GitRootPath string `json:"git_root_path" bson:"git_root_path"`
Git *GitV2 `json:"git,omitempty" bson:"-"`
// stats
Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"`
// execution
Cmd string `json:"cmd" bson:"cmd"` // execute command
Param string `json:"param" bson:"param"` // default task param
Priority int `json:"priority" bson:"priority"`
AutoInstall bool `json:"auto_install" bson:"auto_install"`
}

13
core/models/models/v2/task_queue_item_v2.go Normal file
View File

@@ -0,0 +1,13 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type TaskQueueItemV2 struct {
any `collection:"task_queue"`
models.BaseModelV2[TaskQueueItemV2] `bson:",inline"`
Priority int `json:"p" bson:"p"`
NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"`
}

19
core/models/models/v2/task_stat_v2.go Normal file
View File

@@ -0,0 +1,19 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"time"
)
type TaskStatV2 struct {
any `collection:"task_stats"`
models.BaseModelV2[TaskStatV2] `bson:",inline"`
CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"`
StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"`
EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"`
WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond
RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond
TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond
ResultCount int64 `json:"result_count" bson:"result_count"`
ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"`
}

31
core/models/models/v2/task_v2.go Normal file
View File

@@ -0,0 +1,31 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
"time"
)
type TaskV2 struct {
any `collection:"tasks"`
models.BaseModelV2[TaskV2] `bson:",inline"`
SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"`
Status string `json:"status" bson:"status"`
NodeId primitive.ObjectID `json:"node_id" bson:"node_id"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Error string `json:"error" bson:"error"`
Pid int `json:"pid" bson:"pid"`
ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"`
Type string `json:"type" bson:"type"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"`
Priority int `json:"priority" bson:"priority"`
Stat *TaskStatV2 `json:"stat,omitempty" bson:"-"`
HasSub bool `json:"has_sub" json:"has_sub"`
SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"`
Spider *SpiderV2 `json:"spider,omitempty" bson:"-"`
UserId primitive.ObjectID `json:"-" bson:"-"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
}

0
core/models/models/test.go → core/models/models/v2/test_v2.go
View File

10
core/models/models/v2/token_v2.go Normal file
View File

@@ -0,0 +1,10 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type TokenV2 struct {
any `collection:"tokens"`
models.BaseModelV2[TokenV2] `bson:",inline"`
Name string `json:"name" bson:"name"`
Token string `json:"token" bson:"token"`
}

13
core/models/models/v2/user_role_v2.go Normal file
View File

@@ -0,0 +1,13 @@
package models
import (
"github.com/crawlab-team/crawlab/core/models/models/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type UserRoleV2 struct {
any `collection:"user_roles"`
models.BaseModelV2[UserRoleV2] `bson:",inline"`
RoleId primitive.ObjectID `json:"role_id" bson:"role_id"`
UserId primitive.ObjectID `json:"user_id" bson:"user_id"`
}

12
core/models/models/v2/user_v2.go Normal file
View File

@@ -0,0 +1,12 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type UserV2 struct {
any `collection:"users"`
models.BaseModelV2[UserV2] `bson:",inline"`
Username string `json:"username" bson:"username"`
Password string `json:"-,omitempty" bson:"password"`
Role string `json:"role" bson:"role"`
Email string `json:"email" bson:"email"`
}

11
core/models/models/v2/variable_v2.go Normal file
View File

@@ -0,0 +1,11 @@
package models
import "github.com/crawlab-team/crawlab/core/models/models/v2"
type VariableV2 struct {
any `collection:"variables"`
models.BaseModelV2[VariableV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Value string `json:"value" bson:"value"`
Remark string `json:"remark" bson:"remark"`
}

9
core/models/models/variable_v2.go
View File

@@ -1,9 +0,0 @@
package models
type VariableV2 struct {
any `collection:"variables"`
BaseModelV2[VariableV2] `bson:",inline"`
Key string `json:"key" bson:"key"`
Value string `json:"value" bson:"value"`
Remark string `json:"remark" bson:"remark"`
}