feat: added modules

core/controllers/base.go

@@ -0,0 +1,69 @@
+package controllers
+
+import "github.com/gin-gonic/gin"
+
+const (
+	ControllerIdNode = iota << 1
+	ControllerIdProject
+	ControllerIdSpider
+	ControllerIdTask
+	ControllerIdJob
+	ControllerIdSchedule
+	ControllerIdUser
+	ControllerIdSetting
+	ControllerIdToken
+	ControllerIdVariable
+	ControllerIdTag
+	ControllerIdLogin
+	ControllerIdColor
+	ControllerIdDataSource
+	ControllerIdDataCollection
+	ControllerIdResult
+	ControllerIdStats
+	ControllerIdFiler
+	ControllerIdGit
+	ControllerIdRole
+	ControllerIdPermission
+	ControllerIdExport
+	ControllerIdNotification
+	ControllerIdFilter
+	ControllerIdEnvironment
+	ControllerIdSync
+
+	ControllerIdVersion
+	ControllerIdI18n
+	ControllerIdSystemInfo
+	ControllerIdDemo
+)
+
+type ControllerId int
+
+type BasicController interface {
+	Get(c *gin.Context)
+	Post(c *gin.Context)
+	Put(c *gin.Context)
+	Delete(c *gin.Context)
+}
+
+type ListController interface {
+	BasicController
+	GetList(c *gin.Context)
+	PutList(c *gin.Context)
+	PostList(c *gin.Context)
+	DeleteList(c *gin.Context)
+}
+
+type Action struct {
+	Method      string
+	Path        string
+	HandlerFunc gin.HandlerFunc
+}
+
+type ActionController interface {
+	Actions() (actions []Action)
+}
+
+type ListActionController interface {
+	ListController
+	ActionController
+}

core/controllers/base_v2.go

@@ -0,0 +1,226 @@
+package controllers
+
+import (
+	"errors"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+)
+
+type BaseControllerV2[T any] struct {
+	modelSvc *service.ModelServiceV2[T]
+	actions  []Action
+}
+
+func (ctr *BaseControllerV2[T]) GetById(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	model, err := ctr.modelSvc.GetById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, model)
+}
+
+func (ctr *BaseControllerV2[T]) GetList(c *gin.Context) {
+	// get all if query field "all" is set true
+	all := MustGetFilterAll(c)
+	if all {
+		ctr.getAll(c)
+		return
+	}
+
+	// get list
+	ctr.getList(c)
+}
+
+func (ctr *BaseControllerV2[T]) Post(c *gin.Context) {
+	var model T
+	if err := c.ShouldBindJSON(&model); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	u := GetUserFromContextV2(c)
+	m := any(&model).(interfaces.ModelV2)
+	m.SetId(primitive.NewObjectID())
+	m.SetCreated(u.Id)
+	m.SetUpdated(u.Id)
+	col := ctr.modelSvc.GetCol()
+	res, err := col.GetCollection().InsertOne(col.GetContext(), m)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	result, err := ctr.modelSvc.GetById(res.InsertedID.(primitive.ObjectID))
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, result)
+}
+
+func (ctr *BaseControllerV2[T]) PutById(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	var model T
+	if err := c.ShouldBindJSON(&model); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	u := GetUserFromContextV2(c)
+	m := any(&model).(interfaces.ModelV2)
+	m.SetId(primitive.NewObjectID())
+	m.SetUpdated(u.Id)
+
+	if err := ctr.modelSvc.ReplaceById(id, model); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	result, err := ctr.modelSvc.GetById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, result)
+}
+
+func (ctr *BaseControllerV2[T]) PatchList(c *gin.Context) {
+	type Payload struct {
+		Ids    []primitive.ObjectID `json:"ids"`
+		Update bson.M               `json:"update"`
+	}
+
+	var payload Payload
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// query
+	query := bson.M{
+		"_id": bson.M{
+			"$in": payload.Ids,
+		},
+	}
+
+	// update
+	if err := ctr.modelSvc.UpdateMany(query, bson.M{"$set": payload.Update}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}
+
+func (ctr *BaseControllerV2[T]) DeleteById(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	if err := ctr.modelSvc.DeleteById(id); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}
+
+func (ctr *BaseControllerV2[T]) DeleteList(c *gin.Context) {
+	type Payload struct {
+		Ids []primitive.ObjectID `json:"ids"`
+	}
+
+	var payload Payload
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	if err := ctr.modelSvc.DeleteMany(bson.M{
+		"_id": bson.M{
+			"$in": payload.Ids,
+		},
+	}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}
+
+func (ctr *BaseControllerV2[T]) getAll(c *gin.Context) {
+	models, err := ctr.modelSvc.GetMany(nil, &mongo.FindOptions{
+		Sort: bson.D{{"_id", -1}},
+	})
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	total, err := ctr.modelSvc.Count(nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithListData(c, models, total)
+}
+
+func (ctr *BaseControllerV2[T]) getList(c *gin.Context) {
+	// params
+	pagination := MustGetPagination(c)
+	query := MustGetFilterQuery(c)
+	sort := MustGetSortOption(c)
+
+	// get list
+	models, err := ctr.modelSvc.GetMany(query, &mongo.FindOptions{
+		Sort:  sort,
+		Skip:  pagination.Size * (pagination.Page - 1),
+		Limit: pagination.Size,
+	})
+	if err != nil {
+		if errors.Is(err, mongo2.ErrNoDocuments) {
+			HandleSuccessWithListData(c, nil, 0)
+		} else {
+			HandleErrorInternalServerError(c, err)
+		}
+		return
+	}
+
+	// total count
+	total, err := ctr.modelSvc.Count(query)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// response
+	HandleSuccessWithListData(c, models, total)
+}
+
+func NewControllerV2[T any](actions ...Action) *BaseControllerV2[T] {
+	ctr := &BaseControllerV2[T]{
+		modelSvc: service.NewModelServiceV2[T](),
+		actions:  actions,
+	}
+	return ctr
+}

core/controllers/base_v2_test.go

@@ -0,0 +1,165 @@
+package controllers_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/middlewares"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/user"
+	"github.com/spf13/viper"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/assert"
+)
+
+func init() {
+
+}
+
+// TestModel is a simple struct to be used as a model in tests
+type TestModel models.TestModel
+
+var TestToken string
+
+// SetupTestDB sets up the test database
+func SetupTestDB() {
+	viper.Set("mongo.db", "testdb")
+	modelSvc := service.NewModelServiceV2[models.UserV2]()
+	u := models.UserV2{
+		Username: "admin",
+	}
+	id, err := modelSvc.InsertOne(u)
+	if err != nil {
+		panic(err)
+	}
+	u.SetId(id)
+
+	userSvc, err := user.GetUserServiceV2()
+	if err != nil {
+		panic(err)
+	}
+	token, err := userSvc.MakeToken(&u)
+	if err != nil {
+		panic(err)
+	}
+	TestToken = token
+}
+
+// SetupRouter sets up the gin router for testing
+func SetupRouter() *gin.Engine {
+	router := gin.Default()
+	return router
+}
+
+// CleanupTestDB cleans up the test database
+func CleanupTestDB() {
+	mongo.GetMongoDb("testdb").Drop(context.Background())
+}
+
+func TestBaseControllerV2_GetById(t *testing.T) {
+	SetupTestDB()
+	defer CleanupTestDB()
+
+	// Insert a test document
+	id, err := service.NewModelServiceV2[TestModel]().InsertOne(TestModel{Name: "test"})
+	assert.NoError(t, err)
+
+	// Initialize the controller
+	ctr := controllers.NewControllerV2[TestModel]()
+
+	// Set up the router
+	router := SetupRouter()
+	router.Use(middlewares.AuthorizationMiddlewareV2())
+	router.GET("/testmodels/:id", ctr.GetById)
+
+	// Create a test request
+	req, _ := http.NewRequest("GET", "/testmodels/"+id.Hex(), nil)
+	req.Header.Set("Authorization", TestToken)
+	w := httptest.NewRecorder()
+
+	// Serve the request
+	router.ServeHTTP(w, req)
+
+	// Check the response
+	assert.Equal(t, http.StatusOK, w.Code)
+
+	var response controllers.Response[TestModel]
+	err = json.Unmarshal(w.Body.Bytes(), &response)
+	assert.NoError(t, err)
+	assert.Equal(t, "test", response.Data.Name)
+}
+
+func TestBaseControllerV2_Post(t *testing.T) {
+	SetupTestDB()
+	defer CleanupTestDB()
+
+	// Initialize the controller
+	ctr := controllers.NewControllerV2[TestModel]()
+
+	// Set up the router
+	router := SetupRouter()
+	router.Use(middlewares.AuthorizationMiddlewareV2())
+	router.POST("/testmodels", ctr.Post)
+
+	// Create a test request
+	testModel := TestModel{Name: "test"}
+	jsonValue, _ := json.Marshal(testModel)
+	req, _ := http.NewRequest("POST", "/testmodels", bytes.NewBuffer(jsonValue))
+	req.Header.Set("Authorization", TestToken)
+	w := httptest.NewRecorder()
+
+	// Serve the request
+	router.ServeHTTP(w, req)
+
+	// Check the response
+	assert.Equal(t, http.StatusOK, w.Code)
+
+	var response controllers.Response[TestModel]
+	err := json.Unmarshal(w.Body.Bytes(), &response)
+	assert.NoError(t, err)
+	assert.Equal(t, "test", response.Data.Name)
+
+	// Check if the document was inserted into the database
+	result, err := service.NewModelServiceV2[TestModel]().GetById(response.Data.Id)
+	assert.NoError(t, err)
+	assert.Equal(t, "test", result.Name)
+}
+
+func TestBaseControllerV2_DeleteById(t *testing.T) {
+	SetupTestDB()
+	defer CleanupTestDB()
+
+	// Insert a test document
+	id, err := service.NewModelServiceV2[TestModel]().InsertOne(TestModel{Name: "test"})
+	assert.NoError(t, err)
+
+	// Initialize the controller
+	ctr := controllers.NewControllerV2[TestModel]()
+
+	// Set up the router
+	router := SetupRouter()
+	router.Use(middlewares.AuthorizationMiddlewareV2())
+	router.DELETE("/testmodels/:id", ctr.DeleteById)
+
+	// Create a test request
+	req, _ := http.NewRequest("DELETE", "/testmodels/"+id.Hex(), nil)
+	req.Header.Set("Authorization", TestToken)
+	w := httptest.NewRecorder()
+
+	// Serve the request
+	router.ServeHTTP(w, req)
+
+	// Check the response
+	assert.Equal(t, http.StatusOK, w.Code)
+
+	// Check if the document was deleted from the database
+	_, err = service.NewModelServiceV2[TestModel]().GetById(id)
+	assert.Error(t, err)
+}

core/controllers/binder.go

@@ -0,0 +1,14 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/gin-gonic/gin"
+)
+
+type BinderInterface interface {
+	Bind(c *gin.Context) (res interfaces.Model, err error)
+	BindList(c *gin.Context) (res []interfaces.Model, err error)
+	BindBatchRequestPayload(c *gin.Context) (payload entity.BatchRequestPayload, err error)
+	BindBatchRequestPayloadWithStringData(c *gin.Context) (payload entity.BatchRequestPayloadWithStringData, res interfaces.Model, err error)
+}

core/controllers/binder_json.go

@@ -0,0 +1,208 @@
+package controllers
+
+import (
+	"encoding/json"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/gin-gonic/gin"
+)
+
+func NewJsonBinder(id ControllerId) (b *JsonBinder) {
+	return &JsonBinder{
+		id: id,
+	}
+}
+
+type JsonBinder struct {
+	id ControllerId
+}
+
+func (b *JsonBinder) Bind(c *gin.Context) (res interfaces.Model, err error) {
+	// declare
+	m := models.NewModelMap()
+
+	switch b.id {
+	case ControllerIdNode:
+		err = c.ShouldBindJSON(&m.Node)
+		return &m.Node, err
+	case ControllerIdProject:
+		err = c.ShouldBindJSON(&m.Project)
+		return &m.Project, err
+	case ControllerIdSpider:
+		err = c.ShouldBindJSON(&m.Spider)
+		return &m.Spider, err
+	case ControllerIdTask:
+		err = c.ShouldBindJSON(&m.Task)
+		return &m.Task, err
+	case ControllerIdJob:
+		err = c.ShouldBindJSON(&m.Job)
+		return &m.Job, err
+	case ControllerIdSchedule:
+		err = c.ShouldBindJSON(&m.Schedule)
+		return &m.Schedule, err
+	case ControllerIdUser:
+		err = c.ShouldBindJSON(&m.User)
+		return &m.User, nil
+	case ControllerIdSetting:
+		err = c.ShouldBindJSON(&m.Setting)
+		return &m.Setting, nil
+	case ControllerIdToken:
+		err = c.ShouldBindJSON(&m.Token)
+		return &m.Token, nil
+	case ControllerIdVariable:
+		err = c.ShouldBindJSON(&m.Variable)
+		return &m.Variable, nil
+	case ControllerIdTag:
+		err = c.ShouldBindJSON(&m.Tag)
+		return &m.Tag, nil
+	case ControllerIdDataSource:
+		err = c.ShouldBindJSON(&m.DataSource)
+		return &m.DataSource, nil
+	case ControllerIdDataCollection:
+		err = c.ShouldBindJSON(&m.DataCollection)
+		return &m.DataCollection, nil
+	case ControllerIdGit:
+		err = c.ShouldBindJSON(&m.Git)
+		return &m.Git, nil
+	case ControllerIdRole:
+		err = c.ShouldBindJSON(&m.Role)
+		return &m.Role, nil
+	case ControllerIdPermission:
+		err = c.ShouldBindJSON(&m.Permission)
+		return &m.Permission, nil
+	case ControllerIdEnvironment:
+		err = c.ShouldBindJSON(&m.Environment)
+		return &m.Environment, nil
+	default:
+		return nil, errors.ErrorControllerInvalidControllerId
+	}
+}
+
+func (b *JsonBinder) BindList(c *gin.Context) (res interface{}, err error) {
+	// declare
+	m := models.NewModelListMap()
+
+	// bind
+	switch b.id {
+	case ControllerIdNode:
+		err = c.ShouldBindJSON(&m.Nodes)
+		return m.Nodes, err
+	case ControllerIdProject:
+		err = c.ShouldBindJSON(&m.Projects)
+		return m.Projects, err
+	case ControllerIdSpider:
+		err = c.ShouldBindJSON(&m.Spiders)
+		return m.Spiders, err
+	case ControllerIdTask:
+		err = c.ShouldBindJSON(&m.Tasks)
+		return m.Tasks, err
+	case ControllerIdJob:
+		err = c.ShouldBindJSON(&m.Jobs)
+		return m.Jobs, err
+	case ControllerIdSchedule:
+		err = c.ShouldBindJSON(&m.Schedules)
+		return m.Schedules, err
+	case ControllerIdUser:
+		err = c.ShouldBindJSON(&m.Users)
+		return m.Users, nil
+	case ControllerIdSetting:
+		err = c.ShouldBindJSON(&m.Settings)
+		return m.Settings, nil
+	case ControllerIdToken:
+		err = c.ShouldBindJSON(&m.Tokens)
+		return m.Tokens, nil
+	case ControllerIdVariable:
+		err = c.ShouldBindJSON(&m.Variables)
+		return m.Variables, nil
+	case ControllerIdTag:
+		err = c.ShouldBindJSON(&m.Tags)
+		return m.Tags, nil
+	case ControllerIdDataSource:
+		err = c.ShouldBindJSON(&m.DataSources)
+		return m.DataSources, nil
+	case ControllerIdDataCollection:
+		err = c.ShouldBindJSON(&m.DataCollections)
+		return m.DataCollections, nil
+	case ControllerIdGit:
+		err = c.ShouldBindJSON(&m.Gits)
+		return m.Gits, nil
+	case ControllerIdRole:
+		err = c.ShouldBindJSON(&m.Roles)
+		return m.Roles, nil
+	case ControllerIdEnvironment:
+		err = c.ShouldBindJSON(&m.Environments)
+		return m.Environments, nil
+	default:
+		return nil, errors.ErrorControllerInvalidControllerId
+	}
+}
+
+func (b *JsonBinder) BindBatchRequestPayload(c *gin.Context) (payload entity.BatchRequestPayload, err error) {
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		return payload, err
+	}
+	return payload, nil
+}
+
+func (b *JsonBinder) BindBatchRequestPayloadWithStringData(c *gin.Context) (payload entity.BatchRequestPayloadWithStringData, res interfaces.Model, err error) {
+	// declare
+	m := models.NewModelMap()
+
+	// bind
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		return payload, nil, err
+	}
+
+	// validate
+	if len(payload.Ids) == 0 ||
+		len(payload.Fields) == 0 {
+		return payload, nil, errors.ErrorControllerRequestPayloadInvalid
+	}
+
+	// unmarshall
+	switch b.id {
+	case ControllerIdNode:
+		err = json.Unmarshal([]byte(payload.Data), &m.Node)
+		return payload, &m.Node, err
+	case ControllerIdProject:
+		err = json.Unmarshal([]byte(payload.Data), &m.Project)
+		return payload, &m.Project, err
+	case ControllerIdSpider:
+		err = json.Unmarshal([]byte(payload.Data), &m.Spider)
+		return payload, &m.Spider, err
+	case ControllerIdTask:
+		err = json.Unmarshal([]byte(payload.Data), &m.Task)
+		return payload, &m.Task, err
+	case ControllerIdJob:
+		err = json.Unmarshal([]byte(payload.Data), &m.Job)
+		return payload, &m.Job, err
+	case ControllerIdSchedule:
+		err = json.Unmarshal([]byte(payload.Data), &m.Schedule)
+		return payload, &m.Schedule, err
+	case ControllerIdUser:
+		err = json.Unmarshal([]byte(payload.Data), &m.User)
+		return payload, &m.User, err
+	case ControllerIdSetting:
+		err = json.Unmarshal([]byte(payload.Data), &m.Setting)
+		return payload, &m.Setting, err
+	case ControllerIdToken:
+		err = json.Unmarshal([]byte(payload.Data), &m.Token)
+		return payload, &m.Token, err
+	case ControllerIdVariable:
+		err = json.Unmarshal([]byte(payload.Data), &m.Variable)
+		return payload, &m.Variable, err
+	case ControllerIdDataSource:
+		err = json.Unmarshal([]byte(payload.Data), &m.DataSource)
+		return payload, &m.DataSource, err
+	case ControllerIdDataCollection:
+		err = json.Unmarshal([]byte(payload.Data), &m.DataCollection)
+		return payload, &m.DataCollection, err
+	case ControllerIdEnvironment:
+		err = json.Unmarshal([]byte(payload.Data), &m.Environment)
+		return payload, &m.Environment, err
+	default:
+		return payload, nil, errors.ErrorControllerInvalidControllerId
+	}
+}

core/controllers/data_collection.go

@@ -0,0 +1,103 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"net/http"
+)
+
+var DataCollectionController *dataCollectionController
+
+func getDataCollectionActions() []Action {
+	ctx := newDataCollectionContext()
+	return []Action{
+		{
+			Method:      http.MethodPost,
+			Path:        "/:id/indexes",
+			HandlerFunc: ctx.postIndexes,
+		},
+	}
+}
+
+type dataCollectionController struct {
+	ListActionControllerDelegate
+	d   ListActionControllerDelegate
+	ctx *dataCollectionContext
+}
+
+type dataCollectionContext struct {
+	modelSvc  service.ModelService
+	resultSvc interfaces.ResultService
+}
+
+func (ctx *dataCollectionContext) postIndexes(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	dc, err := ctx.modelSvc.GetDataCollectionById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	for _, f := range dc.Fields {
+		if err := mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{
+			Keys: f.Key,
+		}); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+
+	HandleSuccess(c)
+}
+
+var _dataCollectionCtx *dataCollectionContext
+
+func newDataCollectionContext() *dataCollectionContext {
+	if _dataCollectionCtx != nil {
+		return _dataCollectionCtx
+	}
+
+	// context
+	ctx := &dataCollectionContext{}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(
+		modelSvc service.ModelService,
+	) {
+		ctx.modelSvc = modelSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	_dataCollectionCtx = ctx
+
+	return ctx
+}
+
+func newDataCollectionController() *dataCollectionController {
+	actions := getDataCollectionActions()
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions)
+	d := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions)
+	ctx := newDataCollectionContext()
+
+	return &dataCollectionController{
+		ListActionControllerDelegate: *ctr,
+		d:                            *d,
+		ctx:                          ctx,
+	}
+}

core/controllers/data_source.go

@@ -0,0 +1,148 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/ds"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	interfaces2 "github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"net/http"
+)
+
+var DataSourceController *dataSourceController
+
+func getDataSourceActions() []Action {
+	ctx := newDataSourceContext()
+	return []Action{
+		{
+			Path:        "/:id/change-password",
+			Method:      http.MethodPost,
+			HandlerFunc: ctx.changePassword,
+		},
+	}
+}
+
+type dataSourceController struct {
+	ListActionControllerDelegate
+	d   ListActionControllerDelegate
+	ctx *dataSourceContext
+}
+
+func (ctr *dataSourceController) Post(c *gin.Context) {
+	// data source
+	var _ds models.DataSource
+	if err := c.ShouldBindJSON(&_ds); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// add data source to db
+	if err := mongo.RunTransaction(func(ctx mongo2.SessionContext) error {
+		if err := delegate.NewModelDelegate(&_ds).Add(); err != nil {
+			return trace.TraceError(err)
+		}
+		pwd, err := utils.EncryptAES(_ds.Password)
+		if err != nil {
+			return trace.TraceError(err)
+		}
+		p := models.Password{Id: _ds.Id, Password: pwd}
+		if err := delegate.NewModelDelegate(&p).Add(); err != nil {
+			return trace.TraceError(err)
+		}
+		return nil
+	}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// check data source status
+	go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }()
+
+	HandleSuccess(c)
+}
+
+func (ctr *dataSourceController) Put(c *gin.Context) {
+	// data source
+	var _ds models.DataSource
+	if err := c.ShouldBindJSON(&_ds); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	if err := delegate.NewModelDelegate(&_ds).Save(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// check data source status
+	go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }()
+}
+
+type dataSourceContext struct {
+	dsSvc interfaces.DataSourceService
+}
+
+var _dataSourceCtx *dataSourceContext
+
+func newDataSourceContext() *dataSourceContext {
+	if _dataSourceCtx != nil {
+		return _dataSourceCtx
+	}
+	dsSvc, err := ds.GetDataSourceService()
+	if err != nil {
+		panic(err)
+	}
+	_dataSourceCtx = &dataSourceContext{
+		dsSvc: dsSvc,
+	}
+	return _dataSourceCtx
+}
+
+func (ctx *dataSourceContext) changePassword(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	var payload map[string]string
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	password, ok := payload["password"]
+	if !ok {
+		HandleErrorBadRequest(c, errors.ErrorDataSourceMissingRequiredFields)
+		return
+	}
+	if err := ctx.dsSvc.ChangePassword(id, password); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func newDataSourceController() *dataSourceController {
+	actions := getDataSourceActions()
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions)
+	d := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions)
+	ctx := newDataSourceContext()
+
+	return &dataSourceController{
+		ListActionControllerDelegate: *ctr,
+		d:                            *d,
+		ctx:                          ctx,
+	}
+}

core/controllers/delegate_action.go

@@ -0,0 +1,17 @@
+package controllers
+
+func NewActionControllerDelegate(id ControllerId, actions []Action) (d *ActionControllerDelegate) {
+	return &ActionControllerDelegate{
+		id:      id,
+		actions: actions,
+	}
+}
+
+type ActionControllerDelegate struct {
+	id      ControllerId
+	actions []Action
+}
+
+func (ctr *ActionControllerDelegate) Actions() (actions []Action) {
+	return ctr.actions
+}

core/controllers/delegate_basic.go

@@ -0,0 +1,99 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	delegate2 "github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+)
+
+func NewBasicControllerDelegate(id ControllerId, svc interfaces.ModelBaseService) (d *BasicControllerDelegate) {
+	return &BasicControllerDelegate{
+		id:  id,
+		svc: svc,
+	}
+}
+
+type BasicControllerDelegate struct {
+	id  ControllerId
+	svc interfaces.ModelBaseService
+}
+
+func (d *BasicControllerDelegate) Get(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	doc, err := d.svc.GetById(id)
+	if err == mongo2.ErrNoDocuments {
+		HandleErrorNotFound(c, err)
+		return
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, doc)
+}
+
+func (d *BasicControllerDelegate) Post(c *gin.Context) {
+	doc, err := NewJsonBinder(d.id).Bind(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Add(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, doc)
+}
+
+func (d *BasicControllerDelegate) Put(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	doc, err := NewJsonBinder(d.id).Bind(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if doc.GetId() != id {
+		HandleErrorBadRequest(c, errors.ErrorHttpBadRequest)
+		return
+	}
+	_, err = d.svc.GetById(id)
+	if err != nil {
+		HandleErrorNotFound(c, err)
+		return
+	}
+	if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Save(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, doc)
+}
+
+func (d *BasicControllerDelegate) Delete(c *gin.Context) {
+	id := c.Param("id")
+	oid, err := primitive.ObjectIDFromHex(id)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	doc, err := d.svc.GetById(oid)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Delete(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}

core/controllers/delegate_list.go

@@ -0,0 +1,222 @@
+package controllers
+
+import (
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"reflect"
+	"time"
+)
+
+func NewListControllerDelegate(id ControllerId, svc interfaces.ModelBaseService) (d *ListControllerDelegate) {
+	if svc == nil {
+		panic(errors.ErrorControllerNoModelService)
+	}
+
+	return &ListControllerDelegate{
+		id:  id,
+		svc: svc,
+		bc:  NewBasicControllerDelegate(id, svc),
+	}
+}
+
+type ListControllerDelegate struct {
+	id  ControllerId
+	svc interfaces.ModelBaseService
+	bc  BasicController
+}
+
+func (d *ListControllerDelegate) Get(c *gin.Context) {
+	d.bc.Get(c)
+}
+
+func (d *ListControllerDelegate) Post(c *gin.Context) {
+	d.bc.Post(c)
+}
+
+func (d *ListControllerDelegate) Put(c *gin.Context) {
+	d.bc.Put(c)
+}
+
+func (d *ListControllerDelegate) Delete(c *gin.Context) {
+	d.bc.Delete(c)
+}
+
+func (d *ListControllerDelegate) GetList(c *gin.Context) {
+	// get all if query field "all" is set true
+	all := MustGetFilterAll(c)
+	if all {
+		d.getAll(c)
+		return
+	}
+
+	// get list and total
+	l, total, err := d.getList(c)
+	if err != nil {
+		return
+	}
+
+	// response
+	HandleSuccessWithListData(c, l, total)
+}
+
+func (d *ListControllerDelegate) PostList(c *gin.Context) {
+	// bind
+	docs, err := NewJsonBinder(d.id).BindList(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// success ids
+	var ids []primitive.ObjectID
+
+	// reflect
+	switch reflect.TypeOf(docs).Kind() {
+	case reflect.Slice, reflect.Array:
+		s := reflect.ValueOf(docs)
+		for i := 0; i < s.Len(); i++ {
+			item := s.Index(i)
+			if !item.CanAddr() {
+				HandleErrorInternalServerError(c, errors.ErrorModelInvalidType)
+				return
+			}
+			ptr := item.Addr()
+			doc, ok := ptr.Interface().(interfaces.Model)
+			if !ok {
+				HandleErrorInternalServerError(c, errors.ErrorModelInvalidType)
+				return
+			}
+			if err := delegate.NewModelDelegate(doc, GetUserFromContext(c)).Add(); err != nil {
+				_ = trace.TraceError(err)
+				continue
+			}
+			ids = append(ids, doc.GetId())
+		}
+	}
+
+	// check
+	items, err := utils.GetArrayItems(docs)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	if len(ids) < len(items) {
+		HandleErrorInternalServerError(c, errors.ErrorControllerAddError)
+		return
+	}
+
+	// success
+	HandleSuccessWithData(c, docs)
+}
+
+func (d *ListControllerDelegate) PutList(c *gin.Context) {
+	payload, doc, err := NewJsonBinder(d.id).BindBatchRequestPayloadWithStringData(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// query
+	query := bson.M{
+		"_id": bson.M{
+			"$in": payload.Ids,
+		},
+	}
+
+	// update
+	if err := d.svc.UpdateDoc(query, doc, payload.Fields); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}
+
+func (d *ListControllerDelegate) DeleteList(c *gin.Context) {
+	payload, err := NewJsonBinder(d.id).BindBatchRequestPayload(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := d.svc.DeleteList(bson.M{
+		"_id": bson.M{
+			"$in": payload.Ids,
+		},
+	}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (d *ListControllerDelegate) getAll(c *gin.Context) {
+	// get list
+	tic := time.Now()
+	log.Debugf("getAll -> d.svc.GetMany:start")
+	list, err := d.svc.GetList(nil, &mongo.FindOptions{
+		Sort: bson.D{{"_id", -1}},
+	})
+	if err != nil {
+		if err == mongo2.ErrNoDocuments {
+			HandleErrorNotFound(c, err)
+		} else {
+			HandleErrorInternalServerError(c, err)
+		}
+		return
+	}
+	log.Debugf("getAll -> d.svc.GetMany:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds())
+	tic = time.Now()
+
+	// total count
+	tic = time.Now()
+	log.Debugf("getAll -> d.svc.Count:start")
+	total, err := d.svc.Count(nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	log.Debugf("getAll -> d.svc.Count:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds())
+
+	// response
+	HandleSuccessWithListData(c, list, total)
+}
+
+func (d *ListControllerDelegate) getList(c *gin.Context) (l interfaces.List, total int, err error) {
+	// params
+	pagination := MustGetPagination(c)
+	query := MustGetFilterQuery(c)
+	sort := MustGetSortOption(c)
+
+	// get list
+	l, err = d.svc.GetList(query, &mongo.FindOptions{
+		Sort:  sort,
+		Skip:  pagination.Size * (pagination.Page - 1),
+		Limit: pagination.Size,
+	})
+	if err != nil {
+		if err.Error() == mongo2.ErrNoDocuments.Error() {
+			HandleSuccessWithListData(c, nil, 0)
+		} else {
+			HandleErrorInternalServerError(c, err)
+		}
+		return
+	}
+
+	// total count
+	total, err = d.svc.Count(query)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	return l, total, nil
+}

core/controllers/delegate_list_action.go

@@ -0,0 +1,17 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+)
+
+func NewListPostActionControllerDelegate(id ControllerId, svc interfaces.ModelBaseService, actions []Action) (d *ListActionControllerDelegate) {
+	return &ListActionControllerDelegate{
+		NewListControllerDelegate(id, svc),
+		NewActionControllerDelegate(id, actions),
+	}
+}
+
+type ListActionControllerDelegate struct {
+	ListController
+	ActionController
+}

core/controllers/demo.go

@@ -0,0 +1,73 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+func getDemoActions() []Action {
+	ctx := newDemoContext()
+	return []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/import",
+			HandlerFunc: ctx.import_,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/reimport",
+			HandlerFunc: ctx.reimport,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/cleanup",
+			HandlerFunc: ctx.cleanup,
+		},
+	}
+}
+
+type demoContext struct {
+}
+
+func (ctx *demoContext) import_(c *gin.Context) {
+	if err := utils.ImportDemo(); err != nil {
+		trace.PrintError(err)
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *demoContext) reimport(c *gin.Context) {
+	if err := utils.ReimportDemo(); err != nil {
+		trace.PrintError(err)
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *demoContext) cleanup(c *gin.Context) {
+	if err := utils.ReimportDemo(); err != nil {
+		trace.PrintError(err)
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+var _demoCtx *demoContext
+
+func newDemoContext() *demoContext {
+	if _demoCtx != nil {
+		return _demoCtx
+	}
+
+	_demoCtx = &demoContext{}
+
+	return _demoCtx
+}
+
+var DemoController ActionController

core/controllers/environment.go

@@ -0,0 +1,57 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/service"
+)
+
+var EnvironmentController *environmentController
+
+var EnvironmentActions []Action
+
+type environmentController struct {
+	ListActionControllerDelegate
+	d   ListActionControllerDelegate
+	ctx *environmentContext
+}
+
+type environmentContext struct {
+	modelSvc service.ModelService
+	userSvc  interfaces.UserService
+}
+
+func newEnvironmentContext() *environmentContext {
+	// context
+	ctx := &environmentContext{}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(
+		modelSvc service.ModelService,
+		userSvc interfaces.UserService,
+	) {
+		ctx.modelSvc = modelSvc
+		ctx.userSvc = userSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	return ctx
+}
+
+func newEnvironmentController() *environmentController {
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListPostActionControllerDelegate(ControllerIdEnvironment, modelSvc.GetBaseService(interfaces.ModelIdEnvironment), EnvironmentActions)
+	d := NewListPostActionControllerDelegate(ControllerIdEnvironment, modelSvc.GetBaseService(interfaces.ModelIdEnvironment), EnvironmentActions)
+	ctx := newEnvironmentContext()
+
+	return &environmentController{
+		ListActionControllerDelegate: *ctr,
+		d:                            *d,
+		ctx:                          ctx,
+	}
+}

core/controllers/export.go

@@ -0,0 +1,127 @@
+package controllers
+
+import (
+	"errors"
+	"fmt"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/export"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+var ExportController ActionController
+
+func getExportActions() []Action {
+	ctx := newExportContext()
+	return []Action{
+		{
+			Method:      http.MethodPost,
+			Path:        "/:type",
+			HandlerFunc: ctx.postExport,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:type/:id",
+			HandlerFunc: ctx.getExport,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:type/:id/download",
+			HandlerFunc: ctx.getExportDownload,
+		},
+	}
+}
+
+type exportContext struct {
+	csvSvc  interfaces.ExportService
+	jsonSvc interfaces.ExportService
+}
+
+func (ctx *exportContext) postExport(c *gin.Context) {
+	exportType := c.Param("type")
+	exportTarget := c.Query("target")
+	exportFilter, _ := GetFilter(c)
+
+	var exportId string
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exportId, err = ctx.csvSvc.Export(exportType, exportTarget, exportFilter)
+	case constants.ExportTypeJson:
+		exportId, err = ctx.jsonSvc.Export(exportType, exportTarget, exportFilter)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+		return
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, exportId)
+}
+
+func (ctx *exportContext) getExport(c *gin.Context) {
+	exportType := c.Param("type")
+	exportId := c.Param("id")
+
+	var exp interfaces.Export
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exp, err = ctx.csvSvc.GetExport(exportId)
+	case constants.ExportTypeJson:
+		exp, err = ctx.jsonSvc.GetExport(exportId)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, exp)
+}
+
+func (ctx *exportContext) getExportDownload(c *gin.Context) {
+	exportType := c.Param("type")
+	exportId := c.Param("id")
+
+	var exp interfaces.Export
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exp, err = ctx.csvSvc.GetExport(exportId)
+	case constants.ExportTypeJson:
+		exp, err = ctx.jsonSvc.GetExport(exportId)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	switch exportType {
+	case constants.ExportTypeCsv:
+		c.Header("Content-Type", "text/csv")
+	case constants.ExportTypeJson:
+		c.Header("Content-Type", "text/plain")
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", exp.GetDownloadPath()))
+	c.File(exp.GetDownloadPath())
+}
+
+func newExportContext() *exportContext {
+	return &exportContext{
+		csvSvc:  export.GetCsvService(),
+		jsonSvc: export.GetJsonService(),
+	}
+}

core/controllers/export_v2.go

@@ -0,0 +1,87 @@
+package controllers
+
+import (
+	"errors"
+	"fmt"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/export"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/gin-gonic/gin"
+)
+
+func PostExport(c *gin.Context) {
+	exportType := c.Param("type")
+	exportTarget := c.Query("target")
+	exportFilter, _ := GetFilter(c)
+
+	var exportId string
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exportId, err = export.GetCsvService().Export(exportType, exportTarget, exportFilter)
+	case constants.ExportTypeJson:
+		exportId, err = export.GetJsonService().Export(exportType, exportTarget, exportFilter)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+		return
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, exportId)
+}
+
+func GetExport(c *gin.Context) {
+	exportType := c.Param("type")
+	exportId := c.Param("id")
+
+	var exp interfaces.Export
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exp, err = export.GetCsvService().GetExport(exportId)
+	case constants.ExportTypeJson:
+		exp, err = export.GetJsonService().GetExport(exportId)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, exp)
+}
+
+func GetExportDownload(c *gin.Context) {
+	exportType := c.Param("type")
+	exportId := c.Param("id")
+
+	var exp interfaces.Export
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exp, err = export.GetCsvService().GetExport(exportId)
+	case constants.ExportTypeJson:
+		exp, err = export.GetJsonService().GetExport(exportId)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	switch exportType {
+	case constants.ExportTypeCsv:
+		c.Header("Content-Type", "text/csv")
+	case constants.ExportTypeJson:
+		c.Header("Content-Type", "text/plain")
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", exp.GetDownloadPath()))
+	c.File(exp.GetDownloadPath())
+}

core/controllers/filer.go

@@ -0,0 +1,130 @@
+package controllers
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/gin-gonic/gin"
+	"github.com/imroc/req"
+	"github.com/spf13/viper"
+	"net/http"
+	"strings"
+)
+
+var FilerController ActionController
+
+func getFilerActions() []Action {
+	filerCtx := newFilerContext()
+	return []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "*path",
+			HandlerFunc: filerCtx.do,
+		},
+		{
+			Method:      http.MethodPost,
+			Path:        "*path",
+			HandlerFunc: filerCtx.do,
+		},
+		{
+			Method:      http.MethodPut,
+			Path:        "*path",
+			HandlerFunc: filerCtx.do,
+		},
+		{
+			Method:      http.MethodDelete,
+			Path:        "*path",
+			HandlerFunc: filerCtx.do,
+		},
+	}
+}
+
+type filerContext struct {
+	endpoint string
+}
+
+func (ctx *filerContext) do(c *gin.Context) {
+	// request path
+	requestPath := strings.Replace(c.Request.URL.Path, "/filer", "", 1)
+
+	// request url
+	requestUrl := fmt.Sprintf("%s%s", ctx.endpoint, requestPath)
+	if c.Request.URL.RawQuery != "" {
+		requestUrl += "?" + c.Request.URL.RawQuery
+	}
+
+	// request body
+	bufR := bufio.NewScanner(c.Request.Body)
+	requestBody := req.BodyJSON(bufR.Bytes())
+
+	// request file uploads
+	var requestFileUploads []req.FileUpload
+	form, err := c.MultipartForm()
+	if err == nil {
+		for k, v := range form.File {
+			for _, fh := range v {
+				f, err := fh.Open()
+				if err != nil {
+					HandleErrorInternalServerError(c, err)
+					return
+				}
+				requestFileUploads = append(requestFileUploads, req.FileUpload{
+					FileName:  fh.Filename,
+					FieldName: k,
+					File:      f,
+				})
+			}
+		}
+	}
+
+	// request header
+	requestHeader := req.Header{}
+	for k, v := range c.Request.Header {
+		if len(v) > 0 {
+			requestHeader[k] = v[0]
+		}
+	}
+
+	// perform request
+	res, err := req.Do(c.Request.Method, requestUrl, requestHeader, requestBody, requestFileUploads)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// status code check
+	statusCode := res.Response().StatusCode
+	if statusCode == http.StatusNotFound {
+		HandleErrorNotFoundNoPrint(c, errors.ErrorControllerFilerNotFound)
+		return
+	}
+
+	// response
+	for k, v := range res.Response().Header {
+		if len(v) > 0 {
+			c.Header(k, v[0])
+		}
+	}
+	_, _ = c.Writer.Write(res.Bytes())
+	c.AbortWithStatus(statusCode)
+}
+
+var _filerCtx *filerContext
+
+func newFilerContext() *filerContext {
+	if _filerCtx != nil {
+		return _filerCtx
+	}
+
+	ctx := &filerContext{
+		endpoint: "http://localhost:8888",
+	}
+
+	if viper.GetString("fs.filer.proxy") != "" {
+		ctx.endpoint = viper.GetString("fs.filer.proxy")
+	}
+
+	_filerCtx = ctx
+
+	return ctx
+}

core/controllers/filter.go

@@ -0,0 +1,100 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"net/http"
+)
+
+var FilterController ActionController
+
+func getFilterActions() []Action {
+	ctx := newFilterContext()
+	return []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/:col",
+			HandlerFunc: ctx.getColFieldOptions,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:col/:value",
+			HandlerFunc: ctx.getColFieldOptions,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:col/:value/:label",
+			HandlerFunc: ctx.getColFieldOptions,
+		},
+	}
+}
+
+type filterContext struct {
+}
+
+func (ctx *filterContext) getColFieldOptions(c *gin.Context) {
+	colName := c.Param("col")
+	value := c.Param("value")
+	if value == "" {
+		value = "_id"
+	}
+	label := c.Param("label")
+	if label == "" {
+		label = "name"
+	}
+	query := MustGetFilterQuery(c)
+	pipelines := mongo2.Pipeline{}
+	if query != nil {
+		pipelines = append(pipelines, bson.D{{"$match", query}})
+	}
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$group",
+				bson.M{
+					"_id": bson.M{
+						"value": "$" + value,
+						"label": "$" + label,
+					},
+				},
+			},
+		},
+	)
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$project",
+				bson.M{
+					"value": "$_id.value",
+					"label": bson.M{"$toString": "$_id.label"},
+				},
+			},
+		},
+	)
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$sort",
+				bson.D{
+					{"value", 1},
+				},
+			},
+		},
+	)
+	var options []entity.FilterSelectOption
+	if err := mongo.GetMongoCol(colName).Aggregate(pipelines, nil).All(&options); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, options)
+}
+
+func newFilterContext() *filterContext {
+	return &filterContext{}
+}

core/controllers/filter_v2.go

@@ -0,0 +1,69 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+)
+
+func GetFilterColFieldOptions(c *gin.Context) {
+	colName := c.Param("col")
+	value := c.Param("value")
+	if value == "" {
+		value = "_id"
+	}
+	label := c.Param("label")
+	if label == "" {
+		label = "name"
+	}
+	query := MustGetFilterQuery(c)
+	pipelines := mongo2.Pipeline{}
+	if query != nil {
+		pipelines = append(pipelines, bson.D{{"$match", query}})
+	}
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$group",
+				bson.M{
+					"_id": bson.M{
+						"value": "$" + value,
+						"label": "$" + label,
+					},
+				},
+			},
+		},
+	)
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$project",
+				bson.M{
+					"value": "$_id.value",
+					"label": bson.M{"$toString": "$_id.label"},
+				},
+			},
+		},
+	)
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$sort",
+				bson.D{
+					{"value", 1},
+				},
+			},
+		},
+	)
+	var options []entity.FilterSelectOption
+	if err := mongo.GetMongoCol(colName).Aggregate(pipelines, nil).All(&options); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, options)
+}

core/controllers/git.go

@@ -0,0 +1,3 @@
+package controllers
+
+var GitController ListController

core/controllers/http.go

@@ -0,0 +1,16 @@
+package controllers
+
+type Response[T any] struct {
+	Status  string `json:"status"`
+	Message string `json:"message"`
+	Data    T      `json:"data"`
+	Error   string `json:"error"`
+}
+
+type ListResponse[T any] struct {
+	Status  string `json:"status"`
+	Message string `json:"message"`
+	Total   int    `json:"total"`
+	Data    []T    `json:"data"`
+	Error   string `json:"error"`
+}

core/controllers/init.go

@@ -0,0 +1,42 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/service"
+)
+
+func InitControllers() (err error) {
+	modelSvc, err := service.GetService()
+	if err != nil {
+		return err
+	}
+
+	NodeController = newNodeController()
+	ProjectController = newProjectController()
+	SpiderController = newSpiderController()
+	TaskController = newTaskController()
+	UserController = newUserController()
+	TagController = NewListControllerDelegate(ControllerIdTag, modelSvc.GetBaseService(interfaces.ModelIdTag))
+	SettingController = newSettingController()
+	LoginController = NewActionControllerDelegate(ControllerIdLogin, getLoginActions())
+	DataCollectionController = newDataCollectionController()
+	ResultController = NewActionControllerDelegate(ControllerIdResult, getResultActions())
+	ScheduleController = newScheduleController()
+	StatsController = NewActionControllerDelegate(ControllerIdStats, getStatsActions())
+	TokenController = newTokenController()
+	FilerController = NewActionControllerDelegate(ControllerIdFiler, getFilerActions())
+	GitController = NewListControllerDelegate(ControllerIdGit, modelSvc.GetBaseService(interfaces.ModelIdGit))
+	VersionController = NewActionControllerDelegate(ControllerIdVersion, getVersionActions())
+	SystemInfoController = NewActionControllerDelegate(ControllerIdSystemInfo, getSystemInfoActions())
+	DemoController = NewActionControllerDelegate(ControllerIdDemo, getDemoActions())
+	RoleController = NewListControllerDelegate(ControllerIdRole, modelSvc.GetBaseService(interfaces.ModelIdRole))
+	PermissionController = NewListControllerDelegate(ControllerIdPermission, modelSvc.GetBaseService(interfaces.ModelIdPermission))
+	ExportController = NewActionControllerDelegate(ControllerIdExport, getExportActions())
+	NotificationController = NewActionControllerDelegate(ControllerIdNotification, getNotificationActions())
+	FilterController = NewActionControllerDelegate(ControllerIdFilter, getFilterActions())
+	SyncController = NewActionControllerDelegate(ControllerIdSync, getSyncActions())
+	DataSourceController = newDataSourceController()
+	EnvironmentController = newEnvironmentController()
+
+	return nil
+}

core/controllers/login.go

@@ -0,0 +1,64 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+var LoginController ActionController
+
+func getLoginActions() []Action {
+	loginCtx := newLoginContext()
+	return []Action{
+		{Method: http.MethodPost, Path: "/login", HandlerFunc: loginCtx.login},
+		{Method: http.MethodPost, Path: "/logout", HandlerFunc: loginCtx.logout},
+	}
+}
+
+type loginContext struct {
+	userSvc interfaces.UserService
+}
+
+func (ctx *loginContext) login(c *gin.Context) {
+	var u models.User
+	if err := c.ShouldBindJSON(&u); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	token, loggedInUser, err := ctx.userSvc.Login(&interfaces.UserLoginOptions{
+		Username: u.Username,
+		Password: u.Password,
+	})
+	if err != nil {
+		HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized)
+		return
+	}
+	c.Set(constants.UserContextKey, loggedInUser)
+	HandleSuccessWithData(c, token)
+}
+
+func (ctx *loginContext) logout(c *gin.Context) {
+	c.Set(constants.UserContextKey, nil)
+	HandleSuccess(c)
+}
+
+func newLoginContext() *loginContext {
+	// context
+	ctx := &loginContext{}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(
+		userSvc interfaces.UserService,
+	) {
+		ctx.userSvc = userSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	return ctx
+}

core/controllers/login_v2.go

@@ -0,0 +1,36 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/user"
+	"github.com/gin-gonic/gin"
+)
+
+func PostLogin(c *gin.Context) {
+	var payload struct {
+		Username string `json:"username"`
+		Password string `json:"password"`
+	}
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	userSvc, err := user.GetUserServiceV2()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	token, loggedInUser, err := userSvc.Login(payload.Username, payload.Password)
+	if err != nil {
+		HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized)
+		return
+	}
+	c.Set(constants.UserContextKey, loggedInUser)
+	HandleSuccessWithData(c, token)
+}
+
+func PostLogout(c *gin.Context) {
+	c.Set(constants.UserContextKey, nil)
+	HandleSuccess(c)
+}

core/controllers/node.go

@@ -0,0 +1,94 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/go-trace"
+	"github.com/gin-gonic/gin"
+	"github.com/google/uuid"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+var NodeController *nodeController
+
+type nodeController struct {
+	ListControllerDelegate
+}
+
+func (ctr *nodeController) Post(c *gin.Context) {
+	var n models.Node
+	if err := c.ShouldBindJSON(&n); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	if err := ctr._post(c, &n); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}
+
+func (ctr *nodeController) PostList(c *gin.Context) {
+	// bind
+	var docs []models.Node
+	if err := c.ShouldBindJSON(&docs); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// success ids
+	var ids []primitive.ObjectID
+
+	// iterate nodes
+	for _, n := range docs {
+		if err := ctr._post(c, &n); err != nil {
+			trace.PrintError(err)
+			continue
+		}
+		ids = append(ids, n.Id)
+	}
+
+	// success
+	HandleSuccessWithData(c, docs)
+}
+
+func (ctr *nodeController) _post(c *gin.Context, n *models.Node) (err error) {
+	// set default key
+	if n.Key == "" {
+		id, err := uuid.NewUUID()
+		if err != nil {
+			return trace.TraceError(err)
+		}
+		n.Key = id.String()
+	}
+
+	// set default status
+	if n.Status == "" {
+		n.Status = constants.NodeStatusUnregistered
+	}
+
+	// add
+	if err := delegate.NewModelDelegate(n, GetUserFromContext(c)).Add(); err != nil {
+		return trace.TraceError(err)
+	}
+
+	return nil
+}
+
+func newNodeController() *nodeController {
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListControllerDelegate(ControllerIdNode, modelSvc.GetBaseService(interfaces.ModelIdNode))
+
+	return &nodeController{
+		ListControllerDelegate: *ctr,
+	}
+}

core/controllers/notification.go

@@ -0,0 +1,158 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/notification"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"net/http"
+)
+
+var NotificationController ActionController
+
+func getNotificationActions() []Action {
+	ctx := newNotificationContext()
+	return []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/settings",
+			HandlerFunc: ctx.GetSettingList,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/settings/:id",
+			HandlerFunc: ctx.GetSetting,
+		},
+		{
+			Method:      http.MethodPost,
+			Path:        "/settings",
+			HandlerFunc: ctx.PostSetting,
+		},
+		{
+			Method:      http.MethodPut,
+			Path:        "/settings/:id",
+			HandlerFunc: ctx.PutSetting,
+		},
+		{
+			Method:      http.MethodDelete,
+			Path:        "/settings/:id",
+			HandlerFunc: ctx.DeleteSetting,
+		},
+		{
+			Method:      http.MethodPost,
+			Path:        "/settings/:id/enable",
+			HandlerFunc: ctx.EnableSetting,
+		},
+		{
+			Method:      http.MethodPost,
+			Path:        "/settings/:id/disable",
+			HandlerFunc: ctx.DisableSetting,
+		},
+	}
+}
+
+type notificationContext struct {
+	svc *notification.Service
+}
+
+func (ctx *notificationContext) GetSettingList(c *gin.Context) {
+	query := MustGetFilterQuery(c)
+	pagination := MustGetPagination(c)
+	sort := MustGetSortOption(c)
+	res, total, err := ctx.svc.GetSettingList(query, pagination, sort)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithListData(c, res, total)
+}
+
+func (ctx *notificationContext) GetSetting(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	res, err := ctx.svc.GetSetting(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, res)
+}
+
+func (ctx *notificationContext) PostSetting(c *gin.Context) {
+	var s notification.NotificationSetting
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := ctx.svc.PosSetting(&s); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *notificationContext) PutSetting(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	var s notification.NotificationSetting
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := ctx.svc.PutSetting(id, s); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *notificationContext) DeleteSetting(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := ctx.svc.DeleteSetting(id); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *notificationContext) EnableSetting(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := ctx.svc.EnableSetting(id); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *notificationContext) DisableSetting(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := ctx.svc.DisableSetting(id); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func newNotificationContext() *notificationContext {
+	ctx := &notificationContext{
+		svc: notification.GetService(),
+	}
+	return ctx
+}

core/controllers/permission.go

@@ -0,0 +1,3 @@
+package controllers
+
+var PermissionController ListController

core/controllers/project.go

@@ -0,0 +1,103 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+var ProjectController *projectController
+
+type projectController struct {
+	ListControllerDelegate
+}
+
+func (ctr *projectController) GetList(c *gin.Context) {
+	// get all if query field "all" is set true
+	all := MustGetFilterAll(c)
+	if all {
+		ctr.getAll(c)
+		return
+	}
+
+	// get list
+	list, total, err := ctr.getList(c)
+	if err != nil {
+		return
+	}
+	data := list.GetModels()
+
+	// check empty list
+	if len(list.GetModels()) == 0 {
+		HandleSuccessWithListData(c, nil, 0)
+		return
+	}
+
+	// project ids
+	var ids []primitive.ObjectID
+
+	// count cache
+	cache := map[primitive.ObjectID]int{}
+
+	// iterate
+	for _, d := range data {
+		p, ok := d.(*models.Project)
+		if !ok {
+			HandleErrorInternalServerError(c, errors.ErrorControllerInvalidType)
+			return
+		}
+		ids = append(ids, p.Id)
+		cache[p.Id] = 0
+	}
+
+	// spiders
+	modelSvc, err := service.NewService()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	spiders, err := modelSvc.GetSpiderList(bson.M{
+		"project_id": bson.M{
+			"$in": ids,
+		},
+	}, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	for _, s := range spiders {
+		_, ok := cache[s.ProjectId]
+		if !ok {
+			HandleErrorInternalServerError(c, errors.ErrorControllerMissingInCache)
+			return
+		}
+		cache[s.ProjectId]++
+	}
+
+	// assign
+	var projects []models.Project
+	for _, d := range data {
+		p := d.(*models.Project)
+		p.Spiders = cache[p.Id]
+		projects = append(projects, *p)
+	}
+
+	HandleSuccessWithListData(c, projects, total)
+}
+
+func newProjectController() *projectController {
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListControllerDelegate(ControllerIdProject, modelSvc.GetBaseService(interfaces.ModelIdProject))
+
+	return &projectController{
+		ListControllerDelegate: *ctr,
+	}
+}

core/controllers/result.go

@@ -0,0 +1,150 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-db/generic"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/result"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"net/http"
+)
+
+var ResultController ActionController
+
+func getResultActions() []Action {
+	var resultCtx = newResultContext()
+	return []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/:id",
+			HandlerFunc: resultCtx.getList,
+		},
+	}
+}
+
+type resultContext struct {
+	modelSvc service.ModelService
+}
+
+func (ctx *resultContext) getList(c *gin.Context) {
+	// data collection id
+	dcId, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// data source id
+	var dsId primitive.ObjectID
+	dsIdStr := c.Query("data_source_id")
+	if dsIdStr != "" {
+		dsId, err = primitive.ObjectIDFromHex(dsIdStr)
+		if err != nil {
+			HandleErrorBadRequest(c, err)
+			return
+		}
+	}
+
+	// data collection
+	dc, err := ctx.modelSvc.GetDataCollectionById(dcId)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// data source
+	ds, err := ctx.modelSvc.GetDataSourceById(dsId)
+	if err != nil {
+		if err.Error() == mongo2.ErrNoDocuments.Error() {
+			ds = &models.DataSource{}
+		} else {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+
+	// spider
+	sq := bson.M{
+		"col_id":         dc.Id,
+		"data_source_id": ds.Id,
+	}
+	s, err := ctx.modelSvc.GetSpider(sq, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// service
+	svc, err := result.GetResultService(s.Id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// params
+	pagination := MustGetPagination(c)
+	query := ctx.getListQuery(c)
+
+	// get results
+	data, err := svc.List(query, &generic.ListOptions{
+		Sort:  []generic.ListSort{{"_id", generic.SortDirectionDesc}},
+		Skip:  pagination.Size * (pagination.Page - 1),
+		Limit: pagination.Size,
+	})
+	if err != nil {
+		if err.Error() == mongo2.ErrNoDocuments.Error() {
+			HandleSuccessWithListData(c, nil, 0)
+			return
+		}
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// validate results
+	if len(data) == 0 {
+		HandleSuccessWithListData(c, nil, 0)
+		return
+	}
+
+	// total count
+	total, err := svc.Count(query)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// response
+	HandleSuccessWithListData(c, data, total)
+}
+
+func (ctx *resultContext) getListQuery(c *gin.Context) (q generic.ListQuery) {
+	f, err := GetFilter(c)
+	if err != nil {
+		return q
+	}
+	for _, cond := range f.Conditions {
+		q = append(q, generic.ListQueryCondition{
+			Key:   cond.Key,
+			Op:    cond.Op,
+			Value: utils.NormalizeObjectId(cond.Value),
+		})
+	}
+	return q
+}
+
+func newResultContext() *resultContext {
+	// context
+	ctx := &resultContext{}
+
+	var err error
+	ctx.modelSvc, err = service.NewService()
+	if err != nil {
+		panic(err)
+	}
+
+	return ctx
+}

core/controllers/result_v2.go

@@ -0,0 +1,119 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-db/generic"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/result"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+)
+
+func GetResultList(c *gin.Context) {
+	// data collection id
+	dcId, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// data source id
+	var dsId primitive.ObjectID
+	dsIdStr := c.Query("data_source_id")
+	if dsIdStr != "" {
+		dsId, err = primitive.ObjectIDFromHex(dsIdStr)
+		if err != nil {
+			HandleErrorBadRequest(c, err)
+			return
+		}
+	}
+
+	// data collection
+	dc, err := service.NewModelServiceV2[models.DataCollectionV2]().GetById(dcId)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// data source
+	ds, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(dsId)
+	if err != nil {
+		if err.Error() == mongo2.ErrNoDocuments.Error() {
+			ds = &models.DataSourceV2{}
+		} else {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+
+	// spider
+	sq := bson.M{
+		"col_id":         dc.Id,
+		"data_source_id": ds.Id,
+	}
+	s, err := service.NewModelServiceV2[models.SpiderV2]().GetOne(sq, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// service
+	svc, err := result.GetResultService(s.Id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// params
+	pagination := MustGetPagination(c)
+	query := getResultListQuery(c)
+
+	// get results
+	data, err := svc.List(query, &generic.ListOptions{
+		Sort:  []generic.ListSort{{"_id", generic.SortDirectionDesc}},
+		Skip:  pagination.Size * (pagination.Page - 1),
+		Limit: pagination.Size,
+	})
+	if err != nil {
+		if err.Error() == mongo2.ErrNoDocuments.Error() {
+			HandleSuccessWithListData(c, nil, 0)
+			return
+		}
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// validate results
+	if len(data) == 0 {
+		HandleSuccessWithListData(c, nil, 0)
+		return
+	}
+
+	// total count
+	total, err := svc.Count(query)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// response
+	HandleSuccessWithListData(c, data, total)
+}
+
+func getResultListQuery(c *gin.Context) (q generic.ListQuery) {
+	f, err := GetFilter(c)
+	if err != nil {
+		return q
+	}
+	for _, cond := range f.Conditions {
+		q = append(q, generic.ListQueryCondition{
+			Key:   cond.Key,
+			Op:    cond.Op,
+			Value: utils.NormalizeObjectId(cond.Value),
+		})
+	}
+	return q
+}

core/controllers/role.go

@@ -0,0 +1,3 @@
+package controllers
+
+var RoleController ListController

core/controllers/router_v2.go

@@ -0,0 +1,387 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/middlewares"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+type RouterGroups struct {
+	AuthGroup      *gin.RouterGroup
+	AnonymousGroup *gin.RouterGroup
+}
+
+func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) {
+	return &RouterGroups{
+		AuthGroup:      app.Group("/", middlewares.AuthorizationMiddlewareV2()),
+		AnonymousGroup: app.Group("/"),
+	}
+}
+
+func RegisterController[T any](group *gin.RouterGroup, basePath string, ctr *BaseControllerV2[T]) {
+	actionPaths := make(map[string]bool)
+	for _, action := range ctr.actions {
+		group.Handle(action.Method, basePath+action.Path, action.HandlerFunc)
+		path := basePath + action.Path
+		key := action.Method + " - " + path
+		actionPaths[key] = true
+	}
+	registerBuiltinHandler(group, http.MethodGet, basePath+"", ctr.GetList, actionPaths)
+	registerBuiltinHandler(group, http.MethodGet, basePath+"/:id", ctr.GetById, actionPaths)
+	registerBuiltinHandler(group, http.MethodPost, basePath+"", ctr.Post, actionPaths)
+	registerBuiltinHandler(group, http.MethodPut, basePath+"/:id", ctr.PutById, actionPaths)
+	registerBuiltinHandler(group, http.MethodPatch, basePath+"", ctr.PatchList, actionPaths)
+	registerBuiltinHandler(group, http.MethodDelete, basePath+"/:id", ctr.DeleteById, actionPaths)
+	registerBuiltinHandler(group, http.MethodDelete, basePath+"", ctr.DeleteList, actionPaths)
+}
+
+func RegisterActions(group *gin.RouterGroup, basePath string, actions []Action) {
+	for _, action := range actions {
+		group.Handle(action.Method, basePath+action.Path, action.HandlerFunc)
+	}
+}
+
+func registerBuiltinHandler(group *gin.RouterGroup, method, path string, handlerFunc gin.HandlerFunc, existingActionPaths map[string]bool) {
+	key := method + " - " + path
+	_, ok := existingActionPaths[key]
+	if ok {
+		return
+	}
+	group.Handle(method, path, handlerFunc)
+}
+
+func InitRoutes(app *gin.Engine) (err error) {
+	// routes groups
+	groups := NewRouterGroups(app)
+
+	RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models.DataCollectionV2]())
+	RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models.DataSourceV2]())
+	RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models.EnvironmentV2]())
+	RegisterController(groups.AuthGroup, "/gits", NewControllerV2[models.GitV2]())
+	RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]())
+	RegisterController(groups.AuthGroup, "/notifications/settings", NewControllerV2[models.SettingV2]())
+	RegisterController(groups.AuthGroup, "/permissions", NewControllerV2[models.PermissionV2]())
+	RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models.ProjectV2]())
+	RegisterController(groups.AuthGroup, "/roles", NewControllerV2[models.RoleV2]())
+	RegisterController(groups.AuthGroup, "/schedules", NewControllerV2[models.ScheduleV2](
+		Action{
+			Method:      http.MethodPost,
+			Path:        "",
+			HandlerFunc: PostSchedule,
+		},
+		Action{
+			Method:      http.MethodPut,
+			Path:        "/:id",
+			HandlerFunc: PutScheduleById,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/enable",
+			HandlerFunc: PostScheduleEnable,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/disable",
+			HandlerFunc: PostScheduleDisable,
+		},
+	))
+	RegisterController(groups.AuthGroup, "/spiders", NewControllerV2[models.SpiderV2](
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id",
+			HandlerFunc: GetSpiderById,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "",
+			HandlerFunc: GetSpiderList,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "",
+			HandlerFunc: PostSpider,
+		},
+		Action{
+			Method:      http.MethodPut,
+			Path:        "/:id",
+			HandlerFunc: PutSpiderById,
+		},
+		Action{
+			Method:      http.MethodDelete,
+			Path:        "/:id",
+			HandlerFunc: DeleteSpiderById,
+		},
+		Action{
+			Method:      http.MethodDelete,
+			Path:        "",
+			HandlerFunc: DeleteSpiderList,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/files/list",
+			HandlerFunc: GetSpiderListDir,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/files/get",
+			HandlerFunc: GetSpiderFile,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/files/info",
+			HandlerFunc: GetSpiderFileInfo,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/files/save",
+			HandlerFunc: PostSpiderSaveFile,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/files/save/batch",
+			HandlerFunc: PostSpiderSaveFiles,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/files/save/dir",
+			HandlerFunc: PostSpiderSaveDir,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/files/rename",
+			HandlerFunc: PostSpiderRenameFile,
+		},
+		Action{
+			Method:      http.MethodDelete,
+			Path:        "/:id/files",
+			HandlerFunc: DeleteSpiderFile,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/files/copy",
+			HandlerFunc: PostSpiderCopyFile,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/files/export",
+			HandlerFunc: PostSpiderExport,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/run",
+			HandlerFunc: PostSpiderRun,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/git",
+			HandlerFunc: GetSpiderGit,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/git/remote-refs",
+			HandlerFunc: GetSpiderGitRemoteRefs,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/git/checkout",
+			HandlerFunc: PostSpiderGitCheckout,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/git/pull",
+			HandlerFunc: PostSpiderGitPull,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/git/commit",
+			HandlerFunc: PostSpiderGitCommit,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/data-source",
+			HandlerFunc: GetSpiderDataSource,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/data-source/:ds_id",
+			HandlerFunc: PostSpiderDataSource,
+		},
+	))
+	RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models.TaskV2](
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id",
+			HandlerFunc: GetTaskById,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "",
+			HandlerFunc: GetTaskList,
+		},
+		Action{
+			Method:      http.MethodDelete,
+			Path:        "/:id",
+			HandlerFunc: DeleteTaskById,
+		},
+		Action{
+			Method:      http.MethodDelete,
+			Path:        "",
+			HandlerFunc: DeleteList,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/run",
+			HandlerFunc: PostTaskRun,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/restart",
+			HandlerFunc: PostTaskRestart,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/cancel",
+			HandlerFunc: PostTaskCancel,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/logs",
+			HandlerFunc: GetTaskLogs,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/data",
+			HandlerFunc: GetTaskData,
+		},
+	))
+	RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models.TokenV2](
+		Action{
+			Method:      http.MethodPost,
+			Path:        "",
+			HandlerFunc: PostToken,
+		},
+	))
+	RegisterController(groups.AuthGroup, "/users", NewControllerV2[models.UserV2](
+		Action{
+			Method:      http.MethodPost,
+			Path:        "",
+			HandlerFunc: PostUser,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/change-password",
+			HandlerFunc: PostUserChangePassword,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/me",
+			HandlerFunc: GetUserMe,
+		},
+		Action{
+			Method:      http.MethodPut,
+			Path:        "/me",
+			HandlerFunc: PutUserById,
+		},
+	))
+
+	RegisterActions(groups.AuthGroup, "/results", []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/:id",
+			HandlerFunc: GetResultList,
+		},
+	})
+	RegisterActions(groups.AuthGroup, "/export", []Action{
+		{
+			Method:      http.MethodPost,
+			Path:        "/:type",
+			HandlerFunc: PostExport,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:type/:id",
+			HandlerFunc: GetExport,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:type/:id/download",
+			HandlerFunc: GetExportDownload,
+		},
+	})
+	RegisterActions(groups.AuthGroup, "/filters", []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/:col",
+			HandlerFunc: GetFilterColFieldOptions,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:col/:value",
+			HandlerFunc: GetFilterColFieldOptions,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:col/:value/:label",
+			HandlerFunc: GetFilterColFieldOptions,
+		},
+	})
+	RegisterActions(groups.AuthGroup, "/settings", []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/:id",
+			HandlerFunc: GetSetting,
+		},
+		{
+			Method:      http.MethodPut,
+			Path:        "/:id",
+			HandlerFunc: PutSetting,
+		},
+	})
+	RegisterActions(groups.AuthGroup, "/stats", []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/overview",
+			HandlerFunc: GetStatsOverview,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/daily",
+			HandlerFunc: GetStatsDaily,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/tasks",
+			HandlerFunc: GetStatsTasks,
+		},
+	})
+
+	RegisterActions(groups.AnonymousGroup, "/system-info", []Action{
+		{
+			Path:        "",
+			Method:      http.MethodGet,
+			HandlerFunc: GetSystemInfo,
+		},
+	})
+	RegisterActions(groups.AnonymousGroup, "/version", []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "",
+			HandlerFunc: GetVersion,
+		},
+	})
+	RegisterActions(groups.AnonymousGroup, "/", []Action{
+		{
+			Method:      http.MethodPost,
+			Path:        "/login",
+			HandlerFunc: PostLogin,
+		},
+		{
+			Method:      http.MethodPost,
+			Path:        "/logout",
+			HandlerFunc: PostLogout,
+		},
+	})
+
+	return nil
+}

core/controllers/router_v2_test.go

@@ -0,0 +1,91 @@
+package controllers_test
+
+import (
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestRouterGroups(t *testing.T) {
+	router := gin.Default()
+	groups := controllers.NewRouterGroups(router)
+
+	assertions := []struct {
+		group *gin.RouterGroup
+		name  string
+	}{
+		{groups.AuthGroup, "AuthGroup"},
+		{groups.AnonymousGroup, "AnonymousGroup"},
+	}
+
+	for _, a := range assertions {
+		assert.NotNil(t, a.group, a.name+" should not be nil")
+	}
+}
+
+func TestRegisterController_Routes(t *testing.T) {
+	router := gin.Default()
+	groups := controllers.NewRouterGroups(router)
+	ctr := controllers.NewControllerV2[models.TestModel]()
+	basePath := "/testmodels"
+
+	controllers.RegisterController(groups.AuthGroup, basePath, ctr)
+
+	// Check if all routes are registered
+	routes := router.Routes()
+
+	var methodPaths []string
+	for _, route := range routes {
+		methodPaths = append(methodPaths, route.Method+" - "+route.Path)
+	}
+
+	expectedRoutes := []gin.RouteInfo{
+		{Method: "GET", Path: basePath},
+		{Method: "GET", Path: basePath + "/:id"},
+		{Method: "POST", Path: basePath},
+		{Method: "PUT", Path: basePath + "/:id"},
+		{Method: "PATCH", Path: basePath},
+		{Method: "DELETE", Path: basePath + "/:id"},
+		{Method: "DELETE", Path: basePath},
+	}
+
+	assert.Equal(t, len(expectedRoutes), len(routes))
+	for _, route := range expectedRoutes {
+		assert.Contains(t, methodPaths, route.Method+" - "+route.Path)
+	}
+}
+
+func TestInitRoutes_ProjectsRoute(t *testing.T) {
+	router := gin.Default()
+
+	controllers.InitRoutes(router)
+
+	// Check if the projects route is registered
+	routes := router.Routes()
+
+	var methodPaths []string
+	for _, route := range routes {
+		methodPaths = append(methodPaths, route.Method+" - "+route.Path)
+	}
+
+	expectedRoutes := []gin.RouteInfo{
+		{Method: "GET", Path: "/projects"},
+		{Method: "GET", Path: "/projects/:id"},
+		{Method: "POST", Path: "/projects"},
+		{Method: "PUT", Path: "/projects/:id"},
+		{Method: "PATCH", Path: "/projects"},
+		{Method: "DELETE", Path: "/projects/:id"},
+		{Method: "DELETE", Path: "/projects"},
+	}
+
+	for _, route := range expectedRoutes {
+		assert.Contains(t, methodPaths, route.Method+" - "+route.Path)
+	}
+}
+
+func TestMain(m *testing.M) {
+	gin.SetMode(gin.TestMode)
+	m.Run()
+}

core/controllers/schedule.go

@@ -0,0 +1,221 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"net/http"
+)
+
+var ScheduleController *scheduleController
+
+func getScheduleActions() []Action {
+	scheduleCtx := newScheduleContext()
+	return []Action{
+		{
+			Method:      http.MethodPost,
+			Path:        "/:id/enable",
+			HandlerFunc: scheduleCtx.enable,
+		},
+		{
+			Method:      http.MethodPost,
+			Path:        "/:id/disable",
+			HandlerFunc: scheduleCtx.disable,
+		},
+	}
+}
+
+type scheduleController struct {
+	ListActionControllerDelegate
+	d   ListActionControllerDelegate
+	ctx *scheduleContext
+}
+
+func (ctr *scheduleController) Post(c *gin.Context) {
+	var s models.Schedule
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := delegate.NewModelDelegate(&s, GetUserFromContext(c)).Add(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	if s.Enabled {
+		if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+	HandleSuccessWithData(c, s)
+}
+
+func (ctr *scheduleController) Put(c *gin.Context) {
+	id := c.Param("id")
+	oid, err := primitive.ObjectIDFromHex(id)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	var s models.Schedule
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if s.GetId() != oid {
+		HandleErrorBadRequest(c, errors.ErrorHttpBadRequest)
+		return
+	}
+	if err := delegate.NewModelDelegate(&s).Save(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	if s.Enabled {
+		if err := ctr.ctx.scheduleSvc.Disable(&s, GetUserFromContext(c)); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+		if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+	HandleSuccessWithData(c, s)
+}
+
+func (ctr *scheduleController) Delete(c *gin.Context) {
+	id := c.Param("id")
+	oid, err := primitive.ObjectIDFromHex(id)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	s, err := ctr.ctx.modelSvc.GetScheduleById(oid)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	if err := ctr.ctx.scheduleSvc.Disable(s); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	if err := delegate.NewModelDelegate(s, GetUserFromContext(c)).Delete(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+}
+
+func (ctr *scheduleController) DeleteList(c *gin.Context) {
+	payload, err := NewJsonBinder(interfaces.ModelIdSchedule).BindBatchRequestPayload(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	for _, id := range payload.Ids {
+		s, err := ctr.ctx.modelSvc.GetScheduleById(id)
+		if err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+		if err := ctr.ctx.scheduleSvc.Disable(s); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+	if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdSchedule).DeleteList(bson.M{
+		"_id": bson.M{
+			"$in": payload.Ids,
+		},
+	}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *scheduleContext) enable(c *gin.Context) {
+	s, err := ctx._getSchedule(c)
+	if err != nil {
+		return
+	}
+	if err := ctx.scheduleSvc.Enable(s, GetUserFromContext(c)); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *scheduleContext) disable(c *gin.Context) {
+	s, err := ctx._getSchedule(c)
+	if err != nil {
+		return
+	}
+	if err := ctx.scheduleSvc.Disable(s, GetUserFromContext(c)); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *scheduleContext) _getSchedule(c *gin.Context) (s *models.Schedule, err error) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	s, err = ctx.modelSvc.GetScheduleById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	return s, nil
+}
+
+type scheduleContext struct {
+	modelSvc    service.ModelService
+	scheduleSvc interfaces.ScheduleService
+}
+
+func newScheduleContext() *scheduleContext {
+	// context
+	ctx := &scheduleContext{}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(
+		modelSvc service.ModelService,
+		scheduleSvc interfaces.ScheduleService,
+	) {
+		ctx.modelSvc = modelSvc
+		ctx.scheduleSvc = scheduleSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	return ctx
+}
+
+func newScheduleController() *scheduleController {
+	actions := getScheduleActions()
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions)
+	d := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions)
+	ctx := newScheduleContext()
+
+	return &scheduleController{
+		ListActionControllerDelegate: *ctr,
+		d:                            *d,
+		ctx:                          ctx,
+	}
+}

core/controllers/schedule_v2.go

@@ -0,0 +1,130 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/schedule"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func PostSchedule(c *gin.Context) {
+	var s models.ScheduleV2
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	u := GetUserFromContextV2(c)
+
+	modelSvc := service.NewModelServiceV2[models.ScheduleV2]()
+
+	s.SetCreated(u.Id)
+	s.SetUpdated(u.Id)
+	id, err := modelSvc.InsertOne(s)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	s.Id = id
+
+	if s.Enabled {
+		scheduleSvc, err := schedule.GetScheduleServiceV2()
+		if err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+		if err := scheduleSvc.Enable(s, u.Id); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+
+	HandleSuccessWithData(c, s)
+}
+
+func PutScheduleById(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	var s models.ScheduleV2
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if s.Id != id {
+		HandleErrorBadRequest(c, errors.ErrorHttpBadRequest)
+		return
+	}
+
+	modelSvc := service.NewModelServiceV2[models.ScheduleV2]()
+	err = modelSvc.ReplaceById(id, s)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	scheduleSvc, err := schedule.GetScheduleServiceV2()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	u := GetUserFromContextV2(c)
+
+	if s.Enabled {
+		if err := scheduleSvc.Enable(s, u.Id); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	} else {
+		if err := scheduleSvc.Disable(s, u.Id); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+
+	HandleSuccessWithData(c, s)
+}
+
+func PostScheduleEnable(c *gin.Context) {
+	postScheduleEnableDisableFunc(true)(c)
+}
+
+func PostScheduleDisable(c *gin.Context) {
+	postScheduleEnableDisableFunc(false)(c)
+}
+
+func postScheduleEnableDisableFunc(isEnable bool) func(c *gin.Context) {
+	return func(c *gin.Context) {
+		id, err := primitive.ObjectIDFromHex(c.Param("id"))
+		if err != nil {
+			HandleErrorBadRequest(c, err)
+			return
+		}
+		svc, err := schedule.GetScheduleServiceV2()
+		if err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+		s, err := service.NewModelServiceV2[models.ScheduleV2]().GetById(id)
+		if err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+		u := GetUserFromContextV2(c)
+		if isEnable {
+			err = svc.Enable(*s, u.Id)
+		} else {
+			err = svc.Disable(*s, u.Id)
+		}
+		if err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+		HandleSuccess(c)
+	}
+}

core/controllers/setting.go

@@ -0,0 +1,84 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+)
+
+var SettingController *settingController
+
+type settingController struct {
+	ListControllerDelegate
+}
+
+func (ctr *settingController) Get(c *gin.Context) {
+	// key
+	key := c.Param("id")
+
+	// model service
+	modelSvc, err := service.NewService()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// setting
+	s, err := modelSvc.GetSettingByKey(key, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, s)
+}
+
+func (ctr *settingController) Put(c *gin.Context) {
+	// key
+	key := c.Param("id")
+
+	// settings
+	var s models.Setting
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// model service
+	modelSvc, err := service.NewService()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// setting
+	_s, err := modelSvc.GetSettingByKey(key, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// save
+	_s.Value = s.Value
+	if err := delegate.NewModelDelegate(_s).Save(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}
+
+func newSettingController() *settingController {
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListControllerDelegate(ControllerIdSetting, modelSvc.GetBaseService(interfaces.ModelIdSetting))
+
+	return &settingController{
+		ListControllerDelegate: *ctr,
+	}
+}

core/controllers/setting_v2.go

@@ -0,0 +1,56 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+func GetSetting(c *gin.Context) {
+	// key
+	key := c.Param("id")
+
+	// setting
+	s, err := service.NewModelServiceV2[models.SettingV2]().GetOne(bson.M{"key": key}, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, s)
+}
+
+func PutSetting(c *gin.Context) {
+	// key
+	key := c.Param("id")
+
+	// settings
+	var s models.Setting
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	modelSvc := service.NewModelServiceV2[models.SettingV2]()
+
+	// setting
+	_s, err := modelSvc.GetOne(bson.M{"key": key}, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	u := GetUserFromContextV2(c)
+
+	// save
+	_s.Value = s.Value
+	_s.SetUpdated(u.Id)
+	err = modelSvc.ReplaceOne(bson.M{"key": key}, *_s)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}

core/controllers/spider_v2_test.go

@@ -0,0 +1,249 @@
+package controllers_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/middlewares"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestCreateSpider(t *testing.T) {
+	SetupTestDB()
+	defer CleanupTestDB()
+
+	gin.SetMode(gin.TestMode)
+
+	router := gin.Default()
+	router.Use(middlewares.AuthorizationMiddlewareV2())
+	router.POST("/spiders", controllers.PostSpider)
+
+	payload := models.SpiderV2{
+		Name:    "Test Spider",
+		ColName: "test_spiders",
+	}
+	jsonValue, _ := json.Marshal(payload)
+	req, _ := http.NewRequest("POST", "/spiders", bytes.NewBuffer(jsonValue))
+	req.Header.Set("Authorization", TestToken)
+	resp := httptest.NewRecorder()
+
+	router.ServeHTTP(resp, req)
+
+	assert.Equal(t, http.StatusOK, resp.Code)
+
+	var response controllers.Response[models.SpiderV2]
+	err := json.Unmarshal(resp.Body.Bytes(), &response)
+	require.Nil(t, err)
+	assert.False(t, response.Data.Id.IsZero())
+	assert.Equal(t, payload.Name, response.Data.Name)
+	assert.False(t, response.Data.ColId.IsZero())
+}
+
+func TestGetSpiderById(t *testing.T) {
+	SetupTestDB()
+	defer CleanupTestDB()
+
+	gin.SetMode(gin.TestMode)
+
+	router := gin.Default()
+	router.Use(middlewares.AuthorizationMiddlewareV2())
+	router.GET("/spiders/:id", controllers.GetSpiderById)
+
+	model := models.SpiderV2{
+		Name:    "Test Spider",
+		ColName: "test_spiders",
+	}
+	id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model)
+	require.Nil(t, err)
+	ts := models.SpiderStatV2{}
+	ts.SetId(id)
+	_, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts)
+	require.Nil(t, err)
+
+	req, _ := http.NewRequest("GET", "/spiders/"+id.Hex(), nil)
+	req.Header.Set("Authorization", TestToken)
+	resp := httptest.NewRecorder()
+
+	router.ServeHTTP(resp, req)
+
+	assert.Equal(t, http.StatusOK, resp.Code)
+
+	var response controllers.Response[models.SpiderV2]
+	err = json.Unmarshal(resp.Body.Bytes(), &response)
+	require.Nil(t, err)
+	assert.Equal(t, model.Name, response.Data.Name)
+}
+
+func TestUpdateSpiderById(t *testing.T) {
+	SetupTestDB()
+	defer CleanupTestDB()
+
+	gin.SetMode(gin.TestMode)
+
+	router := gin.Default()
+	router.Use(middlewares.AuthorizationMiddlewareV2())
+	router.PUT("/spiders/:id", controllers.PutSpiderById)
+
+	model := models.SpiderV2{
+		Name:    "Test Spider",
+		ColName: "test_spiders",
+	}
+	id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model)
+	require.Nil(t, err)
+	ts := models.SpiderStatV2{}
+	ts.SetId(id)
+	_, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts)
+	require.Nil(t, err)
+
+	spiderId := id.Hex()
+	payload := models.SpiderV2{
+		Name:    "Updated Spider",
+		ColName: "test_spider",
+	}
+	payload.SetId(id)
+	jsonValue, _ := json.Marshal(payload)
+	req, _ := http.NewRequest("PUT", "/spiders/"+spiderId, bytes.NewBuffer(jsonValue))
+	req.Header.Set("Authorization", TestToken)
+	resp := httptest.NewRecorder()
+
+	router.ServeHTTP(resp, req)
+
+	assert.Equal(t, http.StatusOK, resp.Code)
+
+	var response controllers.Response[models.SpiderV2]
+	err = json.Unmarshal(resp.Body.Bytes(), &response)
+	require.Nil(t, err)
+	assert.Equal(t, payload.Name, response.Data.Name)
+
+	svc := service.NewModelServiceV2[models.SpiderV2]()
+	resModel, err := svc.GetById(id)
+	require.Nil(t, err)
+	assert.Equal(t, payload.Name, resModel.Name)
+}
+
+func TestDeleteSpiderById(t *testing.T) {
+	SetupTestDB()
+	defer CleanupTestDB()
+
+	gin.SetMode(gin.TestMode)
+
+	router := gin.Default()
+	router.Use(middlewares.AuthorizationMiddlewareV2())
+	router.DELETE("/spiders/:id", controllers.DeleteSpiderById)
+
+	model := models.SpiderV2{
+		Name:    "Test Spider",
+		ColName: "test_spiders",
+	}
+	id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model)
+	require.Nil(t, err)
+	ts := models.SpiderStatV2{}
+	ts.SetId(id)
+	_, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts)
+	require.Nil(t, err)
+	task := models.TaskV2{}
+	task.SpiderId = id
+	taskId, err := service.NewModelServiceV2[models.TaskV2]().InsertOne(task)
+	require.Nil(t, err)
+	taskStat := models.TaskStatV2{}
+	taskStat.SetId(taskId)
+	_, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(taskStat)
+	require.Nil(t, err)
+
+	req, _ := http.NewRequest("DELETE", "/spiders/"+id.Hex(), nil)
+	req.Header.Set("Authorization", TestToken)
+	resp := httptest.NewRecorder()
+
+	router.ServeHTTP(resp, req)
+
+	assert.Equal(t, http.StatusOK, resp.Code)
+
+	_, err = service.NewModelServiceV2[models.SpiderV2]().GetById(id)
+	assert.NotNil(t, err)
+	_, err = service.NewModelServiceV2[models.SpiderStatV2]().GetById(id)
+	assert.NotNil(t, err)
+	taskCount, err := service.NewModelServiceV2[models.TaskV2]().Count(bson.M{"spider_id": id})
+	require.Nil(t, err)
+	assert.Equal(t, 0, taskCount)
+	taskStatCount, err := service.NewModelServiceV2[models.TaskStatV2]().Count(bson.M{"_id": taskId})
+	require.Nil(t, err)
+	assert.Equal(t, 0, taskStatCount)
+
+}
+
+func TestDeleteSpiderList(t *testing.T) {
+	SetupTestDB()
+	defer CleanupTestDB()
+
+	gin.SetMode(gin.TestMode)
+
+	router := gin.Default()
+	router.Use(middlewares.AuthorizationMiddlewareV2())
+	router.DELETE("/spiders", controllers.DeleteSpiderList)
+
+	modelList := []models.SpiderV2{
+		{
+			Name:    "Test Name 1",
+			ColName: "test_spiders",
+		}, {
+			Name:    "Test Name 2",
+			ColName: "test_spiders",
+		},
+	}
+	var ids []primitive.ObjectID
+	var taskIds []primitive.ObjectID
+	for _, model := range modelList {
+		id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model)
+		require.Nil(t, err)
+		ts := models.SpiderStatV2{}
+		ts.SetId(id)
+		_, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts)
+		require.Nil(t, err)
+		task := models.TaskV2{}
+		task.SpiderId = id
+		taskId, err := service.NewModelServiceV2[models.TaskV2]().InsertOne(task)
+		require.Nil(t, err)
+		taskStat := models.TaskStatV2{}
+		taskStat.SetId(taskId)
+		_, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(taskStat)
+		require.Nil(t, err)
+		ids = append(ids, id)
+		taskIds = append(taskIds, taskId)
+	}
+
+	payload := struct {
+		Ids []primitive.ObjectID `json:"ids"`
+	}{
+		Ids: ids,
+	}
+	jsonValue, _ := json.Marshal(payload)
+	req, _ := http.NewRequest("DELETE", "/spiders", bytes.NewBuffer(jsonValue))
+	req.Header.Set("Authorization", TestToken)
+	resp := httptest.NewRecorder()
+
+	router.ServeHTTP(resp, req)
+
+	assert.Equal(t, http.StatusOK, resp.Code)
+
+	spiderCount, err := service.NewModelServiceV2[models.SpiderV2]().Count(bson.M{"_id": bson.M{"$in": ids}})
+	require.Nil(t, err)
+	assert.Equal(t, 0, spiderCount)
+	spiderStatCount, err := service.NewModelServiceV2[models.SpiderStatV2]().Count(bson.M{"_id": bson.M{"$in": ids}})
+	require.Nil(t, err)
+	assert.Equal(t, 0, spiderStatCount)
+	taskCount, err := service.NewModelServiceV2[models.TaskV2]().Count(bson.M{"_id": bson.M{"$in": taskIds}})
+	require.Nil(t, err)
+	assert.Equal(t, 0, taskCount)
+	taskStatCount, err := service.NewModelServiceV2[models.TaskStatV2]().Count(bson.M{"_id": bson.M{"$in": taskIds}})
+	require.Nil(t, err)
+	assert.Equal(t, 0, taskStatCount)
+}

core/controllers/stats.go

@@ -0,0 +1,87 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"net/http"
+	"time"
+)
+
+var StatsController ActionController
+
+func getStatsActions() []Action {
+	statsCtx := newStatsContext()
+	return []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/overview",
+			HandlerFunc: statsCtx.getOverview,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/daily",
+			HandlerFunc: statsCtx.getDaily,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/tasks",
+			HandlerFunc: statsCtx.getTasks,
+		},
+	}
+}
+
+type statsContext struct {
+	statsSvc     interfaces.StatsService
+	defaultQuery bson.M
+}
+
+func (svc *statsContext) getOverview(c *gin.Context) {
+	data, err := svc.statsSvc.GetOverviewStats(svc.defaultQuery)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, data)
+}
+
+func (svc *statsContext) getDaily(c *gin.Context) {
+	data, err := svc.statsSvc.GetDailyStats(svc.defaultQuery)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, data)
+}
+
+func (svc *statsContext) getTasks(c *gin.Context) {
+	data, err := svc.statsSvc.GetTaskStats(svc.defaultQuery)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, data)
+}
+
+func newStatsContext() *statsContext {
+	// context
+	ctx := &statsContext{
+		defaultQuery: bson.M{
+			"create_ts": bson.M{
+				"$gte": time.Now().Add(-30 * 24 * time.Hour),
+			},
+		},
+	}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(
+		statsSvc interfaces.StatsService,
+	) {
+		ctx.statsSvc = statsSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	return ctx
+}

core/controllers/stats_v2.go

@@ -0,0 +1,41 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/stats"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"time"
+)
+
+var statsDefaultQuery = bson.M{
+	"create_ts": bson.M{
+		"$gte": time.Now().Add(-30 * 24 * time.Hour),
+	},
+}
+
+func GetStatsOverview(c *gin.Context) {
+	data, err := stats.GetStatsService().GetOverviewStats(statsDefaultQuery)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, data)
+}
+
+func GetStatsDaily(c *gin.Context) {
+	data, err := stats.GetStatsService().GetDailyStats(statsDefaultQuery)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, data)
+}
+
+func GetStatsTasks(c *gin.Context) {
+	data, err := stats.GetStatsService().GetTaskStats(statsDefaultQuery)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, data)
+}

core/controllers/sync.go

@@ -0,0 +1,57 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/gin-gonic/gin"
+	"github.com/spf13/viper"
+	"net/http"
+	"path/filepath"
+)
+
+var SyncController ActionController
+
+func getSyncActions() []Action {
+	var ctx = newSyncContext()
+	return []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/:id/scan",
+			HandlerFunc: ctx.scan,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:id/download",
+			HandlerFunc: ctx.download,
+		},
+	}
+}
+
+type syncContext struct {
+}
+
+func (ctx *syncContext) scan(c *gin.Context) {
+	id := c.Param("id")
+	dir := ctx._getDir(id)
+	files, err := utils.ScanDirectory(dir)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	c.AbortWithStatusJSON(http.StatusOK, files)
+}
+
+func (ctx *syncContext) download(c *gin.Context) {
+	id := c.Param("id")
+	filePath := c.Query("path")
+	dir := ctx._getDir(id)
+	c.File(filepath.Join(dir, filePath))
+}
+
+func (ctx *syncContext) _getDir(id string) string {
+	workspacePath := viper.GetString("workspace")
+	return filepath.Join(workspacePath, id)
+}
+
+func newSyncContext() syncContext {
+	return syncContext{}
+}

core/controllers/system_info.go

@@ -0,0 +1,28 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/gin-gonic/gin"
+	"github.com/spf13/viper"
+	"net/http"
+)
+
+func getSystemInfo(c *gin.Context) {
+	info := &entity.SystemInfo{
+		Edition: viper.GetString("info.edition"),
+		Version: viper.GetString("info.version"),
+	}
+	HandleSuccessWithData(c, info)
+}
+
+func getSystemInfoActions() []Action {
+	return []Action{
+		{
+			Path:        "",
+			Method:      http.MethodGet,
+			HandlerFunc: getSystemInfo,
+		},
+	}
+}
+
+var SystemInfoController ActionController

core/controllers/system_info_v2.go

@@ -0,0 +1,15 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/gin-gonic/gin"
+	"github.com/spf13/viper"
+)
+
+func GetSystemInfo(c *gin.Context) {
+	info := &entity.SystemInfo{
+		Edition: viper.GetString("info.edition"),
+		Version: viper.GetString("info.version"),
+	}
+	HandleSuccessWithData(c, info)
+}

core/controllers/tag.go

@@ -0,0 +1,3 @@
+package controllers
+
+var TagController ListController

core/controllers/task.go

@@ -0,0 +1,534 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-db/generic"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	delegate2 "github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/result"
+	"github.com/crawlab-team/crawlab/core/task/log"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"net/http"
+	"strings"
+)
+
+var TaskController *taskController
+
+func getTaskActions() []Action {
+	taskCtx := newTaskContext()
+	return []Action{
+		{
+			Method:      http.MethodPost,
+			Path:        "/run",
+			HandlerFunc: taskCtx.run,
+		},
+		{
+			Method:      http.MethodPost,
+			Path:        "/:id/restart",
+			HandlerFunc: taskCtx.restart,
+		},
+		{
+			Method:      http.MethodPost,
+			Path:        "/:id/cancel",
+			HandlerFunc: taskCtx.cancel,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:id/logs",
+			HandlerFunc: taskCtx.getLogs,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:id/data",
+			HandlerFunc: taskCtx.getData,
+		},
+	}
+}
+
+type taskController struct {
+	ListActionControllerDelegate
+	d   ListActionControllerDelegate
+	ctx *taskContext
+}
+
+func (ctr *taskController) Get(c *gin.Context) {
+	ctr.ctx.getWithStatsSpider(c)
+}
+
+func (ctr *taskController) Delete(c *gin.Context) {
+	if err := ctr.ctx._delete(c); err != nil {
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctr *taskController) GetList(c *gin.Context) {
+	withStats := c.Query("stats")
+	if withStats == "" {
+		ctr.d.GetList(c)
+		return
+	}
+	ctr.ctx.getListWithStats(c)
+}
+
+func (ctr *taskController) DeleteList(c *gin.Context) {
+	if err := ctr.ctx._deleteList(c); err != nil {
+		return
+	}
+	HandleSuccess(c)
+}
+
+type taskContext struct {
+	modelSvc         service.ModelService
+	modelTaskSvc     interfaces.ModelBaseService
+	modelTaskStatSvc interfaces.ModelBaseService
+	adminSvc         interfaces.SpiderAdminService
+	schedulerSvc     interfaces.TaskSchedulerService
+	l                log.Driver
+}
+
+func (ctx *taskContext) run(c *gin.Context) {
+	// task
+	var t models.Task
+	if err := c.ShouldBindJSON(&t); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// validate spider id
+	if t.GetSpiderId().IsZero() {
+		HandleErrorBadRequest(c, errors.ErrorTaskEmptySpiderId)
+		return
+	}
+
+	// spider
+	s, err := ctx.modelSvc.GetSpiderById(t.GetSpiderId())
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// options
+	opts := &interfaces.SpiderRunOptions{
+		Mode:     t.Mode,
+		NodeIds:  t.NodeIds,
+		Cmd:      t.Cmd,
+		Param:    t.Param,
+		Priority: t.Priority,
+	}
+
+	// user
+	if u := GetUserFromContext(c); u != nil {
+		opts.UserId = u.GetId()
+	}
+
+	// run
+	taskIds, err := ctx.adminSvc.Schedule(s.GetId(), opts)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, taskIds)
+}
+
+func (ctx *taskContext) restart(c *gin.Context) {
+	// id
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// task
+	t, err := ctx.modelSvc.GetTaskById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// options
+	opts := &interfaces.SpiderRunOptions{
+		Mode:     t.Mode,
+		NodeIds:  t.NodeIds,
+		Cmd:      t.Cmd,
+		Param:    t.Param,
+		Priority: t.Priority,
+	}
+
+	// user
+	if u := GetUserFromContext(c); u != nil {
+		opts.UserId = u.GetId()
+	}
+
+	// run
+	taskIds, err := ctx.adminSvc.Schedule(t.SpiderId, opts)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, taskIds)
+}
+
+func (ctx *taskContext) cancel(c *gin.Context) {
+	// id
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// task
+	t, err := ctx.modelSvc.GetTaskById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// validate
+	if !utils.IsCancellable(t.Status) {
+		HandleErrorInternalServerError(c, errors.ErrorControllerNotCancellable)
+		return
+	}
+
+	// cancel
+	if err := ctx.schedulerSvc.Cancel(id, GetUserFromContext(c)); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}
+
+func (ctx *taskContext) getLogs(c *gin.Context) {
+	// id
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// pagination
+	p, err := GetPagination(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// logs
+	logs, err := ctx.l.Find(id.Hex(), "", (p.Page-1)*p.Size, p.Size)
+	if err != nil {
+		if strings.HasSuffix(err.Error(), "Status:404 Not Found") {
+			HandleSuccess(c)
+			return
+		}
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	total, err := ctx.l.Count(id.Hex(), "")
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithListData(c, logs, total)
+}
+
+func (ctx *taskContext) getListWithStats(c *gin.Context) {
+	// params
+	pagination := MustGetPagination(c)
+	query := MustGetFilterQuery(c)
+	sort := MustGetSortOption(c)
+
+	// get list
+	list, err := ctx.modelTaskSvc.GetList(query, &mongo.FindOptions{
+		Sort:  sort,
+		Skip:  pagination.Size * (pagination.Page - 1),
+		Limit: pagination.Size,
+	})
+	if err != nil {
+		if err == mongo2.ErrNoDocuments {
+			HandleErrorNotFound(c, err)
+		} else {
+			HandleErrorInternalServerError(c, err)
+		}
+		return
+	}
+
+	// check empty list
+	if len(list.GetModels()) == 0 {
+		HandleSuccessWithListData(c, nil, 0)
+		return
+	}
+
+	// ids
+	var ids []primitive.ObjectID
+	for _, d := range list.GetModels() {
+		t := d.(interfaces.Model)
+		ids = append(ids, t.GetId())
+	}
+
+	// total count
+	total, err := ctx.modelTaskSvc.Count(query)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// stat list
+	query = bson.M{
+		"_id": bson.M{
+			"$in": ids,
+		},
+	}
+	stats, err := ctx.modelSvc.GetTaskStatList(query, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// cache stat list to dict
+	dict := map[primitive.ObjectID]models.TaskStat{}
+	for _, s := range stats {
+		dict[s.GetId()] = s
+	}
+
+	// iterate list again
+	var data []interface{}
+	for _, d := range list.GetModels() {
+		t := d.(*models.Task)
+		s, ok := dict[t.GetId()]
+		if ok {
+			t.Stat = &s
+		}
+		data = append(data, *t)
+	}
+
+	// response
+	HandleSuccessWithListData(c, data, total)
+}
+
+func (ctx *taskContext) getWithStatsSpider(c *gin.Context) {
+	// id
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// task
+	t, err := ctx.modelSvc.GetTaskById(id)
+	if err == mongo2.ErrNoDocuments {
+		HandleErrorNotFound(c, err)
+		return
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// spider
+	t.Spider, _ = ctx.modelSvc.GetSpiderById(t.SpiderId)
+
+	// skip if task status is pending
+	if t.Status == constants.TaskStatusPending {
+		HandleSuccessWithData(c, t)
+		return
+	}
+
+	// task stat
+	t.Stat, _ = ctx.modelSvc.GetTaskStatById(id)
+
+	HandleSuccessWithData(c, t)
+}
+
+func (ctx *taskContext) getData(c *gin.Context) {
+	// id
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// pagination
+	p, err := GetPagination(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// task
+	t, err := ctx.modelSvc.GetTaskById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// result service
+	resultSvc, err := result.GetResultService(t.SpiderId)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// query
+	query := generic.ListQuery{
+		generic.ListQueryCondition{
+			Key:   constants.TaskKey,
+			Op:    generic.OpEqual,
+			Value: t.Id,
+		},
+	}
+
+	// list
+	data, err := resultSvc.List(query, &generic.ListOptions{
+		Skip:  (p.Page - 1) * p.Size,
+		Limit: p.Size,
+		Sort:  []generic.ListSort{{"_id", generic.SortDirectionDesc}},
+	})
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// total
+	total, err := resultSvc.Count(query)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithListData(c, data, total)
+}
+
+func (ctx *taskContext) _delete(c *gin.Context) (err error) {
+	id := c.Param("id")
+	oid, err := primitive.ObjectIDFromHex(id)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) {
+		// delete task
+		task, err := ctx.modelSvc.GetTaskById(oid)
+		if err != nil {
+			return err
+		}
+		if err := delegate2.NewModelDelegate(task, GetUserFromContext(c)).Delete(); err != nil {
+			return err
+		}
+
+		// delete task stat
+		taskStat, err := ctx.modelSvc.GetTaskStatById(oid)
+		if err != nil {
+			return err
+		}
+		if err := delegate2.NewModelDelegate(taskStat, GetUserFromContext(c)).Delete(); err != nil {
+			return err
+		}
+
+		return nil
+	}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return err
+	}
+
+	return nil
+}
+
+func (ctx *taskContext) _deleteList(c *gin.Context) (err error) {
+	payload, err := NewJsonBinder(ControllerIdTask).BindBatchRequestPayload(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	if err := mongo.RunTransaction(func(context mongo2.SessionContext) error {
+		// delete tasks
+		if err := ctx.modelTaskSvc.DeleteList(bson.M{
+			"_id": bson.M{
+				"$in": payload.Ids,
+			},
+		}); err != nil {
+			return err
+		}
+
+		// delete task stats
+		if err := ctx.modelTaskStatSvc.DeleteList(bson.M{
+			"_id": bson.M{
+				"$in": payload.Ids,
+			},
+		}); err != nil {
+			return err
+		}
+
+		return nil
+	}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return err
+	}
+
+	return nil
+}
+
+func newTaskContext() *taskContext {
+	// context
+	ctx := &taskContext{}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(
+		modelSvc service.ModelService,
+		adminSvc interfaces.SpiderAdminService,
+		schedulerSvc interfaces.TaskSchedulerService,
+	) {
+		ctx.modelSvc = modelSvc
+		ctx.adminSvc = adminSvc
+		ctx.schedulerSvc = schedulerSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	// model task service
+	ctx.modelTaskSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTask)
+
+	// model task stat service
+	ctx.modelTaskStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTaskStat)
+
+	// log driver
+	l, err := log.GetLogDriver(log.DriverTypeFile)
+	if err != nil {
+		panic(err)
+	}
+	ctx.l = l
+
+	return ctx
+}
+
+func newTaskController() *taskController {
+	actions := getTaskActions()
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListPostActionControllerDelegate(ControllerIdTask, modelSvc.GetBaseService(interfaces.ModelIdTask), actions)
+	d := NewListPostActionControllerDelegate(ControllerIdTask, modelSvc.GetBaseService(interfaces.ModelIdTask), actions)
+	ctx := newTaskContext()
+
+	return &taskController{
+		ListActionControllerDelegate: *ctr,
+		d:                            *d,
+		ctx:                          ctx,
+	}
+}

core/controllers/task_v2.go

@@ -0,0 +1,465 @@
+package controllers
+
+import (
+	"errors"
+	log2 "github.com/apex/log"
+	"github.com/crawlab-team/crawlab-db/generic"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/result"
+	"github.com/crawlab-team/crawlab/core/spider/admin"
+	"github.com/crawlab-team/crawlab/core/task/log"
+	"github.com/crawlab-team/crawlab/core/task/scheduler"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/gin-gonic/gin"
+	"github.com/spf13/viper"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+)
+
+func GetTaskById(c *gin.Context) {
+	// id
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// task
+	t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id)
+	if errors.Is(err, mongo2.ErrNoDocuments) {
+		HandleErrorNotFound(c, err)
+		return
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// spider
+	t.Spider, _ = service.NewModelServiceV2[models.SpiderV2]().GetById(t.SpiderId)
+
+	// skip if task status is pending
+	if t.Status == constants.TaskStatusPending {
+		HandleSuccessWithData(c, t)
+		return
+	}
+
+	// task stat
+	t.Stat, _ = service.NewModelServiceV2[models.TaskStatV2]().GetById(id)
+
+	HandleSuccessWithData(c, t)
+}
+
+func GetTaskList(c *gin.Context) {
+	withStats := c.Query("stats")
+	if withStats == "" {
+		NewControllerV2[models.TaskV2]().GetList(c)
+		return
+	}
+
+	// params
+	pagination := MustGetPagination(c)
+	query := MustGetFilterQuery(c)
+	sort := MustGetSortOption(c)
+
+	// get list
+	list, err := service.NewModelServiceV2[models.TaskV2]().GetMany(query, &mongo.FindOptions{
+		Sort:  sort,
+		Skip:  pagination.Size * (pagination.Page - 1),
+		Limit: pagination.Size,
+	})
+	if err != nil {
+		if errors.Is(err, mongo2.ErrNoDocuments) {
+			HandleErrorNotFound(c, err)
+		} else {
+			HandleErrorInternalServerError(c, err)
+		}
+		return
+	}
+
+	// check empty list
+	if len(list) == 0 {
+		HandleSuccessWithListData(c, nil, 0)
+		return
+	}
+
+	// ids
+	var ids []primitive.ObjectID
+	for _, t := range list {
+		ids = append(ids, t.Id)
+	}
+
+	// total count
+	total, err := service.NewModelServiceV2[models.TaskV2]().Count(query)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// stat list
+	query = bson.M{
+		"_id": bson.M{
+			"$in": ids,
+		},
+	}
+	stats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(query, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// cache stat list to dict
+	dict := map[primitive.ObjectID]models.TaskStatV2{}
+	for _, s := range stats {
+		dict[s.Id] = s
+	}
+
+	// iterate list again
+	for i, t := range list {
+		ts, ok := dict[t.Id]
+		if ok {
+			list[i].Stat = &ts
+		}
+	}
+
+	// response
+	HandleSuccessWithListData(c, list, total)
+}
+
+func DeleteTaskById(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// delete in db
+	if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) {
+		// delete task
+		_, err = service.NewModelServiceV2[models.TaskV2]().GetById(id)
+		if err != nil {
+			return err
+		}
+		err = service.NewModelServiceV2[models.TaskV2]().DeleteById(id)
+		if err != nil {
+			return err
+		}
+
+		// delete task stat
+		_, err = service.NewModelServiceV2[models.TaskStatV2]().GetById(id)
+		if err != nil {
+			log2.Warnf("delete task stat error: %s", err.Error())
+			return nil
+		}
+		err = service.NewModelServiceV2[models.TaskStatV2]().DeleteById(id)
+		if err != nil {
+			log2.Warnf("delete task stat error: %s", err.Error())
+			return nil
+		}
+
+		return nil
+	}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// delete task logs
+	logPath := filepath.Join(viper.GetString("log.path"), id.Hex())
+	if err := os.RemoveAll(logPath); err != nil {
+		log2.Warnf("failed to remove task log directory: %s", logPath)
+	}
+
+	HandleSuccess(c)
+}
+
+func DeleteList(c *gin.Context) {
+	var payload struct {
+		Ids []primitive.ObjectID `json:"ids"`
+	}
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	if err := mongo.RunTransaction(func(context mongo2.SessionContext) error {
+		// delete tasks
+		if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{
+			"_id": bson.M{
+				"$in": payload.Ids,
+			},
+		}); err != nil {
+			return err
+		}
+
+		// delete task stats
+		if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{
+			"_id": bson.M{
+				"$in": payload.Ids,
+			},
+		}); err != nil {
+			log2.Warnf("delete task stat error: %s", err.Error())
+			return nil
+		}
+
+		return nil
+	}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// delete tasks logs
+	wg := sync.WaitGroup{}
+	wg.Add(len(payload.Ids))
+	for _, id := range payload.Ids {
+		go func(id string) {
+			// delete task logs
+			logPath := filepath.Join(viper.GetString("log.path"), id)
+			if err := os.RemoveAll(logPath); err != nil {
+				log2.Warnf("failed to remove task log directory: %s", logPath)
+			}
+			wg.Done()
+		}(id.Hex())
+	}
+	wg.Wait()
+
+	HandleSuccess(c)
+}
+
+func PostTaskRun(c *gin.Context) {
+	// task
+	var t models.TaskV2
+	if err := c.ShouldBindJSON(&t); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// validate spider id
+	if t.SpiderId.IsZero() {
+		HandleErrorBadRequest(c, errors.New("spider id is required"))
+		return
+	}
+
+	// spider
+	s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(t.SpiderId)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// options
+	opts := &interfaces.SpiderRunOptions{
+		Mode:     t.Mode,
+		NodeIds:  t.NodeIds,
+		Cmd:      t.Cmd,
+		Param:    t.Param,
+		Priority: t.Priority,
+	}
+
+	// user
+	if u := GetUserFromContextV2(c); u != nil {
+		opts.UserId = u.Id
+	}
+
+	// run
+	adminSvc, err := admin.GetSpiderAdminServiceV2()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	taskIds, err := adminSvc.Schedule(s.Id, opts)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, taskIds)
+
+}
+
+func PostTaskRestart(c *gin.Context) {
+	// id
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// task
+	t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// options
+	opts := &interfaces.SpiderRunOptions{
+		Mode:     t.Mode,
+		NodeIds:  t.NodeIds,
+		Cmd:      t.Cmd,
+		Param:    t.Param,
+		Priority: t.Priority,
+	}
+
+	// user
+	if u := GetUserFromContextV2(c); u != nil {
+		opts.UserId = u.Id
+	}
+
+	// run
+	adminSvc, err := admin.GetSpiderAdminServiceV2()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	taskIds, err := adminSvc.Schedule(t.SpiderId, opts)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, taskIds)
+}
+
+func PostTaskCancel(c *gin.Context) {
+	// id
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// task
+	t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// validate
+	if !utils.IsCancellable(t.Status) {
+		HandleErrorInternalServerError(c, errors.New("task is not cancellable"))
+		return
+	}
+
+	u := GetUserFromContextV2(c)
+
+	// cancel
+	schedulerSvc, err := scheduler.GetTaskSchedulerServiceV2()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	if err := schedulerSvc.Cancel(id, u.Id); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}
+
+func GetTaskLogs(c *gin.Context) {
+	// id
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// pagination
+	p, err := GetPagination(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// logs
+	logDriver, err := log.GetFileLogDriver()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	logs, err := logDriver.Find(id.Hex(), "", (p.Page-1)*p.Size, p.Size)
+	if err != nil {
+		if strings.HasSuffix(err.Error(), "Status:404 Not Found") {
+			HandleSuccess(c)
+			return
+		}
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	total, err := logDriver.Count(id.Hex(), "")
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithListData(c, logs, total)
+}
+
+func GetTaskData(c *gin.Context) {
+	// id
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// pagination
+	p, err := GetPagination(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// task
+	t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// result service
+	resultSvc, err := result.GetResultService(t.SpiderId)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// query
+	query := generic.ListQuery{
+		generic.ListQueryCondition{
+			Key:   constants.TaskKey,
+			Op:    generic.OpEqual,
+			Value: t.Id,
+		},
+	}
+
+	// list
+	data, err := resultSvc.List(query, &generic.ListOptions{
+		Skip:  (p.Page - 1) * p.Size,
+		Limit: p.Size,
+		Sort:  []generic.ListSort{{"_id", generic.SortDirectionDesc}},
+	})
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// total
+	total, err := resultSvc.Count(query)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithListData(c, data, total)
+}

core/controllers/test/base.go

@@ -0,0 +1,104 @@
+package test
+
+import (
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/routes"
+	"github.com/crawlab-team/go-trace"
+	"github.com/gavv/httpexpect/v2"
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/dig"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func init() {
+	var err error
+	T, err = NewTest()
+	if err != nil {
+		panic(err)
+	}
+}
+
+type Test struct {
+	// dependencies
+	modelSvc service.ModelService
+
+	// internals
+	app *gin.Engine
+	svr *httptest.Server
+
+	// test data
+	TestUsername string
+	TestPassword string
+	TestToken    string
+}
+
+func (t *Test) Setup(t2 *testing.T) {
+	//if err := controllers.InitControllers(); err != nil {
+	//	panic(err)
+	//}
+	//t2.Cleanup(t.Cleanup)
+}
+
+func (t *Test) Cleanup() {
+	_ = t.modelSvc.DropAll()
+	time.Sleep(200 * time.Millisecond)
+}
+
+func (t *Test) NewExpect(t2 *testing.T) (e *httpexpect.Expect) {
+	e = httpexpect.New(t2, t.svr.URL)
+	res := e.POST("/login").WithJSON(map[string]string{
+		"username": t.TestUsername,
+		"password": t.TestPassword,
+	}).Expect().JSON().Object()
+	t.TestToken = res.Path("$.data").String().Raw()
+	require.NotEmpty(t2, t.TestToken)
+	return e
+}
+
+func (t *Test) WithAuth(req *httpexpect.Request) *httpexpect.Request {
+	return req.WithHeader("Authorization", t.TestToken)
+}
+
+var T *Test
+
+func NewTest() (res *Test, err error) {
+	// test
+	t := &Test{}
+
+	// gin app
+	t.app = gin.New()
+
+	// http test server
+	t.svr = httptest.NewServer(t.app)
+
+	// init controllers
+	if err := controllers.InitControllers(); err != nil {
+		return nil, err
+	}
+
+	// init routes
+	if err := routes.InitRoutes(t.app); err != nil {
+		return nil, err
+	}
+
+	// dependency injection
+	c := dig.New()
+	if err := c.Provide(service.NewService); err != nil {
+		return nil, trace.TraceError(err)
+	}
+	if err := c.Invoke(func(modelSvc service.ModelService) {
+		t.modelSvc = modelSvc
+	}); err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	// test data
+	t.TestUsername = "admin"
+	t.TestPassword = "admin"
+
+	return t, nil
+}

core/controllers/test/export_test.go

@@ -0,0 +1,59 @@
+package test
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+	"net/http"
+	"testing"
+	"time"
+)
+
+func init() {
+	viper.Set("mongo.db", "crawlab_test")
+}
+
+func TestExportController_Csv(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	// mongo collection
+	colName := "test_collection_for_export"
+	col := mongo.GetMongoCol(colName)
+
+	// insert test data to mongo collection
+	for i := 0; i < 10; i++ {
+		_, err := col.Insert(bson.M{
+			"field1": i + 1,
+			"field2": i + 2,
+			"field3": i + 3,
+			"field4": i + 4,
+		})
+		require.Nil(t, err)
+	}
+
+	// export from mongo collection
+	res := T.WithAuth(e.POST("/export/csv")).
+		WithQuery("target", colName).
+		Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data").NotNull()
+
+	// export id
+	exportId := res.Path("$.data").String().Raw()
+
+	// poll export with export id
+	for i := 0; i < 10; i++ {
+		res = T.WithAuth(e.GET("/export/csv/" + exportId)).Expect().Status(http.StatusOK).JSON().Object()
+		status := res.Path("$.data.status").String().Raw()
+		if status == constants.TaskStatusFinished {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+
+	// download exported csv file
+	csvFileBody := T.WithAuth(e.GET("/export/csv/" + exportId + "/download")).Expect().Status(http.StatusOK).Body()
+	csvFileBody.NotEmpty()
+}

core/controllers/test/filter_test.go

@@ -0,0 +1,76 @@
+package test
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"net/http"
+	"testing"
+)
+
+func init() {
+	viper.Set("mongo.db", "crawlab_test")
+}
+
+func TestFilterController_GetColFieldOptions(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	// mongo collection
+	colName := "test_collection_for_filter"
+	field1 := "field1"
+	field2 := "field2"
+	value1 := "value1"
+	col := mongo.GetMongoCol(colName)
+	n := 10
+	var ids []primitive.ObjectID
+	var names []string
+	for i := 0; i < n; i++ {
+		_id := primitive.NewObjectID()
+		ids = append(ids, _id)
+		name := fmt.Sprintf("name_%d", i)
+		names = append(names, name)
+		_, err := col.Insert(bson.M{field1: value1, field2: i % 2, "name": name, "_id": _id})
+		require.Nil(t, err)
+	}
+
+	// validate filter options field 1
+	res := T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field1, field1))).
+		Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data").NotNull()
+	res.Path("$.data").Array().Length().Equal(1)
+	res.Path("$.data").Array().Element(0).Path("$.value").Equal(value1)
+
+	// validate filter options field 2
+	res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))).
+		Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data").NotNull()
+	res.Path("$.data").Array().Length().Equal(2)
+
+	// validate filter options with query
+	conditions := []entity.Condition{{field2, constants.FilterOpEqual, 0}}
+	conditionsJson, err := json.Marshal(conditions)
+	conditionsJsonStr := string(conditionsJson)
+	require.Nil(t, err)
+	res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))).
+		WithQuery(constants.FilterQueryFieldConditions, conditionsJsonStr).
+		Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data").NotNull()
+	res.Path("$.data").Array().Length().Equal(1)
+
+	// validate filter options (basic path)
+	res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s", colName))).
+		Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data").NotNull()
+	res.Path("$.data").Array().Length().Equal(n)
+	for i := 0; i < n; i++ {
+		res.Path("$.data").Array().Element(i).Object().Value("value").Equal(ids[i])
+		res.Path("$.data").Array().Element(i).Object().Value("label").Equal(names[i])
+	}
+}

core/controllers/test/main_test.go

@@ -0,0 +1,44 @@
+package test
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/user"
+	"go.mongodb.org/mongo-driver/mongo"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	// init user
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+	_, err = modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil)
+	if err != nil {
+		if err.Error() != mongo.ErrNoDocuments.Error() {
+			panic(err)
+		}
+		userSvc, err := user.GetUserService()
+		if err != nil {
+			panic(err)
+		}
+		if err := userSvc.Create(&interfaces.UserCreateOptions{
+			Username: constants.DefaultAdminUsername,
+			Password: constants.DefaultAdminPassword,
+			Role:     constants.RoleAdmin,
+		}); err != nil {
+			panic(err)
+		}
+	}
+
+	if err := controllers.InitControllers(); err != nil {
+		panic(err)
+	}
+
+	m.Run()
+
+	T.Cleanup()
+}

core/controllers/test/project_test.go

@@ -0,0 +1,305 @@
+package test
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"net/http"
+	"testing"
+	"time"
+)
+
+func TestProjectController_Get(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	p := models.Project{
+		Name: "test project",
+	}
+	res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data._id").NotNull()
+	id := res.Path("$.data._id").String().Raw()
+	oid, err := primitive.ObjectIDFromHex(id)
+	require.Nil(t, err)
+	require.False(t, oid.IsZero())
+
+	res = T.WithAuth(e.GET("/projects/" + id)).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data._id").NotNull()
+	res.Path("$.data.name").Equal("test project")
+}
+
+func TestProjectController_Put(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	p := models.Project{
+		Name:        "old name",
+		Description: "old description",
+	}
+
+	// add
+	res := T.WithAuth(e.POST("/projects")).
+		WithJSON(p).
+		Expect().Status(http.StatusOK).
+		JSON().Object()
+	res.Path("$.data._id").NotNull()
+	id := res.Path("$.data._id").String().Raw()
+	oid, err := primitive.ObjectIDFromHex(id)
+	require.Nil(t, err)
+	require.False(t, oid.IsZero())
+
+	// change object
+	p.Id = oid
+	p.Name = "new name"
+	p.Description = "new description"
+
+	// update
+	T.WithAuth(e.PUT("/projects/" + id)).
+		WithJSON(p).
+		Expect().Status(http.StatusOK)
+
+	// check
+	res = T.WithAuth(e.GET("/projects/" + id)).Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data._id").Equal(id)
+	res.Path("$.data.name").Equal("new name")
+	res.Path("$.data.description").Equal("new description")
+}
+
+func TestProjectController_Post(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	p := models.Project{
+		Name:        "test project",
+		Description: "this is a test project",
+	}
+
+	res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data._id").NotNull()
+	res.Path("$.data.name").Equal("test project")
+	res.Path("$.data.description").Equal("this is a test project")
+}
+
+func TestProjectController_Delete(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	p := models.Project{
+		Name:        "test project",
+		Description: "this is a test project",
+	}
+
+	// add
+	res := T.WithAuth(e.POST("/projects")).
+		WithJSON(p).
+		Expect().Status(http.StatusOK).
+		JSON().Object()
+	res.Path("$.data._id").NotNull()
+	id := res.Path("$.data._id").String().Raw()
+	oid, err := primitive.ObjectIDFromHex(id)
+	require.Nil(t, err)
+	require.False(t, oid.IsZero())
+
+	// get
+	res = T.WithAuth(e.GET("/projects/" + id)).
+		Expect().Status(http.StatusOK).
+		JSON().Object()
+	res.Path("$.data._id").NotNull()
+	id = res.Path("$.data._id").String().Raw()
+	oid, err = primitive.ObjectIDFromHex(id)
+	require.Nil(t, err)
+	require.False(t, oid.IsZero())
+
+	// delete
+	T.WithAuth(e.DELETE("/projects/" + id)).
+		Expect().Status(http.StatusOK).
+		JSON().Object()
+
+	// get
+	T.WithAuth(e.GET("/projects/" + id)).
+		Expect().Status(http.StatusNotFound)
+}
+
+func TestProjectController_GetList(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	n := 100 // total
+	bn := 10 // batch
+
+	for i := 0; i < n; i++ {
+		p := models.Project{
+			Name: fmt.Sprintf("test name %d", i+1),
+		}
+		obj := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
+		obj.Path("$.data._id").NotNull()
+	}
+
+	f := entity.Filter{
+		//IsOr: false,
+		Conditions: []*entity.Condition{
+			{Key: "name", Op: constants.FilterOpContains, Value: "test name"},
+		},
+	}
+	condBytes, err := json.Marshal(&f.Conditions)
+	require.Nil(t, err)
+
+	pagination := entity.Pagination{
+		Page: 1,
+		Size: bn,
+	}
+
+	// get list with pagination
+	res := T.WithAuth(e.GET("/projects")).
+		WithQuery("conditions", string(condBytes)).
+		WithQueryObject(pagination).
+		Expect().Status(http.StatusOK).JSON().Object()
+	res.Path("$.data").Array().Length().Equal(bn)
+	res.Path("$.total").Number().Equal(n)
+
+	data := res.Path("$.data").Array()
+	for i := 0; i < bn; i++ {
+		obj := data.Element(i)
+		obj.Path("$.name").Equal(fmt.Sprintf("test name %d", i+1))
+	}
+
+}
+
+func TestProjectController_PostList(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	n := 10
+	var docs []models.Project
+	for i := 0; i < n; i++ {
+		docs = append(docs, models.Project{
+			Name:        fmt.Sprintf("project %d", i+1),
+			Description: "this is a project",
+		})
+	}
+
+	T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK)
+
+	res := T.WithAuth(e.GET("/projects")).
+		WithQueryObject(entity.Pagination{Page: 1, Size: 10}).
+		Expect().Status(http.StatusOK).
+		JSON().Object()
+	res.Path("$.data").Array().Length().Equal(n)
+}
+
+func TestProjectController_DeleteList(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	n := 10
+	var docs []models.Project
+	for i := 0; i < n; i++ {
+		docs = append(docs, models.Project{
+			Name:        fmt.Sprintf("project %d", i+1),
+			Description: "this is a project",
+		})
+	}
+
+	// add
+	res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK).
+		JSON().Object()
+	var ids []primitive.ObjectID
+	data := res.Path("$.data").Array()
+	for i := 0; i < n; i++ {
+		obj := data.Element(i)
+		id := obj.Path("$._id").String().Raw()
+		oid, err := primitive.ObjectIDFromHex(id)
+		require.Nil(t, err)
+		require.False(t, oid.IsZero())
+		ids = append(ids, oid)
+	}
+
+	// delete
+	payload := entity.BatchRequestPayload{
+		Ids: ids,
+	}
+	T.WithAuth(e.DELETE("/projects")).
+		WithJSON(payload).
+		Expect().Status(http.StatusOK)
+
+	// check
+	for _, id := range ids {
+		T.WithAuth(e.GET("/projects/" + id.Hex())).
+			Expect().Status(http.StatusNotFound)
+	}
+
+}
+
+func TestProjectController_PutList(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	// now
+	now := time.Now()
+
+	n := 10
+	var docs []models.Project
+	for i := 0; i < n; i++ {
+		docs = append(docs, models.Project{
+			Name:        "old name",
+			Description: "old description",
+		})
+	}
+
+	// add
+	res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK).
+		JSON().Object()
+	var ids []primitive.ObjectID
+	data := res.Path("$.data").Array()
+	for i := 0; i < n; i++ {
+		obj := data.Element(i)
+		id := obj.Path("$._id").String().Raw()
+		oid, err := primitive.ObjectIDFromHex(id)
+		require.Nil(t, err)
+		require.False(t, oid.IsZero())
+		ids = append(ids, oid)
+	}
+
+	// wait for 100 millisecond
+	time.Sleep(100 * time.Millisecond)
+
+	// update
+	p := models.Project{
+		Name:        "new name",
+		Description: "new description",
+	}
+	dataBytes, err := json.Marshal(&p)
+	require.Nil(t, err)
+	payload := entity.BatchRequestPayloadWithStringData{
+		Ids:  ids,
+		Data: string(dataBytes),
+		Fields: []string{
+			"name",
+			"description",
+		},
+	}
+	T.WithAuth(e.PUT("/projects")).WithJSON(payload).Expect().Status(http.StatusOK)
+
+	// check response data
+	for i := 0; i < n; i++ {
+		res = T.WithAuth(e.GET("/projects/" + ids[i].Hex())).Expect().Status(http.StatusOK).JSON().Object()
+		res.Path("$.data.name").Equal("new name")
+		res.Path("$.data.description").Equal("new description")
+	}
+
+	// check artifacts
+	pl, err := T.modelSvc.GetProjectList(bson.M{"_id": bson.M{"$in": ids}}, nil)
+	require.Nil(t, err)
+	for _, p := range pl {
+		a, err := delegate.NewModelDelegate(&p).GetArtifact()
+		require.Nil(t, err)
+		require.True(t, a.GetSys().GetUpdateTs().After(now))
+		require.True(t, a.GetSys().GetUpdateTs().After(a.GetSys().GetCreateTs()))
+	}
+}

core/controllers/test/spider_test.go

@@ -0,0 +1,183 @@
+package test
+
+import (
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"net/http"
+	"testing"
+)
+
+func TestSpiderController_Delete(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	s := models.Spider{
+		Name:        "test spider",
+		Description: "this is a test spider",
+		ColName:     "test col name",
+	}
+
+	// add spider
+	res := T.WithAuth(e.POST("/spiders")).
+		WithJSON(s).
+		Expect().Status(http.StatusOK).
+		JSON().Object()
+	res.Path("$.data._id").NotNull()
+	id := res.Path("$.data._id").String().Raw()
+	oid, err := primitive.ObjectIDFromHex(id)
+	require.Nil(t, err)
+	require.False(t, oid.IsZero())
+
+	// add tasks
+	var taskIds []primitive.ObjectID
+	tasks := []models.Task{
+		{
+			Id:       primitive.NewObjectID(),
+			SpiderId: oid,
+		},
+		{
+			Id:       primitive.NewObjectID(),
+			SpiderId: oid,
+		},
+	}
+	for _, task := range tasks {
+		// add task
+		err := delegate.NewModelDelegate(&task).Add()
+		require.Nil(t, err)
+
+		// add task stat
+		err = delegate.NewModelDelegate(&models.TaskStat{
+			Id: task.Id,
+		}).Add()
+		require.Nil(t, err)
+
+		taskIds = append(taskIds, task.Id)
+	}
+
+	// delete
+	T.WithAuth(e.DELETE("/spiders/" + id)).
+		Expect().Status(http.StatusOK)
+
+	// get
+	T.WithAuth(e.GET("/spiders/" + id)).
+		Expect().Status(http.StatusNotFound)
+
+	// get tasks
+	for _, task := range tasks {
+		T.WithAuth(e.GET("/tasks/" + task.Id.Hex())).
+			Expect().Status(http.StatusNotFound)
+	}
+
+	// spider stat
+	modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat)
+	spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{
+		"_id": oid,
+	})
+	require.Nil(t, err)
+	require.Zero(t, spiderStatCount)
+
+	// task stats
+	modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
+	taskStatCount, err := modelTaskStatSvc.Count(bson.M{
+		"_id": bson.M{
+			"$in": taskIds,
+		},
+	})
+	require.Nil(t, err)
+	require.Zero(t, taskStatCount)
+}
+
+func TestSpiderController_DeleteList(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	spiders := []models.Spider{
+		{
+			Id:          primitive.NewObjectID(),
+			Name:        "test spider 1",
+			Description: "this is a test spider 1",
+			ColName:     "test col name 1",
+		},
+		{
+			Id:          primitive.NewObjectID(),
+			Name:        "test spider 2",
+			Description: "this is a test spider 2",
+			ColName:     "test col name 2",
+		},
+	}
+
+	// add spiders
+	for _, spider := range spiders {
+		T.WithAuth(e.POST("/spiders")).
+			WithJSON(spider).
+			Expect().Status(http.StatusOK)
+	}
+
+	var spiderIds []primitive.ObjectID
+	var taskIds []primitive.ObjectID
+	for _, spider := range spiders {
+		// task id
+		taskId := primitive.NewObjectID()
+
+		// add task
+		err := delegate.NewModelDelegate(&models.Task{
+			Id:       taskId,
+			SpiderId: spider.Id,
+		}).Add()
+		require.Nil(t, err)
+
+		// add task stats
+		err = delegate.NewModelDelegate(&models.TaskStat{
+			Id: taskId,
+		}).Add()
+		require.Nil(t, err)
+
+		spiderIds = append(spiderIds, spider.Id)
+		taskIds = append(taskIds, taskId)
+	}
+
+	// delete spiders
+	T.WithAuth(e.DELETE("/spiders")).
+		WithJSON(entity.BatchRequestPayload{
+			Ids: spiderIds,
+		}).Expect().Status(http.StatusOK)
+
+	// get spiders
+	for _, spider := range spiders {
+		// get
+		T.WithAuth(e.GET("/spiders/" + spider.Id.Hex())).
+			Expect().Status(http.StatusNotFound)
+	}
+
+	// get tasks
+	for _, taskId := range taskIds {
+		T.WithAuth(e.GET("/tasks/" + taskId.Hex())).
+			Expect().Status(http.StatusNotFound)
+	}
+
+	// spider stat
+	modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat)
+	spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{
+		"_id": bson.M{
+			"$in": spiderIds,
+		},
+	})
+	require.Nil(t, err)
+	require.Zero(t, spiderStatCount)
+
+	// task stats
+	modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
+	taskStatCount, err := modelTaskStatSvc.Count(bson.M{
+		"_id": bson.M{
+			"$in": taskIds,
+		},
+	})
+	require.Nil(t, err)
+	require.Zero(t, taskStatCount)
+}

core/controllers/test/task_test.go

@@ -0,0 +1,102 @@
+package test
+
+import (
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"net/http"
+	"testing"
+)
+
+func TestTaskController_Delete(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	task := models.Task{
+		Id: primitive.NewObjectID(),
+	}
+
+	// add task
+	err := delegate.NewModelDelegate(&task).Add()
+	require.Nil(t, err)
+
+	// add task stat
+	err = delegate.NewModelDelegate(&models.TaskStat{
+		Id: task.Id,
+	}).Add()
+	require.Nil(t, err)
+
+	// delete
+	T.WithAuth(e.DELETE("/tasks/" + task.Id.Hex())).
+		Expect().Status(http.StatusOK)
+
+	// get
+	T.WithAuth(e.GET("/tasks/" + task.Id.Hex())).
+		Expect().Status(http.StatusNotFound)
+
+	// task stats
+	modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
+	taskStatCount, err := modelTaskStatSvc.Count(bson.M{
+		"_id": task.Id,
+	})
+	require.Nil(t, err)
+	require.Zero(t, taskStatCount)
+}
+
+func TestTaskController_DeleteList(t *testing.T) {
+	T.Setup(t)
+	e := T.NewExpect(t)
+
+	tasks := []models.Task{
+		{
+			Id: primitive.NewObjectID(),
+		},
+		{
+			Id: primitive.NewObjectID(),
+		},
+	}
+
+	// add spiders
+	var taskIds []primitive.ObjectID
+	for _, task := range tasks {
+		// add task
+		err := delegate.NewModelDelegate(&task).Add()
+		require.Nil(t, err)
+
+		// add task stat
+		err = delegate.NewModelDelegate(&models.TaskStat{
+			Id: task.Id,
+		}).Add()
+		require.Nil(t, err)
+
+		taskIds = append(taskIds, task.Id)
+	}
+
+	// delete tasks
+	T.WithAuth(e.DELETE("/tasks")).
+		WithJSON(entity.BatchRequestPayload{
+			Ids: taskIds,
+		}).Expect().Status(http.StatusOK)
+
+	// get tasks
+	for _, task := range tasks {
+		// get
+		T.WithAuth(e.GET("/tasks/" + task.Id.Hex())).
+			Expect().Status(http.StatusNotFound)
+	}
+
+	// task stats
+	modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
+	taskStatCount, err := modelTaskStatSvc.Count(bson.M{
+		"_id": bson.M{
+			"$in": taskIds,
+		},
+	})
+	require.Nil(t, err)
+	require.Zero(t, taskStatCount)
+}

core/controllers/token.go

@@ -0,0 +1,84 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+)
+
+var TokenController *tokenController
+
+var TokenActions []Action
+
+type tokenController struct {
+	ListActionControllerDelegate
+	d   ListActionControllerDelegate
+	ctx *tokenContext
+}
+
+func (ctr *tokenController) Post(c *gin.Context) {
+	var err error
+	var t models.Token
+	if err := c.ShouldBindJSON(&t); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	u, err := ctr.ctx.userSvc.GetCurrentUser(c)
+	if err != nil {
+		HandleErrorUnauthorized(c, err)
+		return
+	}
+	t.Token, err = ctr.ctx.userSvc.MakeToken(u)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	if err := delegate.NewModelDelegate(&t, GetUserFromContext(c)).Add(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+type tokenContext struct {
+	modelSvc service.ModelService
+	userSvc  interfaces.UserService
+}
+
+func newTokenContext() *tokenContext {
+	// context
+	ctx := &tokenContext{}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(
+		modelSvc service.ModelService,
+		userSvc interfaces.UserService,
+	) {
+		ctx.modelSvc = modelSvc
+		ctx.userSvc = userSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	return ctx
+}
+
+func newTokenController() *tokenController {
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListPostActionControllerDelegate(ControllerIdToken, modelSvc.GetBaseService(interfaces.ModelIdToken), TokenActions)
+	d := NewListPostActionControllerDelegate(ControllerIdToken, modelSvc.GetBaseService(interfaces.ModelIdToken), TokenActions)
+	ctx := newTokenContext()
+
+	return &tokenController{
+		ListActionControllerDelegate: *ctr,
+		d:                            *d,
+		ctx:                          ctx,
+	}
+}

core/controllers/token_v2.go

@@ -0,0 +1,35 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/user"
+	"github.com/gin-gonic/gin"
+)
+
+func PostToken(c *gin.Context) {
+	var t models.TokenV2
+	if err := c.ShouldBindJSON(&t); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	svc, err := user.GetUserServiceV2()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	u := GetUserFromContextV2(c)
+	t.SetCreated(u.Id)
+	t.SetUpdated(u.Id)
+	t.Token, err = svc.MakeToken(u)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	_, err = service.NewModelServiceV2[models.TokenV2]().InsertOne(t)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}

core/controllers/user.go

@@ -0,0 +1,244 @@
+package controllers
+
+import (
+	"encoding/json"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	delegate2 "github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"net/http"
+)
+
+var UserController *userController
+
+func getUserActions() []Action {
+	userCtx := newUserContext()
+	return []Action{
+		{
+			Method:      http.MethodPost,
+			Path:        "/:id/change-password",
+			HandlerFunc: userCtx.changePassword,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/me",
+			HandlerFunc: userCtx.getMe,
+		},
+		{
+			Method:      http.MethodPut,
+			Path:        "/me",
+			HandlerFunc: userCtx.putMe,
+		},
+	}
+}
+
+type userController struct {
+	ListActionControllerDelegate
+	d   ListActionControllerDelegate
+	ctx *userContext
+}
+
+func (ctr *userController) Post(c *gin.Context) {
+	var u models.User
+	if err := c.ShouldBindJSON(&u); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := ctr.ctx.userSvc.Create(&interfaces.UserCreateOptions{
+		Username: u.Username,
+		Password: u.Password,
+		Email:    u.Email,
+		Role:     u.Role,
+	}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctr *userController) PostList(c *gin.Context) {
+	// users
+	var users []models.User
+	if err := c.ShouldBindJSON(&users); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	for _, u := range users {
+		if err := ctr.ctx.userSvc.Create(&interfaces.UserCreateOptions{
+			Username: u.Username,
+			Password: u.Password,
+			Email:    u.Email,
+			Role:     u.Role,
+		}); err != nil {
+			trace.PrintError(err)
+		}
+	}
+
+	HandleSuccess(c)
+}
+
+func (ctr *userController) PutList(c *gin.Context) {
+	// payload
+	var payload entity.BatchRequestPayloadWithStringData
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// doc to update
+	var doc models.User
+	if err := json.Unmarshal([]byte(payload.Data), &doc); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// query
+	query := bson.M{
+		"_id": bson.M{
+			"$in": payload.Ids,
+		},
+	}
+
+	// update users
+	if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdUser).UpdateDoc(query, &doc, payload.Fields); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// update passwords
+	if utils.Contains(payload.Fields, "password") {
+		for _, id := range payload.Ids {
+			if err := ctr.ctx.userSvc.ChangePassword(id, doc.Password); err != nil {
+				trace.PrintError(err)
+			}
+		}
+	}
+
+	HandleSuccess(c)
+}
+
+type userContext struct {
+	modelSvc service.ModelService
+	userSvc  interfaces.UserService
+}
+
+func (ctx *userContext) changePassword(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	var payload map[string]string
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	password, ok := payload["password"]
+	if !ok {
+		HandleErrorBadRequest(c, errors.ErrorUserMissingRequiredFields)
+		return
+	}
+	if len(password) < 5 {
+		HandleErrorBadRequest(c, errors.ErrorUserInvalidPassword)
+		return
+	}
+	if err := ctx.userSvc.ChangePassword(id, password); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *userContext) getMe(c *gin.Context) {
+	u, err := ctx._getMe(c)
+	if err != nil {
+		HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized)
+		return
+	}
+	HandleSuccessWithData(c, u)
+}
+
+func (ctx *userContext) putMe(c *gin.Context) {
+	// current user
+	u, err := ctx._getMe(c)
+	if err != nil {
+		HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized)
+		return
+	}
+
+	// payload
+	doc, err := NewJsonBinder(ControllerIdUser).Bind(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if doc.GetId() != u.GetId() {
+		HandleErrorBadRequest(c, errors.ErrorHttpBadRequest)
+		return
+	}
+
+	// save to db
+	if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Save(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, doc)
+}
+
+func (ctx *userContext) _getMe(c *gin.Context) (u interfaces.User, err error) {
+	res, ok := c.Get(constants.UserContextKey)
+	if !ok {
+		return nil, trace.TraceError(errors.ErrorUserNotExistsInContext)
+	}
+	u, ok = res.(interfaces.User)
+	if !ok {
+		return nil, trace.TraceError(errors.ErrorUserInvalidType)
+	}
+	return u, nil
+}
+
+func newUserContext() *userContext {
+	// context
+	ctx := &userContext{}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(
+		modelSvc service.ModelService,
+		userSvc interfaces.UserService,
+	) {
+		ctx.modelSvc = modelSvc
+		ctx.userSvc = userSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	return ctx
+}
+
+func newUserController() *userController {
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListPostActionControllerDelegate(ControllerIdUser, modelSvc.GetBaseService(interfaces.ModelIdUser), getUserActions())
+	d := NewListPostActionControllerDelegate(ControllerIdUser, modelSvc.GetBaseService(interfaces.ModelIdUser), getUserActions())
+	ctx := newUserContext()
+
+	return &userController{
+		ListActionControllerDelegate: *ctr,
+		d:                            *d,
+		ctx:                          ctx,
+	}
+}

core/controllers/user_v2.go

@@ -0,0 +1,126 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func PostUser(c *gin.Context) {
+	var payload struct {
+		Username string `json:"username"`
+		Password string `json:"password"`
+		Role     string `json:"role"`
+		Email    string `json:"email"`
+	}
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	u := GetUserFromContextV2(c)
+	model := models.UserV2{
+		Username: payload.Username,
+		Password: utils.EncryptMd5(payload.Password),
+		Role:     payload.Role,
+		Email:    payload.Email,
+	}
+	model.SetCreated(u.Id)
+	model.SetUpdated(u.Id)
+	id, err := service.NewModelServiceV2[models.UserV2]().InsertOne(model)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	result, err := service.NewModelServiceV2[models.UserV2]().GetById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, result)
+
+}
+
+func PostUserChangePassword(c *gin.Context) {
+	// get id
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// get payload
+	var payload struct {
+		Password string `json:"password"`
+	}
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// get user
+	u := GetUserFromContextV2(c)
+	modelSvc := service.NewModelServiceV2[models.UserV2]()
+
+	// update password
+	user, err := modelSvc.GetById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	user.SetUpdated(u.Id)
+	user.Password = utils.EncryptMd5(payload.Password)
+	if err := modelSvc.ReplaceById(user.Id, *user); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// handle success
+	HandleSuccess(c)
+}
+
+func GetUserMe(c *gin.Context) {
+	u := GetUserFromContextV2(c)
+	_u, err := service.NewModelServiceV2[models.UserV2]().GetById(u.Id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, _u)
+}
+
+func PutUserById(c *gin.Context) {
+	// get payload
+	var user models.UserV2
+	if err := c.ShouldBindJSON(&user); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// get user
+	u := GetUserFromContextV2(c)
+
+	modelSvc := service.NewModelServiceV2[models.UserV2]()
+
+	// update user
+	userDb, err := modelSvc.GetById(u.Id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	user.Password = userDb.Password
+	user.SetUpdated(u.Id)
+	if user.Id.IsZero() {
+		user.Id = u.Id
+	}
+	if err := modelSvc.ReplaceById(u.Id, user); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// handle success
+	HandleSuccess(c)
+}

core/controllers/user_v2_test.go

@@ -0,0 +1,89 @@
+package controllers_test
+
+import (
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/middlewares"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/assert"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestPostUserChangePassword_Success(t *testing.T) {
+	SetupTestDB()
+	defer CleanupTestDB()
+
+	modelSvc := service.NewModelServiceV2[models.UserV2]()
+	u := models.UserV2{}
+	id, err := modelSvc.InsertOne(u)
+	assert.Nil(t, err)
+	u.SetId(id)
+
+	router := gin.Default()
+	router.Use(middlewares.AuthorizationMiddlewareV2())
+	router.POST("/users/:id/change-password", controllers.PostUserChangePassword)
+
+	password := "newPassword"
+	reqBody := strings.NewReader(`{"password":"` + password + `"}`)
+	req, _ := http.NewRequest(http.MethodPost, "/users/"+id.Hex()+"/change-password", reqBody)
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", TestToken)
+
+	w := httptest.NewRecorder()
+	router.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusOK, w.Code)
+}
+
+func TestGetUserMe_Success(t *testing.T) {
+	SetupTestDB()
+	defer CleanupTestDB()
+
+	modelSvc := service.NewModelServiceV2[models.UserV2]()
+	u := models.UserV2{}
+	id, err := modelSvc.InsertOne(u)
+	assert.Nil(t, err)
+	u.SetId(id)
+
+	router := gin.Default()
+	router.Use(middlewares.AuthorizationMiddlewareV2())
+	router.GET("/users/me", controllers.GetUserMe)
+
+	req, _ := http.NewRequest(http.MethodGet, "/users/me", nil)
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", TestToken)
+
+	w := httptest.NewRecorder()
+	router.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusOK, w.Code)
+}
+
+func TestPutUserById_Success(t *testing.T) {
+	SetupTestDB()
+	defer CleanupTestDB()
+
+	modelSvc := service.NewModelServiceV2[models.UserV2]()
+	u := models.UserV2{}
+	id, err := modelSvc.InsertOne(u)
+	assert.Nil(t, err)
+	u.SetId(id)
+
+	router := gin.Default()
+	router.Use(middlewares.AuthorizationMiddlewareV2())
+	router.PUT("/users/me", controllers.PutUserById)
+
+	reqBody := strings.NewReader(`{"id":"` + id.Hex() + `","username":"newUsername","email":"newEmail@test.com"}`)
+	req, _ := http.NewRequest(http.MethodPut, "/users/me", reqBody)
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", TestToken)
+
+	w := httptest.NewRecorder()
+	router.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusOK, w.Code)
+}

core/controllers/utils_context.go

@@ -0,0 +1,32 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/gin-gonic/gin"
+)
+
+func GetUserFromContext(c *gin.Context) (u interfaces.User) {
+	value, ok := c.Get(constants.UserContextKey)
+	if !ok {
+		return nil
+	}
+	u, ok = value.(interfaces.User)
+	if !ok {
+		return nil
+	}
+	return u
+}
+
+func GetUserFromContextV2(c *gin.Context) (u *models.UserV2) {
+	value, ok := c.Get(constants.UserContextKey)
+	if !ok {
+		return nil
+	}
+	u, ok = value.(*models.UserV2)
+	if !ok {
+		return nil
+	}
+	return u
+}

core/controllers/utils_filter.go

@@ -0,0 +1,123 @@
+package controllers
+
+import (
+	"encoding/json"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"reflect"
+	"strings"
+)
+
+// GetFilter Get entity.Filter from gin.Context
+func GetFilter(c *gin.Context) (f *entity.Filter, err error) {
+	// bind
+	condStr := c.Query(constants.FilterQueryFieldConditions)
+	var conditions []*entity.Condition
+	if err := json.Unmarshal([]byte(condStr), &conditions); err != nil {
+		return nil, err
+	}
+
+	// attempt to convert object id
+	for i, cond := range conditions {
+		v := reflect.ValueOf(cond.Value)
+		switch v.Kind() {
+		case reflect.String:
+			item := cond.Value.(string)
+			id, err := primitive.ObjectIDFromHex(item)
+			if err == nil {
+				conditions[i].Value = id
+			} else {
+				conditions[i].Value = item
+			}
+		case reflect.Slice, reflect.Array:
+			var items []interface{}
+			for i := 0; i < v.Len(); i++ {
+				vItem := v.Index(i)
+				item := vItem.Interface()
+
+				// string
+				stringItem, ok := item.(string)
+				if ok {
+					id, err := primitive.ObjectIDFromHex(stringItem)
+					if err == nil {
+						items = append(items, id)
+					} else {
+						items = append(items, stringItem)
+					}
+					continue
+				}
+
+				// default
+				items = append(items, item)
+			}
+			conditions[i].Value = items
+		}
+	}
+
+	return &entity.Filter{
+		IsOr:       false,
+		Conditions: conditions,
+	}, nil
+}
+
+// GetFilterQuery Get bson.M from gin.Context
+func GetFilterQuery(c *gin.Context) (q bson.M, err error) {
+	f, err := GetFilter(c)
+	if err != nil {
+		return nil, err
+	}
+
+	if f == nil {
+		return nil, nil
+	}
+
+	// TODO: implement logic OR
+
+	return utils.FilterToQuery(f)
+}
+
+func MustGetFilterQuery(c *gin.Context) (q bson.M) {
+	q, err := GetFilterQuery(c)
+	if err != nil {
+		return nil
+	}
+	return q
+}
+
+// GetFilterAll Get all from gin.Context
+func GetFilterAll(c *gin.Context) (res bool, err error) {
+	resStr := c.Query(constants.FilterQueryFieldAll)
+	switch strings.ToUpper(resStr) {
+	case "1":
+		return true, nil
+	case "0":
+		return false, nil
+	case "Y":
+		return true, nil
+	case "N":
+		return false, nil
+	case "T":
+		return true, nil
+	case "F":
+		return false, nil
+	case "TRUE":
+		return true, nil
+	case "FALSE":
+		return false, nil
+	default:
+		return false, errors.ErrorFilterInvalidOperation
+	}
+}
+
+func MustGetFilterAll(c *gin.Context) (res bool) {
+	res, err := GetFilterAll(c)
+	if err != nil {
+		return false
+	}
+	return res
+}

core/controllers/utils_http.go

@@ -0,0 +1,72 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/go-trace"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+func handleError(statusCode int, c *gin.Context, err error, print bool) {
+	if print {
+		trace.PrintError(err)
+	}
+	c.AbortWithStatusJSON(statusCode, entity.Response{
+		Status:  constants.HttpResponseStatusOk,
+		Message: constants.HttpResponseMessageError,
+		Error:   err.Error(),
+	})
+}
+
+func HandleError(statusCode int, c *gin.Context, err error) {
+	handleError(statusCode, c, err, true)
+}
+
+func HandleErrorNoPrint(statusCode int, c *gin.Context, err error) {
+	handleError(statusCode, c, err, false)
+}
+
+func HandleErrorBadRequest(c *gin.Context, err error) {
+	HandleError(http.StatusBadRequest, c, err)
+}
+
+func HandleErrorUnauthorized(c *gin.Context, err error) {
+	HandleError(http.StatusUnauthorized, c, err)
+}
+
+func HandleErrorNotFound(c *gin.Context, err error) {
+	HandleError(http.StatusNotFound, c, err)
+}
+
+func HandleErrorNotFoundNoPrint(c *gin.Context, err error) {
+	HandleErrorNoPrint(http.StatusNotFound, c, err)
+}
+
+func HandleErrorInternalServerError(c *gin.Context, err error) {
+	HandleError(http.StatusInternalServerError, c, err)
+}
+
+func HandleSuccess(c *gin.Context) {
+	c.AbortWithStatusJSON(http.StatusOK, entity.Response{
+		Status:  constants.HttpResponseStatusOk,
+		Message: constants.HttpResponseMessageSuccess,
+	})
+}
+
+func HandleSuccessWithData(c *gin.Context, data interface{}) {
+	c.AbortWithStatusJSON(http.StatusOK, entity.Response{
+		Status:  constants.HttpResponseStatusOk,
+		Message: constants.HttpResponseMessageSuccess,
+		Data:    data,
+	})
+}
+
+func HandleSuccessWithListData(c *gin.Context, data interface{}, total int) {
+	c.AbortWithStatusJSON(http.StatusOK, entity.ListResponse{
+		Status:  constants.HttpResponseStatusOk,
+		Message: constants.HttpResponseMessageSuccess,
+		Data:    data,
+		Total:   total,
+	})
+}

core/controllers/utils_pagination.go

@@ -0,0 +1,30 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/gin-gonic/gin"
+)
+
+func GetDefaultPagination() (p *entity.Pagination) {
+	return &entity.Pagination{
+		Page: constants.PaginationDefaultPage,
+		Size: constants.PaginationDefaultSize,
+	}
+}
+
+func GetPagination(c *gin.Context) (p *entity.Pagination, err error) {
+	var _p entity.Pagination
+	if err := c.ShouldBindQuery(&_p); err != nil {
+		return GetDefaultPagination(), err
+	}
+	return &_p, nil
+}
+
+func MustGetPagination(c *gin.Context) (p *entity.Pagination) {
+	p, err := GetPagination(c)
+	if err != nil || p == nil {
+		return GetDefaultPagination()
+	}
+	return p
+}

core/controllers/utils_sort.go

@@ -0,0 +1,58 @@
+package controllers
+
+import (
+	"encoding/json"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+// GetSorts Get entity.Sort from gin.Context
+func GetSorts(c *gin.Context) (sorts []entity.Sort, err error) {
+	// bind
+	sortStr := c.Query(constants.SortQueryField)
+	if err := json.Unmarshal([]byte(sortStr), &sorts); err != nil {
+		return nil, err
+	}
+	return sorts, nil
+}
+
+// GetSortsOption Get entity.Sort from gin.Context
+func GetSortsOption(c *gin.Context) (sort bson.D, err error) {
+	sorts, err := GetSorts(c)
+	if err != nil {
+		return nil, err
+	}
+
+	if sorts == nil || len(sorts) == 0 {
+		return bson.D{{"_id", -1}}, nil
+	}
+
+	return SortsToOption(sorts)
+}
+
+func MustGetSortOption(c *gin.Context) (sort bson.D) {
+	sort, err := GetSortsOption(c)
+	if err != nil {
+		return nil
+	}
+	return sort
+}
+
+// SortsToOption Translate entity.Sort to bson.D
+func SortsToOption(sorts []entity.Sort) (sort bson.D, err error) {
+	sort = bson.D{}
+	for _, s := range sorts {
+		switch s.Direction {
+		case constants.ASCENDING:
+			sort = append(sort, bson.E{Key: s.Key, Value: 1})
+		case constants.DESCENDING:
+			sort = append(sort, bson.E{Key: s.Key, Value: -1})
+		}
+	}
+	if len(sort) == 0 {
+		sort = bson.D{{"_id", -1}}
+	}
+	return sort, nil
+}

core/controllers/version.go

@@ -0,0 +1,23 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/config"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+func GetVersion(c *gin.Context) {
+	HandleSuccessWithData(c, config.GetVersion())
+}
+
+func getVersionActions() []Action {
+	return []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "",
+			HandlerFunc: GetVersion,
+		},
+	}
+}
+
+var VersionController ActionController