refactor: code cleanup

Jenkinsfile

@@ -1,56 +0,0 @@
-pipeline {
-    agent {
-        node {
-            label 'crawlab'
-        }
-    }
-
-    stages {
-        stage('Setup') {
-            steps {
-                echo "Running Setup..."
-                script {
-                    if (env.GIT_BRANCH == 'develop') {
-                        env.TAG = 'develop'
-                        env.DOCKERFILE = 'Dockerfile.local'
-                    } else if (env.GIT_BRANCH == 'master') {
-                        env.TAG = 'master'
-                        env.DOCKERFILE = 'Dockerfile.local'
-                    } 
-                }
-            }
-        }
-        stage('Build') {
-            steps {
-                echo "Building..."
-                sh """
-                docker build -t tikazyq/crawlab:${ENV:TAG} -f ${ENV:DOCKERFILE} .
-                """
-            }
-        }
-        stage('Test') {
-            steps {
-                echo 'Testing..'
-            }
-        }
-        stage('Deploy') {
-            steps {
-                echo 'Deploying....'
-                sh """
-                # 重启docker compose
-                cd ./jenkins/${ENV:GIT_BRANCH}
-                docker-compose down | true
-                docker-compose up -d | true
-                """
-            }
-        }
-        stage('Cleanup') {
-            steps {
-                echo 'Cleanup...'
-                sh """
-                docker rmi -f `docker images | grep '<none>' | grep -v IMAGE | awk '{ print \$3 }' | xargs`
-                """
-            }
-        }
-    }
-}

core/apps/api.go

@@ -1,126 +0,0 @@
-package apps
-
-import (
-	"context"
-	"errors"
-	"github.com/apex/log"
-	"github.com/crawlab-team/crawlab/core/controllers"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/crawlab-team/crawlab/core/middlewares"
-	"github.com/crawlab-team/crawlab/core/routes"
-	"github.com/gin-gonic/gin"
-	"github.com/spf13/viper"
-	"net"
-	"net/http"
-	"time"
-)
-
-func init() {
-	// set gin mode
-	if viper.GetString("gin.mode") == "" {
-		gin.SetMode(gin.ReleaseMode)
-	} else {
-		gin.SetMode(viper.GetString("gin.mode"))
-	}
-}
-
-type Api struct {
-	// dependencies
-	interfaces.WithConfigPath
-
-	// internals
-	app   *gin.Engine
-	ln    net.Listener
-	srv   *http.Server
-	ready bool
-}
-
-func (app *Api) Init() {
-	// initialize controllers
-	_ = initModule("controllers", controllers.InitControllers)
-
-	// initialize middlewares
-	_ = app.initModuleWithApp("middlewares", middlewares.InitMiddlewares)
-
-	// initialize routes
-	_ = app.initModuleWithApp("routes", routes.InitRoutes)
-}
-
-func (app *Api) Start() {
-	// address
-	host := viper.GetString("server.host")
-	port := viper.GetString("server.port")
-	address := net.JoinHostPort(host, port)
-
-	// http server
-	app.srv = &http.Server{
-		Handler: app.app,
-		Addr:    address,
-	}
-
-	// listen
-	var err error
-	app.ln, err = net.Listen("tcp", address)
-	if err != nil {
-		panic(err)
-	}
-	app.ready = true
-
-	// serve
-	if err := http.Serve(app.ln, app.app); err != nil {
-		if !errors.Is(err, http.ErrServerClosed) {
-			log.Error("run server error:" + err.Error())
-		} else {
-			log.Info("server graceful down")
-		}
-	}
-}
-
-func (app *Api) Wait() {
-	DefaultWait()
-}
-
-func (app *Api) Stop() {
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	if err := app.srv.Shutdown(ctx); err != nil {
-		log.Error("run server error:" + err.Error())
-	}
-}
-
-func (app *Api) GetGinEngine() *gin.Engine {
-	return app.app
-}
-
-func (app *Api) GetHttpServer() *http.Server {
-	return app.srv
-}
-
-func (app *Api) Ready() (ok bool) {
-	return app.ready
-}
-
-func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) {
-	return initModule(name, func() error {
-		return fn(app.app)
-	})
-}
-
-func NewApi() *Api {
-	api := &Api{
-		app: gin.New(),
-	}
-	api.Init()
-	return api
-}
-
-var api *Api
-
-func GetApi() *Api {
-	if api != nil {
-		return api
-	}
-	api = NewApi()
-	return api
-}

core/cmd/server.go

@@ -16,7 +16,6 @@ var serverCmd = &cobra.Command{
 	Long:    `Start Crawlab node server that can serve as API, task scheduler, task runner, etc.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		// app
-		//svr := apps.GetServer(opts...)
 		svr := apps.GetServerV2()
 
 		// start

core/config/default_config.go

@@ -1,9 +1,8 @@
 package config
 
 var DefaultConfigYaml = `
-info:
-  version: v0.6.3
-  edition: global.edition.community
+version: v0.6.3
+edition: global.edition.community
 mongo:
   host: localhost
   port: 27017

core/controllers/data_collection.go

@@ -1,103 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/container"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/crawlab-team/crawlab/db/mongo"
-	"github.com/gin-gonic/gin"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	mongo2 "go.mongodb.org/mongo-driver/mongo"
-	"net/http"
-)
-
-var DataCollectionController *dataCollectionController
-
-func getDataCollectionActions() []Action {
-	ctx := newDataCollectionContext()
-	return []Action{
-		{
-			Method:      http.MethodPost,
-			Path:        "/:id/indexes",
-			HandlerFunc: ctx.postIndexes,
-		},
-	}
-}
-
-type dataCollectionController struct {
-	ListActionControllerDelegate
-	d   ListActionControllerDelegate
-	ctx *dataCollectionContext
-}
-
-type dataCollectionContext struct {
-	modelSvc  service.ModelService
-	resultSvc interfaces.ResultService
-}
-
-func (ctx *dataCollectionContext) postIndexes(c *gin.Context) {
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	dc, err := ctx.modelSvc.GetDataCollectionById(id)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	for _, f := range dc.Fields {
-		if err := mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{
-			Keys: f.Key,
-		}); err != nil {
-			HandleErrorInternalServerError(c, err)
-			return
-		}
-	}
-
-	HandleSuccess(c)
-}
-
-var _dataCollectionCtx *dataCollectionContext
-
-func newDataCollectionContext() *dataCollectionContext {
-	if _dataCollectionCtx != nil {
-		return _dataCollectionCtx
-	}
-
-	// context
-	ctx := &dataCollectionContext{}
-
-	// dependency injection
-	if err := container.GetContainer().Invoke(func(
-		modelSvc service.ModelService,
-	) {
-		ctx.modelSvc = modelSvc
-	}); err != nil {
-		panic(err)
-	}
-
-	_dataCollectionCtx = ctx
-
-	return ctx
-}
-
-func newDataCollectionController() *dataCollectionController {
-	actions := getDataCollectionActions()
-	modelSvc, err := service.GetService()
-	if err != nil {
-		panic(err)
-	}
-
-	ctr := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions)
-	d := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions)
-	ctx := newDataCollectionContext()
-
-	return &dataCollectionController{
-		ListActionControllerDelegate: *ctr,
-		d:                            *d,
-		ctx:                          ctx,
-	}
-}

core/controllers/data_source.go

@@ -1,148 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/ds"
-	"github.com/crawlab-team/crawlab/core/errors"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	interfaces2 "github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/crawlab-team/crawlab/core/models/delegate"
-	"github.com/crawlab-team/crawlab/core/models/models"
-	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/crawlab-team/crawlab/core/utils"
-	"github.com/crawlab-team/crawlab/db/mongo"
-	"github.com/crawlab-team/crawlab/trace"
-	"github.com/gin-gonic/gin"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	mongo2 "go.mongodb.org/mongo-driver/mongo"
-	"net/http"
-)
-
-var DataSourceController *dataSourceController
-
-func getDataSourceActions() []Action {
-	ctx := newDataSourceContext()
-	return []Action{
-		{
-			Path:        "/:id/change-password",
-			Method:      http.MethodPost,
-			HandlerFunc: ctx.changePassword,
-		},
-	}
-}
-
-type dataSourceController struct {
-	ListActionControllerDelegate
-	d   ListActionControllerDelegate
-	ctx *dataSourceContext
-}
-
-func (ctr *dataSourceController) Post(c *gin.Context) {
-	// data source
-	var _ds models.DataSource
-	if err := c.ShouldBindJSON(&_ds); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// add data source to db
-	if err := mongo.RunTransaction(func(ctx mongo2.SessionContext) error {
-		if err := delegate.NewModelDelegate(&_ds).Add(); err != nil {
-			return trace.TraceError(err)
-		}
-		pwd, err := utils.EncryptAES(_ds.Password)
-		if err != nil {
-			return trace.TraceError(err)
-		}
-		p := models.Password{Id: _ds.Id, Password: pwd}
-		if err := delegate.NewModelDelegate(&p).Add(); err != nil {
-			return trace.TraceError(err)
-		}
-		return nil
-	}); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// check data source status
-	go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }()
-
-	HandleSuccess(c)
-}
-
-func (ctr *dataSourceController) Put(c *gin.Context) {
-	// data source
-	var _ds models.DataSource
-	if err := c.ShouldBindJSON(&_ds); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	if err := delegate.NewModelDelegate(&_ds).Save(); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// check data source status
-	go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }()
-}
-
-type dataSourceContext struct {
-	dsSvc interfaces.DataSourceService
-}
-
-var _dataSourceCtx *dataSourceContext
-
-func newDataSourceContext() *dataSourceContext {
-	if _dataSourceCtx != nil {
-		return _dataSourceCtx
-	}
-	dsSvc, err := ds.GetDataSourceService()
-	if err != nil {
-		panic(err)
-	}
-	_dataSourceCtx = &dataSourceContext{
-		dsSvc: dsSvc,
-	}
-	return _dataSourceCtx
-}
-
-func (ctx *dataSourceContext) changePassword(c *gin.Context) {
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	var payload map[string]string
-	if err := c.ShouldBindJSON(&payload); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	password, ok := payload["password"]
-	if !ok {
-		HandleErrorBadRequest(c, errors.ErrorDataSourceMissingRequiredFields)
-		return
-	}
-	if err := ctx.dsSvc.ChangePassword(id, password); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-func newDataSourceController() *dataSourceController {
-	actions := getDataSourceActions()
-	modelSvc, err := service.GetService()
-	if err != nil {
-		panic(err)
-	}
-
-	ctr := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions)
-	d := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions)
-	ctx := newDataSourceContext()
-
-	return &dataSourceController{
-		ListActionControllerDelegate: *ctr,
-		d:                            *d,
-		ctx:                          ctx,
-	}
-}

core/controllers/demo.go

@@ -1,73 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/utils"
-	"github.com/crawlab-team/crawlab/trace"
-	"github.com/gin-gonic/gin"
-	"net/http"
-)
-
-func getDemoActions() []Action {
-	ctx := newDemoContext()
-	return []Action{
-		{
-			Method:      http.MethodGet,
-			Path:        "/import",
-			HandlerFunc: ctx.import_,
-		},
-		{
-			Method:      http.MethodGet,
-			Path:        "/reimport",
-			HandlerFunc: ctx.reimport,
-		},
-		{
-			Method:      http.MethodGet,
-			Path:        "/cleanup",
-			HandlerFunc: ctx.cleanup,
-		},
-	}
-}
-
-type demoContext struct {
-}
-
-func (ctx *demoContext) import_(c *gin.Context) {
-	if err := utils.ImportDemo(); err != nil {
-		trace.PrintError(err)
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-func (ctx *demoContext) reimport(c *gin.Context) {
-	if err := utils.ReimportDemo(); err != nil {
-		trace.PrintError(err)
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-func (ctx *demoContext) cleanup(c *gin.Context) {
-	if err := utils.ReimportDemo(); err != nil {
-		trace.PrintError(err)
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-var _demoCtx *demoContext
-
-func newDemoContext() *demoContext {
-	if _demoCtx != nil {
-		return _demoCtx
-	}
-
-	_demoCtx = &demoContext{}
-
-	return _demoCtx
-}
-
-var DemoController ActionController

core/controllers/export.go

@@ -1,127 +0,0 @@
-package controllers
-
-import (
-	"errors"
-	"fmt"
-	"github.com/crawlab-team/crawlab/core/constants"
-	"github.com/crawlab-team/crawlab/core/export"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/gin-gonic/gin"
-	"net/http"
-)
-
-var ExportController ActionController
-
-func getExportActions() []Action {
-	ctx := newExportContext()
-	return []Action{
-		{
-			Method:      http.MethodPost,
-			Path:        "/:type",
-			HandlerFunc: ctx.postExport,
-		},
-		{
-			Method:      http.MethodGet,
-			Path:        "/:type/:id",
-			HandlerFunc: ctx.getExport,
-		},
-		{
-			Method:      http.MethodGet,
-			Path:        "/:type/:id/download",
-			HandlerFunc: ctx.getExportDownload,
-		},
-	}
-}
-
-type exportContext struct {
-	csvSvc  interfaces.ExportService
-	jsonSvc interfaces.ExportService
-}
-
-func (ctx *exportContext) postExport(c *gin.Context) {
-	exportType := c.Param("type")
-	exportTarget := c.Query("target")
-	exportFilter, _ := GetFilter(c)
-
-	var exportId string
-	var err error
-	switch exportType {
-	case constants.ExportTypeCsv:
-		exportId, err = ctx.csvSvc.Export(exportType, exportTarget, exportFilter)
-	case constants.ExportTypeJson:
-		exportId, err = ctx.jsonSvc.Export(exportType, exportTarget, exportFilter)
-	default:
-		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
-		return
-	}
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	HandleSuccessWithData(c, exportId)
-}
-
-func (ctx *exportContext) getExport(c *gin.Context) {
-	exportType := c.Param("type")
-	exportId := c.Param("id")
-
-	var exp interfaces.Export
-	var err error
-	switch exportType {
-	case constants.ExportTypeCsv:
-		exp, err = ctx.csvSvc.GetExport(exportId)
-	case constants.ExportTypeJson:
-		exp, err = ctx.jsonSvc.GetExport(exportId)
-	default:
-		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
-	}
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	HandleSuccessWithData(c, exp)
-}
-
-func (ctx *exportContext) getExportDownload(c *gin.Context) {
-	exportType := c.Param("type")
-	exportId := c.Param("id")
-
-	var exp interfaces.Export
-	var err error
-	switch exportType {
-	case constants.ExportTypeCsv:
-		exp, err = ctx.csvSvc.GetExport(exportId)
-	case constants.ExportTypeJson:
-		exp, err = ctx.jsonSvc.GetExport(exportId)
-	default:
-		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
-	}
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	switch exportType {
-	case constants.ExportTypeCsv:
-		c.Header("Content-Type", "text/csv")
-	case constants.ExportTypeJson:
-		c.Header("Content-Type", "text/plain")
-	default:
-		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
-	}
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", exp.GetDownloadPath()))
-	c.File(exp.GetDownloadPath())
-}
-
-func newExportContext() *exportContext {
-	return &exportContext{
-		csvSvc:  export.GetCsvService(),
-		jsonSvc: export.GetJsonService(),
-	}
-}

core/controllers/filer.go

@@ -1,130 +0,0 @@
-package controllers
-
-import (
-	"bufio"
-	"fmt"
-	"github.com/crawlab-team/crawlab/core/errors"
-	"github.com/gin-gonic/gin"
-	"github.com/imroc/req"
-	"github.com/spf13/viper"
-	"net/http"
-	"strings"
-)
-
-var FilerController ActionController
-
-func getFilerActions() []Action {
-	filerCtx := newFilerContext()
-	return []Action{
-		{
-			Method:      http.MethodGet,
-			Path:        "*path",
-			HandlerFunc: filerCtx.do,
-		},
-		{
-			Method:      http.MethodPost,
-			Path:        "*path",
-			HandlerFunc: filerCtx.do,
-		},
-		{
-			Method:      http.MethodPut,
-			Path:        "*path",
-			HandlerFunc: filerCtx.do,
-		},
-		{
-			Method:      http.MethodDelete,
-			Path:        "*path",
-			HandlerFunc: filerCtx.do,
-		},
-	}
-}
-
-type filerContext struct {
-	endpoint string
-}
-
-func (ctx *filerContext) do(c *gin.Context) {
-	// request path
-	requestPath := strings.Replace(c.Request.URL.Path, "/filer", "", 1)
-
-	// request url
-	requestUrl := fmt.Sprintf("%s%s", ctx.endpoint, requestPath)
-	if c.Request.URL.RawQuery != "" {
-		requestUrl += "?" + c.Request.URL.RawQuery
-	}
-
-	// request body
-	bufR := bufio.NewScanner(c.Request.Body)
-	requestBody := req.BodyJSON(bufR.Bytes())
-
-	// request file uploads
-	var requestFileUploads []req.FileUpload
-	form, err := c.MultipartForm()
-	if err == nil {
-		for k, v := range form.File {
-			for _, fh := range v {
-				f, err := fh.Open()
-				if err != nil {
-					HandleErrorInternalServerError(c, err)
-					return
-				}
-				requestFileUploads = append(requestFileUploads, req.FileUpload{
-					FileName:  fh.Filename,
-					FieldName: k,
-					File:      f,
-				})
-			}
-		}
-	}
-
-	// request header
-	requestHeader := req.Header{}
-	for k, v := range c.Request.Header {
-		if len(v) > 0 {
-			requestHeader[k] = v[0]
-		}
-	}
-
-	// perform request
-	res, err := req.Do(c.Request.Method, requestUrl, requestHeader, requestBody, requestFileUploads)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// status code check
-	statusCode := res.Response().StatusCode
-	if statusCode == http.StatusNotFound {
-		HandleErrorNotFoundNoPrint(c, errors.ErrorControllerFilerNotFound)
-		return
-	}
-
-	// response
-	for k, v := range res.Response().Header {
-		if len(v) > 0 {
-			c.Header(k, v[0])
-		}
-	}
-	_, _ = c.Writer.Write(res.Bytes())
-	c.AbortWithStatus(statusCode)
-}
-
-var _filerCtx *filerContext
-
-func newFilerContext() *filerContext {
-	if _filerCtx != nil {
-		return _filerCtx
-	}
-
-	ctx := &filerContext{
-		endpoint: "http://localhost:8888",
-	}
-
-	if viper.GetString("fs.filer.proxy") != "" {
-		ctx.endpoint = viper.GetString("fs.filer.proxy")
-	}
-
-	_filerCtx = ctx
-
-	return ctx
-}

core/controllers/filter.go

@@ -1,100 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/entity"
-	"github.com/crawlab-team/crawlab/db/mongo"
-	"github.com/gin-gonic/gin"
-	"go.mongodb.org/mongo-driver/bson"
-	mongo2 "go.mongodb.org/mongo-driver/mongo"
-	"net/http"
-)
-
-var FilterController ActionController
-
-func getFilterActions() []Action {
-	ctx := newFilterContext()
-	return []Action{
-		{
-			Method:      http.MethodGet,
-			Path:        "/:col",
-			HandlerFunc: ctx.getColFieldOptions,
-		},
-		{
-			Method:      http.MethodGet,
-			Path:        "/:col/:value",
-			HandlerFunc: ctx.getColFieldOptions,
-		},
-		{
-			Method:      http.MethodGet,
-			Path:        "/:col/:value/:label",
-			HandlerFunc: ctx.getColFieldOptions,
-		},
-	}
-}
-
-type filterContext struct {
-}
-
-func (ctx *filterContext) getColFieldOptions(c *gin.Context) {
-	colName := c.Param("col")
-	value := c.Param("value")
-	if value == "" {
-		value = "_id"
-	}
-	label := c.Param("label")
-	if label == "" {
-		label = "name"
-	}
-	query := MustGetFilterQuery(c)
-	pipelines := mongo2.Pipeline{}
-	if query != nil {
-		pipelines = append(pipelines, bson.D{{"$match", query}})
-	}
-	pipelines = append(
-		pipelines,
-		bson.D{
-			{
-				"$group",
-				bson.M{
-					"_id": bson.M{
-						"value": "$" + value,
-						"label": "$" + label,
-					},
-				},
-			},
-		},
-	)
-	pipelines = append(
-		pipelines,
-		bson.D{
-			{
-				"$project",
-				bson.M{
-					"value": "$_id.value",
-					"label": bson.M{"$toString": "$_id.label"},
-				},
-			},
-		},
-	)
-	pipelines = append(
-		pipelines,
-		bson.D{
-			{
-				"$sort",
-				bson.D{
-					{"value", 1},
-				},
-			},
-		},
-	)
-	var options []entity.FilterSelectOption
-	if err := mongo.GetMongoCol(colName).Aggregate(pipelines, nil).All(&options); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccessWithData(c, options)
-}
-
-func newFilterContext() *filterContext {
-	return &filterContext{}
-}

core/controllers/git.go

@@ -1,3 +0,0 @@
-package controllers
-
-var GitController ListController

core/controllers/init.go

@@ -1,42 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/crawlab-team/crawlab/core/models/service"
-)
-
-func InitControllers() (err error) {
-	modelSvc, err := service.GetService()
-	if err != nil {
-		return err
-	}
-
-	NodeController = newNodeController()
-	ProjectController = newProjectController()
-	SpiderController = newSpiderController()
-	TaskController = newTaskController()
-	UserController = newUserController()
-	TagController = NewListControllerDelegate(ControllerIdTag, modelSvc.GetBaseService(interfaces.ModelIdTag))
-	SettingController = newSettingController()
-	LoginController = NewActionControllerDelegate(ControllerIdLogin, getLoginActions())
-	DataCollectionController = newDataCollectionController()
-	ResultController = NewActionControllerDelegate(ControllerIdResult, getResultActions())
-	ScheduleController = newScheduleController()
-	StatsController = NewActionControllerDelegate(ControllerIdStats, getStatsActions())
-	TokenController = newTokenController()
-	FilerController = NewActionControllerDelegate(ControllerIdFiler, getFilerActions())
-	GitController = NewListControllerDelegate(ControllerIdGit, modelSvc.GetBaseService(interfaces.ModelIdGit))
-	VersionController = NewActionControllerDelegate(ControllerIdVersion, getVersionActions())
-	SystemInfoController = NewActionControllerDelegate(ControllerIdSystemInfo, getSystemInfoActions())
-	DemoController = NewActionControllerDelegate(ControllerIdDemo, getDemoActions())
-	RoleController = NewListControllerDelegate(ControllerIdRole, modelSvc.GetBaseService(interfaces.ModelIdRole))
-	PermissionController = NewListControllerDelegate(ControllerIdPermission, modelSvc.GetBaseService(interfaces.ModelIdPermission))
-	ExportController = NewActionControllerDelegate(ControllerIdExport, getExportActions())
-	NotificationController = NewActionControllerDelegate(ControllerIdNotification, getNotificationActions())
-	FilterController = NewActionControllerDelegate(ControllerIdFilter, getFilterActions())
-	SyncController = NewActionControllerDelegate(ControllerIdSync, getSyncActions())
-	DataSourceController = newDataSourceController()
-	EnvironmentController = newEnvironmentController()
-
-	return nil
-}

core/controllers/login.go

@@ -1,64 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/constants"
-	"github.com/crawlab-team/crawlab/core/container"
-	"github.com/crawlab-team/crawlab/core/errors"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/crawlab-team/crawlab/core/models/models"
-	"github.com/gin-gonic/gin"
-	"net/http"
-)
-
-var LoginController ActionController
-
-func getLoginActions() []Action {
-	loginCtx := newLoginContext()
-	return []Action{
-		{Method: http.MethodPost, Path: "/login", HandlerFunc: loginCtx.login},
-		{Method: http.MethodPost, Path: "/logout", HandlerFunc: loginCtx.logout},
-	}
-}
-
-type loginContext struct {
-	userSvc interfaces.UserService
-}
-
-func (ctx *loginContext) login(c *gin.Context) {
-	var u models.User
-	if err := c.ShouldBindJSON(&u); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	token, loggedInUser, err := ctx.userSvc.Login(&interfaces.UserLoginOptions{
-		Username: u.Username,
-		Password: u.Password,
-	})
-	if err != nil {
-		HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized)
-		return
-	}
-	c.Set(constants.UserContextKey, loggedInUser)
-	HandleSuccessWithData(c, token)
-}
-
-func (ctx *loginContext) logout(c *gin.Context) {
-	c.Set(constants.UserContextKey, nil)
-	HandleSuccess(c)
-}
-
-func newLoginContext() *loginContext {
-	// context
-	ctx := &loginContext{}
-
-	// dependency injection
-	if err := container.GetContainer().Invoke(func(
-		userSvc interfaces.UserService,
-	) {
-		ctx.userSvc = userSvc
-	}); err != nil {
-		panic(err)
-	}
-
-	return ctx
-}

core/controllers/node.go

@@ -1,94 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/constants"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/crawlab-team/crawlab/core/models/delegate"
-	"github.com/crawlab-team/crawlab/core/models/models"
-	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/crawlab-team/crawlab/trace"
-	"github.com/gin-gonic/gin"
-	"github.com/google/uuid"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-var NodeController *nodeController
-
-type nodeController struct {
-	ListControllerDelegate
-}
-
-func (ctr *nodeController) Post(c *gin.Context) {
-	var n models.Node
-	if err := c.ShouldBindJSON(&n); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	if err := ctr._post(c, &n); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	HandleSuccess(c)
-}
-
-func (ctr *nodeController) PostList(c *gin.Context) {
-	// bind
-	var docs []models.Node
-	if err := c.ShouldBindJSON(&docs); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// success ids
-	var ids []primitive.ObjectID
-
-	// iterate nodes
-	for _, n := range docs {
-		if err := ctr._post(c, &n); err != nil {
-			trace.PrintError(err)
-			continue
-		}
-		ids = append(ids, n.Id)
-	}
-
-	// success
-	HandleSuccessWithData(c, docs)
-}
-
-func (ctr *nodeController) _post(c *gin.Context, n *models.Node) (err error) {
-	// set default key
-	if n.Key == "" {
-		id, err := uuid.NewUUID()
-		if err != nil {
-			return trace.TraceError(err)
-		}
-		n.Key = id.String()
-	}
-
-	// set default status
-	if n.Status == "" {
-		n.Status = constants.NodeStatusUnregistered
-	}
-
-	// add
-	if err := delegate.NewModelDelegate(n, GetUserFromContext(c)).Add(); err != nil {
-		return trace.TraceError(err)
-	}
-
-	return nil
-}
-
-func newNodeController() *nodeController {
-	modelSvc, err := service.GetService()
-	if err != nil {
-		panic(err)
-	}
-
-	ctr := NewListControllerDelegate(ControllerIdNode, modelSvc.GetBaseService(interfaces.ModelIdNode))
-
-	return &nodeController{
-		ListControllerDelegate: *ctr,
-	}
-}

core/controllers/notification.go

@@ -1,158 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/notification"
-	"github.com/gin-gonic/gin"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"net/http"
-)
-
-var NotificationController ActionController
-
-func getNotificationActions() []Action {
-	ctx := newNotificationContext()
-	return []Action{
-		{
-			Method:      http.MethodGet,
-			Path:        "/settings",
-			HandlerFunc: ctx.GetSettingList,
-		},
-		{
-			Method:      http.MethodGet,
-			Path:        "/settings/:id",
-			HandlerFunc: ctx.GetSetting,
-		},
-		{
-			Method:      http.MethodPost,
-			Path:        "/settings",
-			HandlerFunc: ctx.PostSetting,
-		},
-		{
-			Method:      http.MethodPut,
-			Path:        "/settings/:id",
-			HandlerFunc: ctx.PutSetting,
-		},
-		{
-			Method:      http.MethodDelete,
-			Path:        "/settings/:id",
-			HandlerFunc: ctx.DeleteSetting,
-		},
-		{
-			Method:      http.MethodPost,
-			Path:        "/settings/:id/enable",
-			HandlerFunc: ctx.EnableSetting,
-		},
-		{
-			Method:      http.MethodPost,
-			Path:        "/settings/:id/disable",
-			HandlerFunc: ctx.DisableSetting,
-		},
-	}
-}
-
-type notificationContext struct {
-	svc *notification.Service
-}
-
-func (ctx *notificationContext) GetSettingList(c *gin.Context) {
-	query := MustGetFilterQuery(c)
-	pagination := MustGetPagination(c)
-	sort := MustGetSortOption(c)
-	res, total, err := ctx.svc.GetSettingList(query, pagination, sort)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccessWithListData(c, res, total)
-}
-
-func (ctx *notificationContext) GetSetting(c *gin.Context) {
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	res, err := ctx.svc.GetSetting(id)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccessWithData(c, res)
-}
-
-func (ctx *notificationContext) PostSetting(c *gin.Context) {
-	var s notification.NotificationSetting
-	if err := c.ShouldBindJSON(&s); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	if err := ctx.svc.PosSetting(&s); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-func (ctx *notificationContext) PutSetting(c *gin.Context) {
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	var s notification.NotificationSetting
-	if err := c.ShouldBindJSON(&s); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	if err := ctx.svc.PutSetting(id, s); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-func (ctx *notificationContext) DeleteSetting(c *gin.Context) {
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	if err := ctx.svc.DeleteSetting(id); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-func (ctx *notificationContext) EnableSetting(c *gin.Context) {
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	if err := ctx.svc.EnableSetting(id); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-func (ctx *notificationContext) DisableSetting(c *gin.Context) {
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	if err := ctx.svc.DisableSetting(id); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-func newNotificationContext() *notificationContext {
-	ctx := &notificationContext{
-		svc: notification.GetService(),
-	}
-	return ctx
-}

core/controllers/permission.go

@@ -1,3 +0,0 @@
-package controllers
-
-var PermissionController ListController

core/controllers/project.go

@@ -1,103 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/errors"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/crawlab-team/crawlab/core/models/models"
-	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/gin-gonic/gin"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-var ProjectController *projectController
-
-type projectController struct {
-	ListControllerDelegate
-}
-
-func (ctr *projectController) GetList(c *gin.Context) {
-	// get all if query field "all" is set true
-	all := MustGetFilterAll(c)
-	if all {
-		ctr.getAll(c)
-		return
-	}
-
-	// get list
-	list, total, err := ctr.getList(c)
-	if err != nil {
-		return
-	}
-	data := list.GetModels()
-
-	// check empty list
-	if len(list.GetModels()) == 0 {
-		HandleSuccessWithListData(c, nil, 0)
-		return
-	}
-
-	// project ids
-	var ids []primitive.ObjectID
-
-	// count cache
-	cache := map[primitive.ObjectID]int{}
-
-	// iterate
-	for _, d := range data {
-		p, ok := d.(*models.Project)
-		if !ok {
-			HandleErrorInternalServerError(c, errors.ErrorControllerInvalidType)
-			return
-		}
-		ids = append(ids, p.Id)
-		cache[p.Id] = 0
-	}
-
-	// spiders
-	modelSvc, err := service.NewService()
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	spiders, err := modelSvc.GetSpiderList(bson.M{
-		"project_id": bson.M{
-			"$in": ids,
-		},
-	}, nil)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	for _, s := range spiders {
-		_, ok := cache[s.ProjectId]
-		if !ok {
-			HandleErrorInternalServerError(c, errors.ErrorControllerMissingInCache)
-			return
-		}
-		cache[s.ProjectId]++
-	}
-
-	// assign
-	var projects []models.Project
-	for _, d := range data {
-		p := d.(*models.Project)
-		p.Spiders = cache[p.Id]
-		projects = append(projects, *p)
-	}
-
-	HandleSuccessWithListData(c, projects, total)
-}
-
-func newProjectController() *projectController {
-	modelSvc, err := service.GetService()
-	if err != nil {
-		panic(err)
-	}
-
-	ctr := NewListControllerDelegate(ControllerIdProject, modelSvc.GetBaseService(interfaces.ModelIdProject))
-
-	return &projectController{
-		ListControllerDelegate: *ctr,
-	}
-}

core/controllers/result.go

@@ -1,150 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/models/models"
-	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/crawlab-team/crawlab/core/result"
-	"github.com/crawlab-team/crawlab/core/utils"
-	"github.com/crawlab-team/crawlab/db/generic"
-	"github.com/gin-gonic/gin"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	mongo2 "go.mongodb.org/mongo-driver/mongo"
-	"net/http"
-)
-
-var ResultController ActionController
-
-func getResultActions() []Action {
-	var resultCtx = newResultContext()
-	return []Action{
-		{
-			Method:      http.MethodGet,
-			Path:        "/:id",
-			HandlerFunc: resultCtx.getList,
-		},
-	}
-}
-
-type resultContext struct {
-	modelSvc service.ModelService
-}
-
-func (ctx *resultContext) getList(c *gin.Context) {
-	// data collection id
-	dcId, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// data source id
-	var dsId primitive.ObjectID
-	dsIdStr := c.Query("data_source_id")
-	if dsIdStr != "" {
-		dsId, err = primitive.ObjectIDFromHex(dsIdStr)
-		if err != nil {
-			HandleErrorBadRequest(c, err)
-			return
-		}
-	}
-
-	// data collection
-	dc, err := ctx.modelSvc.GetDataCollectionById(dcId)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// data source
-	ds, err := ctx.modelSvc.GetDataSourceById(dsId)
-	if err != nil {
-		if err.Error() == mongo2.ErrNoDocuments.Error() {
-			ds = &models.DataSource{}
-		} else {
-			HandleErrorInternalServerError(c, err)
-			return
-		}
-	}
-
-	// spider
-	sq := bson.M{
-		"col_id":         dc.Id,
-		"data_source_id": ds.Id,
-	}
-	s, err := ctx.modelSvc.GetSpider(sq, nil)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// service
-	svc, err := result.GetResultService(s.Id)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// params
-	pagination := MustGetPagination(c)
-	query := ctx.getListQuery(c)
-
-	// get results
-	data, err := svc.List(query, &generic.ListOptions{
-		Sort:  []generic.ListSort{{"_id", generic.SortDirectionDesc}},
-		Skip:  pagination.Size * (pagination.Page - 1),
-		Limit: pagination.Size,
-	})
-	if err != nil {
-		if err.Error() == mongo2.ErrNoDocuments.Error() {
-			HandleSuccessWithListData(c, nil, 0)
-			return
-		}
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// validate results
-	if len(data) == 0 {
-		HandleSuccessWithListData(c, nil, 0)
-		return
-	}
-
-	// total count
-	total, err := svc.Count(query)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// response
-	HandleSuccessWithListData(c, data, total)
-}
-
-func (ctx *resultContext) getListQuery(c *gin.Context) (q generic.ListQuery) {
-	f, err := GetFilter(c)
-	if err != nil {
-		return q
-	}
-	for _, cond := range f.Conditions {
-		q = append(q, generic.ListQueryCondition{
-			Key:   cond.Key,
-			Op:    cond.Op,
-			Value: utils.NormalizeObjectId(cond.Value),
-		})
-	}
-	return q
-}
-
-func newResultContext() *resultContext {
-	// context
-	ctx := &resultContext{}
-
-	var err error
-	ctx.modelSvc, err = service.NewService()
-	if err != nil {
-		panic(err)
-	}
-
-	return ctx
-}

core/controllers/role.go

@@ -1,3 +0,0 @@
-package controllers
-
-var RoleController ListController

core/controllers/router_v2.go

@@ -357,13 +357,6 @@ func InitRoutes(app *gin.Engine) (err error) {
 			HandlerFunc: GetSystemInfo,
 		},
 	})
-	RegisterActions(groups.AnonymousGroup, "/version", []Action{
-		{
-			Method:      http.MethodGet,
-			Path:        "",
-			HandlerFunc: GetVersion,
-		},
-	})
 	RegisterActions(groups.AnonymousGroup, "/", []Action{
 		{
 			Method:      http.MethodPost,

core/controllers/schedule.go

@@ -1,221 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/container"
-	"github.com/crawlab-team/crawlab/core/errors"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/crawlab-team/crawlab/core/models/delegate"
-	"github.com/crawlab-team/crawlab/core/models/models"
-	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/gin-gonic/gin"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"net/http"
-)
-
-var ScheduleController *scheduleController
-
-func getScheduleActions() []Action {
-	scheduleCtx := newScheduleContext()
-	return []Action{
-		{
-			Method:      http.MethodPost,
-			Path:        "/:id/enable",
-			HandlerFunc: scheduleCtx.enable,
-		},
-		{
-			Method:      http.MethodPost,
-			Path:        "/:id/disable",
-			HandlerFunc: scheduleCtx.disable,
-		},
-	}
-}
-
-type scheduleController struct {
-	ListActionControllerDelegate
-	d   ListActionControllerDelegate
-	ctx *scheduleContext
-}
-
-func (ctr *scheduleController) Post(c *gin.Context) {
-	var s models.Schedule
-	if err := c.ShouldBindJSON(&s); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	if err := delegate.NewModelDelegate(&s, GetUserFromContext(c)).Add(); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	if s.Enabled {
-		if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil {
-			HandleErrorInternalServerError(c, err)
-			return
-		}
-	}
-	HandleSuccessWithData(c, s)
-}
-
-func (ctr *scheduleController) Put(c *gin.Context) {
-	id := c.Param("id")
-	oid, err := primitive.ObjectIDFromHex(id)
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	var s models.Schedule
-	if err := c.ShouldBindJSON(&s); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	if s.GetId() != oid {
-		HandleErrorBadRequest(c, errors.ErrorHttpBadRequest)
-		return
-	}
-	if err := delegate.NewModelDelegate(&s).Save(); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	if s.Enabled {
-		if err := ctr.ctx.scheduleSvc.Disable(&s, GetUserFromContext(c)); err != nil {
-			HandleErrorInternalServerError(c, err)
-			return
-		}
-		if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil {
-			HandleErrorInternalServerError(c, err)
-			return
-		}
-	}
-	HandleSuccessWithData(c, s)
-}
-
-func (ctr *scheduleController) Delete(c *gin.Context) {
-	id := c.Param("id")
-	oid, err := primitive.ObjectIDFromHex(id)
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	s, err := ctr.ctx.modelSvc.GetScheduleById(oid)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	if err := ctr.ctx.scheduleSvc.Disable(s); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	if err := delegate.NewModelDelegate(s, GetUserFromContext(c)).Delete(); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-}
-
-func (ctr *scheduleController) DeleteList(c *gin.Context) {
-	payload, err := NewJsonBinder(interfaces.ModelIdSchedule).BindBatchRequestPayload(c)
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	for _, id := range payload.Ids {
-		s, err := ctr.ctx.modelSvc.GetScheduleById(id)
-		if err != nil {
-			HandleErrorInternalServerError(c, err)
-			return
-		}
-		if err := ctr.ctx.scheduleSvc.Disable(s); err != nil {
-			HandleErrorInternalServerError(c, err)
-			return
-		}
-	}
-	if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdSchedule).DeleteList(bson.M{
-		"_id": bson.M{
-			"$in": payload.Ids,
-		},
-	}); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-func (ctx *scheduleContext) enable(c *gin.Context) {
-	s, err := ctx._getSchedule(c)
-	if err != nil {
-		return
-	}
-	if err := ctx.scheduleSvc.Enable(s, GetUserFromContext(c)); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-func (ctx *scheduleContext) disable(c *gin.Context) {
-	s, err := ctx._getSchedule(c)
-	if err != nil {
-		return
-	}
-	if err := ctx.scheduleSvc.Disable(s, GetUserFromContext(c)); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-func (ctx *scheduleContext) _getSchedule(c *gin.Context) (s *models.Schedule, err error) {
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	s, err = ctx.modelSvc.GetScheduleById(id)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	return s, nil
-}
-
-type scheduleContext struct {
-	modelSvc    service.ModelService
-	scheduleSvc interfaces.ScheduleService
-}
-
-func newScheduleContext() *scheduleContext {
-	// context
-	ctx := &scheduleContext{}
-
-	// dependency injection
-	if err := container.GetContainer().Invoke(func(
-		modelSvc service.ModelService,
-		scheduleSvc interfaces.ScheduleService,
-	) {
-		ctx.modelSvc = modelSvc
-		ctx.scheduleSvc = scheduleSvc
-	}); err != nil {
-		panic(err)
-	}
-
-	return ctx
-}
-
-func newScheduleController() *scheduleController {
-	actions := getScheduleActions()
-	modelSvc, err := service.GetService()
-	if err != nil {
-		panic(err)
-	}
-
-	ctr := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions)
-	d := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions)
-	ctx := newScheduleContext()
-
-	return &scheduleController{
-		ListActionControllerDelegate: *ctr,
-		d:                            *d,
-		ctx:                          ctx,
-	}
-}

core/controllers/setting.go

@@ -1,84 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/crawlab-team/crawlab/core/models/delegate"
-	"github.com/crawlab-team/crawlab/core/models/models"
-	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/gin-gonic/gin"
-)
-
-var SettingController *settingController
-
-type settingController struct {
-	ListControllerDelegate
-}
-
-func (ctr *settingController) Get(c *gin.Context) {
-	// key
-	key := c.Param("id")
-
-	// model service
-	modelSvc, err := service.NewService()
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// setting
-	s, err := modelSvc.GetSettingByKey(key, nil)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	HandleSuccessWithData(c, s)
-}
-
-func (ctr *settingController) Put(c *gin.Context) {
-	// key
-	key := c.Param("id")
-
-	// settings
-	var s models.Setting
-	if err := c.ShouldBindJSON(&s); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// model service
-	modelSvc, err := service.NewService()
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// setting
-	_s, err := modelSvc.GetSettingByKey(key, nil)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// save
-	_s.Value = s.Value
-	if err := delegate.NewModelDelegate(_s).Save(); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	HandleSuccess(c)
-}
-
-func newSettingController() *settingController {
-	modelSvc, err := service.GetService()
-	if err != nil {
-		panic(err)
-	}
-
-	ctr := NewListControllerDelegate(ControllerIdSetting, modelSvc.GetBaseService(interfaces.ModelIdSetting))
-
-	return &settingController{
-		ListControllerDelegate: *ctr,
-	}
-}

core/controllers/stats.go

@@ -1,87 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/container"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/gin-gonic/gin"
-	"go.mongodb.org/mongo-driver/bson"
-	"net/http"
-	"time"
-)
-
-var StatsController ActionController
-
-func getStatsActions() []Action {
-	statsCtx := newStatsContext()
-	return []Action{
-		{
-			Method:      http.MethodGet,
-			Path:        "/overview",
-			HandlerFunc: statsCtx.getOverview,
-		},
-		{
-			Method:      http.MethodGet,
-			Path:        "/daily",
-			HandlerFunc: statsCtx.getDaily,
-		},
-		{
-			Method:      http.MethodGet,
-			Path:        "/tasks",
-			HandlerFunc: statsCtx.getTasks,
-		},
-	}
-}
-
-type statsContext struct {
-	statsSvc     interfaces.StatsService
-	defaultQuery bson.M
-}
-
-func (svc *statsContext) getOverview(c *gin.Context) {
-	data, err := svc.statsSvc.GetOverviewStats(svc.defaultQuery)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccessWithData(c, data)
-}
-
-func (svc *statsContext) getDaily(c *gin.Context) {
-	data, err := svc.statsSvc.GetDailyStats(svc.defaultQuery)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccessWithData(c, data)
-}
-
-func (svc *statsContext) getTasks(c *gin.Context) {
-	data, err := svc.statsSvc.GetTaskStats(svc.defaultQuery)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccessWithData(c, data)
-}
-
-func newStatsContext() *statsContext {
-	// context
-	ctx := &statsContext{
-		defaultQuery: bson.M{
-			"create_ts": bson.M{
-				"$gte": time.Now().Add(-30 * 24 * time.Hour),
-			},
-		},
-	}
-
-	// dependency injection
-	if err := container.GetContainer().Invoke(func(
-		statsSvc interfaces.StatsService,
-	) {
-		ctx.statsSvc = statsSvc
-	}); err != nil {
-		panic(err)
-	}
-
-	return ctx
-}

core/controllers/sync.go

@@ -1,58 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/utils"
-	"github.com/gin-gonic/gin"
-	"github.com/spf13/viper"
-	"net/http"
-	"path/filepath"
-)
-
-var SyncController ActionController
-
-func getSyncActions() []Action {
-	var ctx = newSyncContext()
-	return []Action{
-		{
-			Method:      http.MethodGet,
-			Path:        "/:id/scan",
-			HandlerFunc: ctx.scan,
-		},
-		{
-			Method:      http.MethodGet,
-			Path:        "/:id/download",
-			HandlerFunc: ctx.download,
-		},
-	}
-}
-
-type syncContext struct {
-}
-
-func (ctx *syncContext) scan(c *gin.Context) {
-	id := c.Param("id")
-	path := c.Query("path")
-	dir := ctx._getDir(id, path)
-	files, err := utils.ScanDirectory(dir)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	c.AbortWithStatusJSON(http.StatusOK, files)
-}
-
-func (ctx *syncContext) download(c *gin.Context) {
-	id := c.Param("id")
-	filePath := c.Query("path")
-	dir := ctx._getDir(id, "")
-	c.File(filepath.Join(dir, filePath))
-}
-
-func (ctx *syncContext) _getDir(id string, path string) string {
-	workspacePath := viper.GetString("workspace")
-	return filepath.Join(workspacePath, id, path)
-}
-
-func newSyncContext() syncContext {
-	return syncContext{}
-}

core/controllers/system_info.go

@@ -1,28 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/entity"
-	"github.com/gin-gonic/gin"
-	"github.com/spf13/viper"
-	"net/http"
-)
-
-func getSystemInfo(c *gin.Context) {
-	info := &entity.SystemInfo{
-		Edition: viper.GetString("info.edition"),
-		Version: viper.GetString("info.version"),
-	}
-	HandleSuccessWithData(c, info)
-}
-
-func getSystemInfoActions() []Action {
-	return []Action{
-		{
-			Path:        "",
-			Method:      http.MethodGet,
-			HandlerFunc: getSystemInfo,
-		},
-	}
-}
-
-var SystemInfoController ActionController

core/controllers/system_info_v2.go

@@ -8,8 +8,8 @@ import (
 
 func GetSystemInfo(c *gin.Context) {
 	info := &entity.SystemInfo{
-		Edition: viper.GetString("info.edition"),
-		Version: viper.GetString("info.version"),
+		Edition: viper.GetString("edition"),
+		Version: viper.GetString("version"),
 	}
 	HandleSuccessWithData(c, info)
 }

core/controllers/tag.go

@@ -1,3 +0,0 @@
-package controllers
-
-var TagController ListController

core/controllers/task.go

@@ -1,534 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/constants"
-	"github.com/crawlab-team/crawlab/core/container"
-	"github.com/crawlab-team/crawlab/core/errors"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	delegate2 "github.com/crawlab-team/crawlab/core/models/delegate"
-	"github.com/crawlab-team/crawlab/core/models/models"
-	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/crawlab-team/crawlab/core/result"
-	"github.com/crawlab-team/crawlab/core/task/log"
-	"github.com/crawlab-team/crawlab/core/utils"
-	"github.com/crawlab-team/crawlab/db/generic"
-	"github.com/crawlab-team/crawlab/db/mongo"
-	"github.com/gin-gonic/gin"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	mongo2 "go.mongodb.org/mongo-driver/mongo"
-	"net/http"
-	"strings"
-)
-
-var TaskController *taskController
-
-func getTaskActions() []Action {
-	taskCtx := newTaskContext()
-	return []Action{
-		{
-			Method:      http.MethodPost,
-			Path:        "/run",
-			HandlerFunc: taskCtx.run,
-		},
-		{
-			Method:      http.MethodPost,
-			Path:        "/:id/restart",
-			HandlerFunc: taskCtx.restart,
-		},
-		{
-			Method:      http.MethodPost,
-			Path:        "/:id/cancel",
-			HandlerFunc: taskCtx.cancel,
-		},
-		{
-			Method:      http.MethodGet,
-			Path:        "/:id/logs",
-			HandlerFunc: taskCtx.getLogs,
-		},
-		{
-			Method:      http.MethodGet,
-			Path:        "/:id/data",
-			HandlerFunc: taskCtx.getData,
-		},
-	}
-}
-
-type taskController struct {
-	ListActionControllerDelegate
-	d   ListActionControllerDelegate
-	ctx *taskContext
-}
-
-func (ctr *taskController) Get(c *gin.Context) {
-	ctr.ctx.getWithStatsSpider(c)
-}
-
-func (ctr *taskController) Delete(c *gin.Context) {
-	if err := ctr.ctx._delete(c); err != nil {
-		return
-	}
-	HandleSuccess(c)
-}
-
-func (ctr *taskController) GetList(c *gin.Context) {
-	withStats := c.Query("stats")
-	if withStats == "" {
-		ctr.d.GetList(c)
-		return
-	}
-	ctr.ctx.getListWithStats(c)
-}
-
-func (ctr *taskController) DeleteList(c *gin.Context) {
-	if err := ctr.ctx._deleteList(c); err != nil {
-		return
-	}
-	HandleSuccess(c)
-}
-
-type taskContext struct {
-	modelSvc         service.ModelService
-	modelTaskSvc     interfaces.ModelBaseService
-	modelTaskStatSvc interfaces.ModelBaseService
-	adminSvc         interfaces.SpiderAdminService
-	schedulerSvc     interfaces.TaskSchedulerService
-	l                log.Driver
-}
-
-func (ctx *taskContext) run(c *gin.Context) {
-	// task
-	var t models.Task
-	if err := c.ShouldBindJSON(&t); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// validate spider id
-	if t.GetSpiderId().IsZero() {
-		HandleErrorBadRequest(c, errors.ErrorTaskEmptySpiderId)
-		return
-	}
-
-	// spider
-	s, err := ctx.modelSvc.GetSpiderById(t.GetSpiderId())
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// options
-	opts := &interfaces.SpiderRunOptions{
-		Mode:     t.Mode,
-		NodeIds:  t.NodeIds,
-		Cmd:      t.Cmd,
-		Param:    t.Param,
-		Priority: t.Priority,
-	}
-
-	// user
-	if u := GetUserFromContext(c); u != nil {
-		opts.UserId = u.GetId()
-	}
-
-	// run
-	taskIds, err := ctx.adminSvc.Schedule(s.GetId(), opts)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	HandleSuccessWithData(c, taskIds)
-}
-
-func (ctx *taskContext) restart(c *gin.Context) {
-	// id
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// task
-	t, err := ctx.modelSvc.GetTaskById(id)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// options
-	opts := &interfaces.SpiderRunOptions{
-		Mode:     t.Mode,
-		NodeIds:  t.NodeIds,
-		Cmd:      t.Cmd,
-		Param:    t.Param,
-		Priority: t.Priority,
-	}
-
-	// user
-	if u := GetUserFromContext(c); u != nil {
-		opts.UserId = u.GetId()
-	}
-
-	// run
-	taskIds, err := ctx.adminSvc.Schedule(t.SpiderId, opts)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	HandleSuccessWithData(c, taskIds)
-}
-
-func (ctx *taskContext) cancel(c *gin.Context) {
-	// id
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// task
-	t, err := ctx.modelSvc.GetTaskById(id)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// validate
-	if !utils.IsCancellable(t.Status) {
-		HandleErrorInternalServerError(c, errors.ErrorControllerNotCancellable)
-		return
-	}
-
-	// cancel
-	if err := ctx.schedulerSvc.Cancel(id, GetUserFromContext(c)); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	HandleSuccess(c)
-}
-
-func (ctx *taskContext) getLogs(c *gin.Context) {
-	// id
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// pagination
-	p, err := GetPagination(c)
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// logs
-	logs, err := ctx.l.Find(id.Hex(), "", (p.Page-1)*p.Size, p.Size)
-	if err != nil {
-		if strings.HasSuffix(err.Error(), "Status:404 Not Found") {
-			HandleSuccess(c)
-			return
-		}
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	total, err := ctx.l.Count(id.Hex(), "")
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	HandleSuccessWithListData(c, logs, total)
-}
-
-func (ctx *taskContext) getListWithStats(c *gin.Context) {
-	// params
-	pagination := MustGetPagination(c)
-	query := MustGetFilterQuery(c)
-	sort := MustGetSortOption(c)
-
-	// get list
-	list, err := ctx.modelTaskSvc.GetList(query, &mongo.FindOptions{
-		Sort:  sort,
-		Skip:  pagination.Size * (pagination.Page - 1),
-		Limit: pagination.Size,
-	})
-	if err != nil {
-		if err == mongo2.ErrNoDocuments {
-			HandleErrorNotFound(c, err)
-		} else {
-			HandleErrorInternalServerError(c, err)
-		}
-		return
-	}
-
-	// check empty list
-	if len(list.GetModels()) == 0 {
-		HandleSuccessWithListData(c, nil, 0)
-		return
-	}
-
-	// ids
-	var ids []primitive.ObjectID
-	for _, d := range list.GetModels() {
-		t := d.(interfaces.Model)
-		ids = append(ids, t.GetId())
-	}
-
-	// total count
-	total, err := ctx.modelTaskSvc.Count(query)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// stat list
-	query = bson.M{
-		"_id": bson.M{
-			"$in": ids,
-		},
-	}
-	stats, err := ctx.modelSvc.GetTaskStatList(query, nil)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// cache stat list to dict
-	dict := map[primitive.ObjectID]models.TaskStat{}
-	for _, s := range stats {
-		dict[s.GetId()] = s
-	}
-
-	// iterate list again
-	var data []interface{}
-	for _, d := range list.GetModels() {
-		t := d.(*models.Task)
-		s, ok := dict[t.GetId()]
-		if ok {
-			t.Stat = &s
-		}
-		data = append(data, *t)
-	}
-
-	// response
-	HandleSuccessWithListData(c, data, total)
-}
-
-func (ctx *taskContext) getWithStatsSpider(c *gin.Context) {
-	// id
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// task
-	t, err := ctx.modelSvc.GetTaskById(id)
-	if err == mongo2.ErrNoDocuments {
-		HandleErrorNotFound(c, err)
-		return
-	}
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// spider
-	t.Spider, _ = ctx.modelSvc.GetSpiderById(t.SpiderId)
-
-	// skip if task status is pending
-	if t.Status == constants.TaskStatusPending {
-		HandleSuccessWithData(c, t)
-		return
-	}
-
-	// task stat
-	t.Stat, _ = ctx.modelSvc.GetTaskStatById(id)
-
-	HandleSuccessWithData(c, t)
-}
-
-func (ctx *taskContext) getData(c *gin.Context) {
-	// id
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// pagination
-	p, err := GetPagination(c)
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// task
-	t, err := ctx.modelSvc.GetTaskById(id)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// result service
-	resultSvc, err := result.GetResultService(t.SpiderId)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// query
-	query := generic.ListQuery{
-		generic.ListQueryCondition{
-			Key:   constants.TaskKey,
-			Op:    generic.OpEqual,
-			Value: t.Id,
-		},
-	}
-
-	// list
-	data, err := resultSvc.List(query, &generic.ListOptions{
-		Skip:  (p.Page - 1) * p.Size,
-		Limit: p.Size,
-		Sort:  []generic.ListSort{{"_id", generic.SortDirectionDesc}},
-	})
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// total
-	total, err := resultSvc.Count(query)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	HandleSuccessWithListData(c, data, total)
-}
-
-func (ctx *taskContext) _delete(c *gin.Context) (err error) {
-	id := c.Param("id")
-	oid, err := primitive.ObjectIDFromHex(id)
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) {
-		// delete task
-		task, err := ctx.modelSvc.GetTaskById(oid)
-		if err != nil {
-			return err
-		}
-		if err := delegate2.NewModelDelegate(task, GetUserFromContext(c)).Delete(); err != nil {
-			return err
-		}
-
-		// delete task stat
-		taskStat, err := ctx.modelSvc.GetTaskStatById(oid)
-		if err != nil {
-			return err
-		}
-		if err := delegate2.NewModelDelegate(taskStat, GetUserFromContext(c)).Delete(); err != nil {
-			return err
-		}
-
-		return nil
-	}); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return err
-	}
-
-	return nil
-}
-
-func (ctx *taskContext) _deleteList(c *gin.Context) (err error) {
-	payload, err := NewJsonBinder(ControllerIdTask).BindBatchRequestPayload(c)
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	if err := mongo.RunTransaction(func(context mongo2.SessionContext) error {
-		// delete tasks
-		if err := ctx.modelTaskSvc.DeleteList(bson.M{
-			"_id": bson.M{
-				"$in": payload.Ids,
-			},
-		}); err != nil {
-			return err
-		}
-
-		// delete task stats
-		if err := ctx.modelTaskStatSvc.DeleteList(bson.M{
-			"_id": bson.M{
-				"$in": payload.Ids,
-			},
-		}); err != nil {
-			return err
-		}
-
-		return nil
-	}); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return err
-	}
-
-	return nil
-}
-
-func newTaskContext() *taskContext {
-	// context
-	ctx := &taskContext{}
-
-	// dependency injection
-	if err := container.GetContainer().Invoke(func(
-		modelSvc service.ModelService,
-		adminSvc interfaces.SpiderAdminService,
-		schedulerSvc interfaces.TaskSchedulerService,
-	) {
-		ctx.modelSvc = modelSvc
-		ctx.adminSvc = adminSvc
-		ctx.schedulerSvc = schedulerSvc
-	}); err != nil {
-		panic(err)
-	}
-
-	// model task service
-	ctx.modelTaskSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTask)
-
-	// model task stat service
-	ctx.modelTaskStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTaskStat)
-
-	// log driver
-	l, err := log.GetLogDriver(log.DriverTypeFile)
-	if err != nil {
-		panic(err)
-	}
-	ctx.l = l
-
-	return ctx
-}
-
-func newTaskController() *taskController {
-	actions := getTaskActions()
-	modelSvc, err := service.GetService()
-	if err != nil {
-		panic(err)
-	}
-
-	ctr := NewListPostActionControllerDelegate(ControllerIdTask, modelSvc.GetBaseService(interfaces.ModelIdTask), actions)
-	d := NewListPostActionControllerDelegate(ControllerIdTask, modelSvc.GetBaseService(interfaces.ModelIdTask), actions)
-	ctx := newTaskContext()
-
-	return &taskController{
-		ListActionControllerDelegate: *ctr,
-		d:                            *d,
-		ctx:                          ctx,
-	}
-}

core/controllers/test/base.go

@@ -1,104 +0,0 @@
-package test
-
-import (
-	"github.com/crawlab-team/crawlab/core/controllers"
-	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/crawlab-team/crawlab/core/routes"
-	"github.com/crawlab-team/crawlab/trace"
-	"github.com/gavv/httpexpect/v2"
-	"github.com/gin-gonic/gin"
-	"github.com/stretchr/testify/require"
-	"go.uber.org/dig"
-	"net/http/httptest"
-	"testing"
-	"time"
-)
-
-func init() {
-	var err error
-	T, err = NewTest()
-	if err != nil {
-		panic(err)
-	}
-}
-
-type Test struct {
-	// dependencies
-	modelSvc service.ModelService
-
-	// internals
-	app *gin.Engine
-	svr *httptest.Server
-
-	// test data
-	TestUsername string
-	TestPassword string
-	TestToken    string
-}
-
-func (t *Test) Setup(t2 *testing.T) {
-	//if err := controllers.InitControllers(); err != nil {
-	//	panic(err)
-	//}
-	//t2.Cleanup(t.Cleanup)
-}
-
-func (t *Test) Cleanup() {
-	_ = t.modelSvc.DropAll()
-	time.Sleep(200 * time.Millisecond)
-}
-
-func (t *Test) NewExpect(t2 *testing.T) (e *httpexpect.Expect) {
-	e = httpexpect.New(t2, t.svr.URL)
-	res := e.POST("/login").WithJSON(map[string]string{
-		"username": t.TestUsername,
-		"password": t.TestPassword,
-	}).Expect().JSON().Object()
-	t.TestToken = res.Path("$.data").String().Raw()
-	require.NotEmpty(t2, t.TestToken)
-	return e
-}
-
-func (t *Test) WithAuth(req *httpexpect.Request) *httpexpect.Request {
-	return req.WithHeader("Authorization", t.TestToken)
-}
-
-var T *Test
-
-func NewTest() (res *Test, err error) {
-	// test
-	t := &Test{}
-
-	// gin app
-	t.app = gin.New()
-
-	// http test server
-	t.svr = httptest.NewServer(t.app)
-
-	// init controllers
-	if err := controllers.InitControllers(); err != nil {
-		return nil, err
-	}
-
-	// init routes
-	if err := routes.InitRoutes(t.app); err != nil {
-		return nil, err
-	}
-
-	// dependency injection
-	c := dig.New()
-	if err := c.Provide(service.NewService); err != nil {
-		return nil, trace.TraceError(err)
-	}
-	if err := c.Invoke(func(modelSvc service.ModelService) {
-		t.modelSvc = modelSvc
-	}); err != nil {
-		return nil, trace.TraceError(err)
-	}
-
-	// test data
-	t.TestUsername = "admin"
-	t.TestPassword = "admin"
-
-	return t, nil
-}

core/controllers/test/export_test.go

@@ -1,59 +0,0 @@
-package test
-
-import (
-	"github.com/crawlab-team/crawlab/core/constants"
-	"github.com/crawlab-team/crawlab/db/mongo"
-	"github.com/spf13/viper"
-	"github.com/stretchr/testify/require"
-	"go.mongodb.org/mongo-driver/bson"
-	"net/http"
-	"testing"
-	"time"
-)
-
-func init() {
-	viper.Set("mongo.db", "crawlab_test")
-}
-
-func TestExportController_Csv(t *testing.T) {
-	T.Setup(t)
-	e := T.NewExpect(t)
-
-	// mongo collection
-	colName := "test_collection_for_export"
-	col := mongo.GetMongoCol(colName)
-
-	// insert test data to mongo collection
-	for i := 0; i < 10; i++ {
-		_, err := col.Insert(bson.M{
-			"field1": i + 1,
-			"field2": i + 2,
-			"field3": i + 3,
-			"field4": i + 4,
-		})
-		require.Nil(t, err)
-	}
-
-	// export from mongo collection
-	res := T.WithAuth(e.POST("/export/csv")).
-		WithQuery("target", colName).
-		Expect().Status(http.StatusOK).JSON().Object()
-	res.Path("$.data").NotNull()
-
-	// export id
-	exportId := res.Path("$.data").String().Raw()
-
-	// poll export with export id
-	for i := 0; i < 10; i++ {
-		res = T.WithAuth(e.GET("/export/csv/" + exportId)).Expect().Status(http.StatusOK).JSON().Object()
-		status := res.Path("$.data.status").String().Raw()
-		if status == constants.TaskStatusFinished {
-			break
-		}
-		time.Sleep(1 * time.Second)
-	}
-
-	// download exported csv file
-	csvFileBody := T.WithAuth(e.GET("/export/csv/" + exportId + "/download")).Expect().Status(http.StatusOK).Body()
-	csvFileBody.NotEmpty()
-}

core/controllers/test/filter_test.go

@@ -1,76 +0,0 @@
-package test
-
-import (
-	"encoding/json"
-	"fmt"
-	"github.com/crawlab-team/crawlab/core/constants"
-	"github.com/crawlab-team/crawlab/core/entity"
-	"github.com/crawlab-team/crawlab/db/mongo"
-	"github.com/spf13/viper"
-	"github.com/stretchr/testify/require"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"net/http"
-	"testing"
-)
-
-func init() {
-	viper.Set("mongo.db", "crawlab_test")
-}
-
-func TestFilterController_GetColFieldOptions(t *testing.T) {
-	T.Setup(t)
-	e := T.NewExpect(t)
-
-	// mongo collection
-	colName := "test_collection_for_filter"
-	field1 := "field1"
-	field2 := "field2"
-	value1 := "value1"
-	col := mongo.GetMongoCol(colName)
-	n := 10
-	var ids []primitive.ObjectID
-	var names []string
-	for i := 0; i < n; i++ {
-		_id := primitive.NewObjectID()
-		ids = append(ids, _id)
-		name := fmt.Sprintf("name_%d", i)
-		names = append(names, name)
-		_, err := col.Insert(bson.M{field1: value1, field2: i % 2, "name": name, "_id": _id})
-		require.Nil(t, err)
-	}
-
-	// validate filter options field 1
-	res := T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field1, field1))).
-		Expect().Status(http.StatusOK).JSON().Object()
-	res.Path("$.data").NotNull()
-	res.Path("$.data").Array().Length().Equal(1)
-	res.Path("$.data").Array().Element(0).Path("$.value").Equal(value1)
-
-	// validate filter options field 2
-	res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))).
-		Expect().Status(http.StatusOK).JSON().Object()
-	res.Path("$.data").NotNull()
-	res.Path("$.data").Array().Length().Equal(2)
-
-	// validate filter options with query
-	conditions := []entity.Condition{{field2, constants.FilterOpEqual, 0}}
-	conditionsJson, err := json.Marshal(conditions)
-	conditionsJsonStr := string(conditionsJson)
-	require.Nil(t, err)
-	res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))).
-		WithQuery(constants.FilterQueryFieldConditions, conditionsJsonStr).
-		Expect().Status(http.StatusOK).JSON().Object()
-	res.Path("$.data").NotNull()
-	res.Path("$.data").Array().Length().Equal(1)
-
-	// validate filter options (basic path)
-	res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s", colName))).
-		Expect().Status(http.StatusOK).JSON().Object()
-	res.Path("$.data").NotNull()
-	res.Path("$.data").Array().Length().Equal(n)
-	for i := 0; i < n; i++ {
-		res.Path("$.data").Array().Element(i).Object().Value("value").Equal(ids[i])
-		res.Path("$.data").Array().Element(i).Object().Value("label").Equal(names[i])
-	}
-}

core/controllers/test/main_test.go

@@ -1,44 +0,0 @@
-package test
-
-import (
-	"github.com/crawlab-team/crawlab/core/constants"
-	"github.com/crawlab-team/crawlab/core/controllers"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/crawlab-team/crawlab/core/user"
-	"go.mongodb.org/mongo-driver/mongo"
-	"testing"
-)
-
-func TestMain(m *testing.M) {
-	// init user
-	modelSvc, err := service.GetService()
-	if err != nil {
-		panic(err)
-	}
-	_, err = modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil)
-	if err != nil {
-		if err.Error() != mongo.ErrNoDocuments.Error() {
-			panic(err)
-		}
-		userSvc, err := user.GetUserService()
-		if err != nil {
-			panic(err)
-		}
-		if err := userSvc.Create(&interfaces.UserCreateOptions{
-			Username: constants.DefaultAdminUsername,
-			Password: constants.DefaultAdminPassword,
-			Role:     constants.RoleAdmin,
-		}); err != nil {
-			panic(err)
-		}
-	}
-
-	if err := controllers.InitControllers(); err != nil {
-		panic(err)
-	}
-
-	m.Run()
-
-	T.Cleanup()
-}

core/controllers/test/project_test.go

@@ -1,305 +0,0 @@
-package test
-
-import (
-	"encoding/json"
-	"fmt"
-	"github.com/crawlab-team/crawlab/core/constants"
-	"github.com/crawlab-team/crawlab/core/entity"
-	"github.com/crawlab-team/crawlab/core/models/delegate"
-	"github.com/crawlab-team/crawlab/core/models/models"
-	"github.com/stretchr/testify/require"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"net/http"
-	"testing"
-	"time"
-)
-
-func TestProjectController_Get(t *testing.T) {
-	T.Setup(t)
-	e := T.NewExpect(t)
-
-	p := models.Project{
-		Name: "test project",
-	}
-	res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
-	res.Path("$.data._id").NotNull()
-	id := res.Path("$.data._id").String().Raw()
-	oid, err := primitive.ObjectIDFromHex(id)
-	require.Nil(t, err)
-	require.False(t, oid.IsZero())
-
-	res = T.WithAuth(e.GET("/projects/" + id)).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
-	res.Path("$.data._id").NotNull()
-	res.Path("$.data.name").Equal("test project")
-}
-
-func TestProjectController_Put(t *testing.T) {
-	T.Setup(t)
-	e := T.NewExpect(t)
-
-	p := models.Project{
-		Name:        "old name",
-		Description: "old description",
-	}
-
-	// add
-	res := T.WithAuth(e.POST("/projects")).
-		WithJSON(p).
-		Expect().Status(http.StatusOK).
-		JSON().Object()
-	res.Path("$.data._id").NotNull()
-	id := res.Path("$.data._id").String().Raw()
-	oid, err := primitive.ObjectIDFromHex(id)
-	require.Nil(t, err)
-	require.False(t, oid.IsZero())
-
-	// change object
-	p.Id = oid
-	p.Name = "new name"
-	p.Description = "new description"
-
-	// update
-	T.WithAuth(e.PUT("/projects/" + id)).
-		WithJSON(p).
-		Expect().Status(http.StatusOK)
-
-	// check
-	res = T.WithAuth(e.GET("/projects/" + id)).Expect().Status(http.StatusOK).JSON().Object()
-	res.Path("$.data._id").Equal(id)
-	res.Path("$.data.name").Equal("new name")
-	res.Path("$.data.description").Equal("new description")
-}
-
-func TestProjectController_Post(t *testing.T) {
-	T.Setup(t)
-	e := T.NewExpect(t)
-
-	p := models.Project{
-		Name:        "test project",
-		Description: "this is a test project",
-	}
-
-	res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
-	res.Path("$.data._id").NotNull()
-	res.Path("$.data.name").Equal("test project")
-	res.Path("$.data.description").Equal("this is a test project")
-}
-
-func TestProjectController_Delete(t *testing.T) {
-	T.Setup(t)
-	e := T.NewExpect(t)
-
-	p := models.Project{
-		Name:        "test project",
-		Description: "this is a test project",
-	}
-
-	// add
-	res := T.WithAuth(e.POST("/projects")).
-		WithJSON(p).
-		Expect().Status(http.StatusOK).
-		JSON().Object()
-	res.Path("$.data._id").NotNull()
-	id := res.Path("$.data._id").String().Raw()
-	oid, err := primitive.ObjectIDFromHex(id)
-	require.Nil(t, err)
-	require.False(t, oid.IsZero())
-
-	// get
-	res = T.WithAuth(e.GET("/projects/" + id)).
-		Expect().Status(http.StatusOK).
-		JSON().Object()
-	res.Path("$.data._id").NotNull()
-	id = res.Path("$.data._id").String().Raw()
-	oid, err = primitive.ObjectIDFromHex(id)
-	require.Nil(t, err)
-	require.False(t, oid.IsZero())
-
-	// delete
-	T.WithAuth(e.DELETE("/projects/" + id)).
-		Expect().Status(http.StatusOK).
-		JSON().Object()
-
-	// get
-	T.WithAuth(e.GET("/projects/" + id)).
-		Expect().Status(http.StatusNotFound)
-}
-
-func TestProjectController_GetList(t *testing.T) {
-	T.Setup(t)
-	e := T.NewExpect(t)
-
-	n := 100 // total
-	bn := 10 // batch
-
-	for i := 0; i < n; i++ {
-		p := models.Project{
-			Name: fmt.Sprintf("test name %d", i+1),
-		}
-		obj := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
-		obj.Path("$.data._id").NotNull()
-	}
-
-	f := entity.Filter{
-		//IsOr: false,
-		Conditions: []*entity.Condition{
-			{Key: "name", Op: constants.FilterOpContains, Value: "test name"},
-		},
-	}
-	condBytes, err := json.Marshal(&f.Conditions)
-	require.Nil(t, err)
-
-	pagination := entity.Pagination{
-		Page: 1,
-		Size: bn,
-	}
-
-	// get list with pagination
-	res := T.WithAuth(e.GET("/projects")).
-		WithQuery("conditions", string(condBytes)).
-		WithQueryObject(pagination).
-		Expect().Status(http.StatusOK).JSON().Object()
-	res.Path("$.data").Array().Length().Equal(bn)
-	res.Path("$.total").Number().Equal(n)
-
-	data := res.Path("$.data").Array()
-	for i := 0; i < bn; i++ {
-		obj := data.Element(i)
-		obj.Path("$.name").Equal(fmt.Sprintf("test name %d", i+1))
-	}
-
-}
-
-func TestProjectController_PostList(t *testing.T) {
-	T.Setup(t)
-	e := T.NewExpect(t)
-
-	n := 10
-	var docs []models.Project
-	for i := 0; i < n; i++ {
-		docs = append(docs, models.Project{
-			Name:        fmt.Sprintf("project %d", i+1),
-			Description: "this is a project",
-		})
-	}
-
-	T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK)
-
-	res := T.WithAuth(e.GET("/projects")).
-		WithQueryObject(entity.Pagination{Page: 1, Size: 10}).
-		Expect().Status(http.StatusOK).
-		JSON().Object()
-	res.Path("$.data").Array().Length().Equal(n)
-}
-
-func TestProjectController_DeleteList(t *testing.T) {
-	T.Setup(t)
-	e := T.NewExpect(t)
-
-	n := 10
-	var docs []models.Project
-	for i := 0; i < n; i++ {
-		docs = append(docs, models.Project{
-			Name:        fmt.Sprintf("project %d", i+1),
-			Description: "this is a project",
-		})
-	}
-
-	// add
-	res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK).
-		JSON().Object()
-	var ids []primitive.ObjectID
-	data := res.Path("$.data").Array()
-	for i := 0; i < n; i++ {
-		obj := data.Element(i)
-		id := obj.Path("$._id").String().Raw()
-		oid, err := primitive.ObjectIDFromHex(id)
-		require.Nil(t, err)
-		require.False(t, oid.IsZero())
-		ids = append(ids, oid)
-	}
-
-	// delete
-	payload := entity.BatchRequestPayload{
-		Ids: ids,
-	}
-	T.WithAuth(e.DELETE("/projects")).
-		WithJSON(payload).
-		Expect().Status(http.StatusOK)
-
-	// check
-	for _, id := range ids {
-		T.WithAuth(e.GET("/projects/" + id.Hex())).
-			Expect().Status(http.StatusNotFound)
-	}
-
-}
-
-func TestProjectController_PutList(t *testing.T) {
-	T.Setup(t)
-	e := T.NewExpect(t)
-
-	// now
-	now := time.Now()
-
-	n := 10
-	var docs []models.Project
-	for i := 0; i < n; i++ {
-		docs = append(docs, models.Project{
-			Name:        "old name",
-			Description: "old description",
-		})
-	}
-
-	// add
-	res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK).
-		JSON().Object()
-	var ids []primitive.ObjectID
-	data := res.Path("$.data").Array()
-	for i := 0; i < n; i++ {
-		obj := data.Element(i)
-		id := obj.Path("$._id").String().Raw()
-		oid, err := primitive.ObjectIDFromHex(id)
-		require.Nil(t, err)
-		require.False(t, oid.IsZero())
-		ids = append(ids, oid)
-	}
-
-	// wait for 100 millisecond
-	time.Sleep(100 * time.Millisecond)
-
-	// update
-	p := models.Project{
-		Name:        "new name",
-		Description: "new description",
-	}
-	dataBytes, err := json.Marshal(&p)
-	require.Nil(t, err)
-	payload := entity.BatchRequestPayloadWithStringData{
-		Ids:  ids,
-		Data: string(dataBytes),
-		Fields: []string{
-			"name",
-			"description",
-		},
-	}
-	T.WithAuth(e.PUT("/projects")).WithJSON(payload).Expect().Status(http.StatusOK)
-
-	// check response data
-	for i := 0; i < n; i++ {
-		res = T.WithAuth(e.GET("/projects/" + ids[i].Hex())).Expect().Status(http.StatusOK).JSON().Object()
-		res.Path("$.data.name").Equal("new name")
-		res.Path("$.data.description").Equal("new description")
-	}
-
-	// check artifacts
-	pl, err := T.modelSvc.GetProjectList(bson.M{"_id": bson.M{"$in": ids}}, nil)
-	require.Nil(t, err)
-	for _, p := range pl {
-		a, err := delegate.NewModelDelegate(&p).GetArtifact()
-		require.Nil(t, err)
-		require.True(t, a.GetSys().GetUpdateTs().After(now))
-		require.True(t, a.GetSys().GetUpdateTs().After(a.GetSys().GetCreateTs()))
-	}
-}

core/controllers/test/spider_test.go

@@ -1,183 +0,0 @@
-package test
-
-import (
-	"github.com/crawlab-team/crawlab/core/entity"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/crawlab-team/crawlab/core/models/delegate"
-	"github.com/crawlab-team/crawlab/core/models/models"
-	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/stretchr/testify/require"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"net/http"
-	"testing"
-)
-
-func TestSpiderController_Delete(t *testing.T) {
-	T.Setup(t)
-	e := T.NewExpect(t)
-
-	s := models.Spider{
-		Name:        "test spider",
-		Description: "this is a test spider",
-		ColName:     "test col name",
-	}
-
-	// add spider
-	res := T.WithAuth(e.POST("/spiders")).
-		WithJSON(s).
-		Expect().Status(http.StatusOK).
-		JSON().Object()
-	res.Path("$.data._id").NotNull()
-	id := res.Path("$.data._id").String().Raw()
-	oid, err := primitive.ObjectIDFromHex(id)
-	require.Nil(t, err)
-	require.False(t, oid.IsZero())
-
-	// add tasks
-	var taskIds []primitive.ObjectID
-	tasks := []models.Task{
-		{
-			Id:       primitive.NewObjectID(),
-			SpiderId: oid,
-		},
-		{
-			Id:       primitive.NewObjectID(),
-			SpiderId: oid,
-		},
-	}
-	for _, task := range tasks {
-		// add task
-		err := delegate.NewModelDelegate(&task).Add()
-		require.Nil(t, err)
-
-		// add task stat
-		err = delegate.NewModelDelegate(&models.TaskStat{
-			Id: task.Id,
-		}).Add()
-		require.Nil(t, err)
-
-		taskIds = append(taskIds, task.Id)
-	}
-
-	// delete
-	T.WithAuth(e.DELETE("/spiders/" + id)).
-		Expect().Status(http.StatusOK)
-
-	// get
-	T.WithAuth(e.GET("/spiders/" + id)).
-		Expect().Status(http.StatusNotFound)
-
-	// get tasks
-	for _, task := range tasks {
-		T.WithAuth(e.GET("/tasks/" + task.Id.Hex())).
-			Expect().Status(http.StatusNotFound)
-	}
-
-	// spider stat
-	modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat)
-	spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{
-		"_id": oid,
-	})
-	require.Nil(t, err)
-	require.Zero(t, spiderStatCount)
-
-	// task stats
-	modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
-	taskStatCount, err := modelTaskStatSvc.Count(bson.M{
-		"_id": bson.M{
-			"$in": taskIds,
-		},
-	})
-	require.Nil(t, err)
-	require.Zero(t, taskStatCount)
-}
-
-func TestSpiderController_DeleteList(t *testing.T) {
-	T.Setup(t)
-	e := T.NewExpect(t)
-
-	spiders := []models.Spider{
-		{
-			Id:          primitive.NewObjectID(),
-			Name:        "test spider 1",
-			Description: "this is a test spider 1",
-			ColName:     "test col name 1",
-		},
-		{
-			Id:          primitive.NewObjectID(),
-			Name:        "test spider 2",
-			Description: "this is a test spider 2",
-			ColName:     "test col name 2",
-		},
-	}
-
-	// add spiders
-	for _, spider := range spiders {
-		T.WithAuth(e.POST("/spiders")).
-			WithJSON(spider).
-			Expect().Status(http.StatusOK)
-	}
-
-	var spiderIds []primitive.ObjectID
-	var taskIds []primitive.ObjectID
-	for _, spider := range spiders {
-		// task id
-		taskId := primitive.NewObjectID()
-
-		// add task
-		err := delegate.NewModelDelegate(&models.Task{
-			Id:       taskId,
-			SpiderId: spider.Id,
-		}).Add()
-		require.Nil(t, err)
-
-		// add task stats
-		err = delegate.NewModelDelegate(&models.TaskStat{
-			Id: taskId,
-		}).Add()
-		require.Nil(t, err)
-
-		spiderIds = append(spiderIds, spider.Id)
-		taskIds = append(taskIds, taskId)
-	}
-
-	// delete spiders
-	T.WithAuth(e.DELETE("/spiders")).
-		WithJSON(entity.BatchRequestPayload{
-			Ids: spiderIds,
-		}).Expect().Status(http.StatusOK)
-
-	// get spiders
-	for _, spider := range spiders {
-		// get
-		T.WithAuth(e.GET("/spiders/" + spider.Id.Hex())).
-			Expect().Status(http.StatusNotFound)
-	}
-
-	// get tasks
-	for _, taskId := range taskIds {
-		T.WithAuth(e.GET("/tasks/" + taskId.Hex())).
-			Expect().Status(http.StatusNotFound)
-	}
-
-	// spider stat
-	modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat)
-	spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{
-		"_id": bson.M{
-			"$in": spiderIds,
-		},
-	})
-	require.Nil(t, err)
-	require.Zero(t, spiderStatCount)
-
-	// task stats
-	modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
-	taskStatCount, err := modelTaskStatSvc.Count(bson.M{
-		"_id": bson.M{
-			"$in": taskIds,
-		},
-	})
-	require.Nil(t, err)
-	require.Zero(t, taskStatCount)
-}

core/controllers/test/task_test.go

@@ -1,102 +0,0 @@
-package test
-
-import (
-	"github.com/crawlab-team/crawlab/core/entity"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/crawlab-team/crawlab/core/models/delegate"
-	"github.com/crawlab-team/crawlab/core/models/models"
-	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/stretchr/testify/require"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"net/http"
-	"testing"
-)
-
-func TestTaskController_Delete(t *testing.T) {
-	T.Setup(t)
-	e := T.NewExpect(t)
-
-	task := models.Task{
-		Id: primitive.NewObjectID(),
-	}
-
-	// add task
-	err := delegate.NewModelDelegate(&task).Add()
-	require.Nil(t, err)
-
-	// add task stat
-	err = delegate.NewModelDelegate(&models.TaskStat{
-		Id: task.Id,
-	}).Add()
-	require.Nil(t, err)
-
-	// delete
-	T.WithAuth(e.DELETE("/tasks/" + task.Id.Hex())).
-		Expect().Status(http.StatusOK)
-
-	// get
-	T.WithAuth(e.GET("/tasks/" + task.Id.Hex())).
-		Expect().Status(http.StatusNotFound)
-
-	// task stats
-	modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
-	taskStatCount, err := modelTaskStatSvc.Count(bson.M{
-		"_id": task.Id,
-	})
-	require.Nil(t, err)
-	require.Zero(t, taskStatCount)
-}
-
-func TestTaskController_DeleteList(t *testing.T) {
-	T.Setup(t)
-	e := T.NewExpect(t)
-
-	tasks := []models.Task{
-		{
-			Id: primitive.NewObjectID(),
-		},
-		{
-			Id: primitive.NewObjectID(),
-		},
-	}
-
-	// add spiders
-	var taskIds []primitive.ObjectID
-	for _, task := range tasks {
-		// add task
-		err := delegate.NewModelDelegate(&task).Add()
-		require.Nil(t, err)
-
-		// add task stat
-		err = delegate.NewModelDelegate(&models.TaskStat{
-			Id: task.Id,
-		}).Add()
-		require.Nil(t, err)
-
-		taskIds = append(taskIds, task.Id)
-	}
-
-	// delete tasks
-	T.WithAuth(e.DELETE("/tasks")).
-		WithJSON(entity.BatchRequestPayload{
-			Ids: taskIds,
-		}).Expect().Status(http.StatusOK)
-
-	// get tasks
-	for _, task := range tasks {
-		// get
-		T.WithAuth(e.GET("/tasks/" + task.Id.Hex())).
-			Expect().Status(http.StatusNotFound)
-	}
-
-	// task stats
-	modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
-	taskStatCount, err := modelTaskStatSvc.Count(bson.M{
-		"_id": bson.M{
-			"$in": taskIds,
-		},
-	})
-	require.Nil(t, err)
-	require.Zero(t, taskStatCount)
-}

core/controllers/token.go

@@ -1,84 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/container"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/crawlab-team/crawlab/core/models/delegate"
-	"github.com/crawlab-team/crawlab/core/models/models"
-	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/gin-gonic/gin"
-)
-
-var TokenController *tokenController
-
-var TokenActions []Action
-
-type tokenController struct {
-	ListActionControllerDelegate
-	d   ListActionControllerDelegate
-	ctx *tokenContext
-}
-
-func (ctr *tokenController) Post(c *gin.Context) {
-	var err error
-	var t models.Token
-	if err := c.ShouldBindJSON(&t); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	u, err := ctr.ctx.userSvc.GetCurrentUser(c)
-	if err != nil {
-		HandleErrorUnauthorized(c, err)
-		return
-	}
-	t.Token, err = ctr.ctx.userSvc.MakeToken(u)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	if err := delegate.NewModelDelegate(&t, GetUserFromContext(c)).Add(); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-type tokenContext struct {
-	modelSvc service.ModelService
-	userSvc  interfaces.UserService
-}
-
-func newTokenContext() *tokenContext {
-	// context
-	ctx := &tokenContext{}
-
-	// dependency injection
-	if err := container.GetContainer().Invoke(func(
-		modelSvc service.ModelService,
-		userSvc interfaces.UserService,
-	) {
-		ctx.modelSvc = modelSvc
-		ctx.userSvc = userSvc
-	}); err != nil {
-		panic(err)
-	}
-
-	return ctx
-}
-
-func newTokenController() *tokenController {
-	modelSvc, err := service.GetService()
-	if err != nil {
-		panic(err)
-	}
-
-	ctr := NewListPostActionControllerDelegate(ControllerIdToken, modelSvc.GetBaseService(interfaces.ModelIdToken), TokenActions)
-	d := NewListPostActionControllerDelegate(ControllerIdToken, modelSvc.GetBaseService(interfaces.ModelIdToken), TokenActions)
-	ctx := newTokenContext()
-
-	return &tokenController{
-		ListActionControllerDelegate: *ctr,
-		d:                            *d,
-		ctx:                          ctx,
-	}
-}

core/controllers/user.go

@@ -1,244 +0,0 @@
-package controllers
-
-import (
-	"encoding/json"
-	"github.com/crawlab-team/crawlab/core/constants"
-	"github.com/crawlab-team/crawlab/core/container"
-	"github.com/crawlab-team/crawlab/core/entity"
-	"github.com/crawlab-team/crawlab/core/errors"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	delegate2 "github.com/crawlab-team/crawlab/core/models/delegate"
-	"github.com/crawlab-team/crawlab/core/models/models"
-	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/crawlab-team/crawlab/core/utils"
-	"github.com/crawlab-team/crawlab/trace"
-	"github.com/gin-gonic/gin"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"net/http"
-)
-
-var UserController *userController
-
-func getUserActions() []Action {
-	userCtx := newUserContext()
-	return []Action{
-		{
-			Method:      http.MethodPost,
-			Path:        "/:id/change-password",
-			HandlerFunc: userCtx.changePassword,
-		},
-		{
-			Method:      http.MethodGet,
-			Path:        "/me",
-			HandlerFunc: userCtx.getMe,
-		},
-		{
-			Method:      http.MethodPut,
-			Path:        "/me",
-			HandlerFunc: userCtx.putMe,
-		},
-	}
-}
-
-type userController struct {
-	ListActionControllerDelegate
-	d   ListActionControllerDelegate
-	ctx *userContext
-}
-
-func (ctr *userController) Post(c *gin.Context) {
-	var u models.User
-	if err := c.ShouldBindJSON(&u); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	if err := ctr.ctx.userSvc.Create(&interfaces.UserCreateOptions{
-		Username: u.Username,
-		Password: u.Password,
-		Email:    u.Email,
-		Role:     u.Role,
-	}); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-func (ctr *userController) PostList(c *gin.Context) {
-	// users
-	var users []models.User
-	if err := c.ShouldBindJSON(&users); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	for _, u := range users {
-		if err := ctr.ctx.userSvc.Create(&interfaces.UserCreateOptions{
-			Username: u.Username,
-			Password: u.Password,
-			Email:    u.Email,
-			Role:     u.Role,
-		}); err != nil {
-			trace.PrintError(err)
-		}
-	}
-
-	HandleSuccess(c)
-}
-
-func (ctr *userController) PutList(c *gin.Context) {
-	// payload
-	var payload entity.BatchRequestPayloadWithStringData
-	if err := c.ShouldBindJSON(&payload); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// doc to update
-	var doc models.User
-	if err := json.Unmarshal([]byte(payload.Data), &doc); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// query
-	query := bson.M{
-		"_id": bson.M{
-			"$in": payload.Ids,
-		},
-	}
-
-	// update users
-	if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdUser).UpdateDoc(query, &doc, payload.Fields); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// update passwords
-	if utils.Contains(payload.Fields, "password") {
-		for _, id := range payload.Ids {
-			if err := ctr.ctx.userSvc.ChangePassword(id, doc.Password); err != nil {
-				trace.PrintError(err)
-			}
-		}
-	}
-
-	HandleSuccess(c)
-}
-
-type userContext struct {
-	modelSvc service.ModelService
-	userSvc  interfaces.UserService
-}
-
-func (ctx *userContext) changePassword(c *gin.Context) {
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	var payload map[string]string
-	if err := c.ShouldBindJSON(&payload); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	password, ok := payload["password"]
-	if !ok {
-		HandleErrorBadRequest(c, errors.ErrorUserMissingRequiredFields)
-		return
-	}
-	if len(password) < 5 {
-		HandleErrorBadRequest(c, errors.ErrorUserInvalidPassword)
-		return
-	}
-	if err := ctx.userSvc.ChangePassword(id, password); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	HandleSuccess(c)
-}
-
-func (ctx *userContext) getMe(c *gin.Context) {
-	u, err := ctx._getMe(c)
-	if err != nil {
-		HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized)
-		return
-	}
-	HandleSuccessWithData(c, u)
-}
-
-func (ctx *userContext) putMe(c *gin.Context) {
-	// current user
-	u, err := ctx._getMe(c)
-	if err != nil {
-		HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized)
-		return
-	}
-
-	// payload
-	doc, err := NewJsonBinder(ControllerIdUser).Bind(c)
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-	if doc.GetId() != u.GetId() {
-		HandleErrorBadRequest(c, errors.ErrorHttpBadRequest)
-		return
-	}
-
-	// save to db
-	if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Save(); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	HandleSuccessWithData(c, doc)
-}
-
-func (ctx *userContext) _getMe(c *gin.Context) (u interfaces.User, err error) {
-	res, ok := c.Get(constants.UserContextKey)
-	if !ok {
-		return nil, trace.TraceError(errors.ErrorUserNotExistsInContext)
-	}
-	u, ok = res.(interfaces.User)
-	if !ok {
-		return nil, trace.TraceError(errors.ErrorUserInvalidType)
-	}
-	return u, nil
-}
-
-func newUserContext() *userContext {
-	// context
-	ctx := &userContext{}
-
-	// dependency injection
-	if err := container.GetContainer().Invoke(func(
-		modelSvc service.ModelService,
-		userSvc interfaces.UserService,
-	) {
-		ctx.modelSvc = modelSvc
-		ctx.userSvc = userSvc
-	}); err != nil {
-		panic(err)
-	}
-
-	return ctx
-}
-
-func newUserController() *userController {
-	modelSvc, err := service.GetService()
-	if err != nil {
-		panic(err)
-	}
-
-	ctr := NewListPostActionControllerDelegate(ControllerIdUser, modelSvc.GetBaseService(interfaces.ModelIdUser), getUserActions())
-	d := NewListPostActionControllerDelegate(ControllerIdUser, modelSvc.GetBaseService(interfaces.ModelIdUser), getUserActions())
-	ctx := newUserContext()
-
-	return &userController{
-		ListActionControllerDelegate: *ctr,
-		d:                            *d,
-		ctx:                          ctx,
-	}
-}

core/controllers/version.go

@@ -1,23 +0,0 @@
-package controllers
-
-import (
-	"github.com/crawlab-team/crawlab/core/config"
-	"github.com/gin-gonic/gin"
-	"net/http"
-)
-
-func GetVersion(c *gin.Context) {
-	HandleSuccessWithData(c, config.GetVersion())
-}
-
-func getVersionActions() []Action {
-	return []Action{
-		{
-			Method:      http.MethodGet,
-			Path:        "",
-			HandlerFunc: GetVersion,
-		},
-	}
-}
-
-var VersionController ActionController

core/routes/group.go

@@ -1,20 +0,0 @@
-package routes
-
-import (
-	"github.com/crawlab-team/crawlab/core/middlewares"
-	"github.com/gin-gonic/gin"
-)
-
-type RouterGroups struct {
-	AuthGroup      *gin.RouterGroup
-	AnonymousGroup *gin.RouterGroup
-	FilerGroup     *gin.RouterGroup
-}
-
-func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) {
-	return &RouterGroups{
-		AuthGroup:      app.Group("/", middlewares.AuthorizationMiddleware()),
-		AnonymousGroup: app.Group("/"),
-		FilerGroup:     app.Group("/filer", middlewares.FilerAuthorizationMiddleware()),
-	}
-}

core/routes/router.go

@@ -1,178 +0,0 @@
-package routes
-
-import (
-	"fmt"
-	"github.com/apex/log"
-	"github.com/crawlab-team/crawlab/core/controllers"
-	"github.com/gin-gonic/gin"
-	"net/http"
-	"path"
-)
-
-type RouterServiceInterface interface {
-	RegisterControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListController)
-	RegisterHandlerToGroup(group *gin.RouterGroup, path string, method string, handler gin.HandlerFunc)
-}
-
-type RouterService struct {
-	app *gin.Engine
-}
-
-func NewRouterService(app *gin.Engine) (svc *RouterService) {
-	return &RouterService{
-		app: app,
-	}
-}
-
-func (svc *RouterService) RegisterControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.BasicController) {
-	group.GET(basePath, ctr.Get)
-	group.POST(basePath, ctr.Post)
-	group.PUT(basePath, ctr.Put)
-	group.DELETE(basePath, ctr.Delete)
-}
-
-func (svc *RouterService) RegisterListControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListController) {
-	group.GET(basePath+"/:id", ctr.Get)
-	group.GET(basePath, ctr.GetList)
-	group.POST(basePath, ctr.Post)
-	group.POST(basePath+"/batch", ctr.PostList)
-	group.PUT(basePath+"/:id", ctr.Put)
-	group.PUT(basePath, ctr.PutList)
-	group.DELETE(basePath+"/:id", ctr.Delete)
-	group.DELETE(basePath, ctr.DeleteList)
-}
-
-func (svc *RouterService) RegisterActionControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ActionController) {
-	for _, action := range ctr.Actions() {
-		routerPath := path.Join(basePath, action.Path)
-		switch action.Method {
-		case http.MethodGet:
-			group.GET(routerPath, action.HandlerFunc)
-		case http.MethodPost:
-			group.POST(routerPath, action.HandlerFunc)
-		case http.MethodPut:
-			group.PUT(routerPath, action.HandlerFunc)
-		case http.MethodDelete:
-			group.DELETE(routerPath, action.HandlerFunc)
-		}
-	}
-}
-
-func (svc *RouterService) RegisterListActionControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListActionController) {
-	svc.RegisterListControllerToGroup(group, basePath, ctr)
-	svc.RegisterActionControllerToGroup(group, basePath, ctr)
-}
-
-func (svc *RouterService) RegisterHandlerToGroup(group *gin.RouterGroup, path string, method string, handler gin.HandlerFunc) {
-	switch method {
-	case http.MethodGet:
-		group.GET(path, handler)
-	case http.MethodPost:
-		group.POST(path, handler)
-	case http.MethodPut:
-		group.PUT(path, handler)
-	case http.MethodDelete:
-		group.DELETE(path, handler)
-	default:
-		log.Warn(fmt.Sprintf("%s is not a valid http method", method))
-	}
-}
-
-func InitRoutes(app *gin.Engine) (err error) {
-	// routes groups
-	groups := NewRouterGroups(app)
-
-	// router service
-	svc := NewRouterService(app)
-
-	// register routes
-	registerRoutesAnonymousGroup(svc, groups)
-	registerRoutesAuthGroup(svc, groups)
-	registerRoutesFilterGroup(svc, groups)
-
-	return nil
-}
-
-func registerRoutesAnonymousGroup(svc *RouterService, groups *RouterGroups) {
-	// login
-	svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/", controllers.LoginController)
-
-	// version
-	svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/version", controllers.VersionController)
-
-	// system info
-	svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/system-info", controllers.SystemInfoController)
-
-	// demo
-	svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/demo", controllers.DemoController)
-
-	// sync
-	svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/sync", controllers.SyncController)
-}
-
-func registerRoutesAuthGroup(svc *RouterService, groups *RouterGroups) {
-	// node
-	svc.RegisterListControllerToGroup(groups.AuthGroup, "/nodes", controllers.NodeController)
-
-	// project
-	svc.RegisterListControllerToGroup(groups.AuthGroup, "/projects", controllers.ProjectController)
-
-	// user
-	svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/users", controllers.UserController)
-
-	// spider
-	svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/spiders", controllers.SpiderController)
-
-	// task
-	svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/tasks", controllers.TaskController)
-
-	// tag
-	svc.RegisterListControllerToGroup(groups.AuthGroup, "/tags", controllers.TagController)
-
-	// setting
-	svc.RegisterListControllerToGroup(groups.AuthGroup, "/settings", controllers.SettingController)
-
-	// data collection
-	svc.RegisterListControllerToGroup(groups.AuthGroup, "/data/collections", controllers.DataCollectionController)
-
-	// result
-	svc.RegisterActionControllerToGroup(groups.AuthGroup, "/results", controllers.ResultController)
-
-	// schedule
-	svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/schedules", controllers.ScheduleController)
-
-	// stats
-	svc.RegisterActionControllerToGroup(groups.AuthGroup, "/stats", controllers.StatsController)
-
-	// token
-	svc.RegisterListControllerToGroup(groups.AuthGroup, "/tokens", controllers.TokenController)
-
-	// git
-	svc.RegisterListControllerToGroup(groups.AuthGroup, "/gits", controllers.GitController)
-
-	// role
-	svc.RegisterListControllerToGroup(groups.AuthGroup, "/roles", controllers.RoleController)
-
-	// permission
-	svc.RegisterListControllerToGroup(groups.AuthGroup, "/permissions", controllers.PermissionController)
-
-	// export
-	svc.RegisterActionControllerToGroup(groups.AuthGroup, "/export", controllers.ExportController)
-
-	// notification
-	svc.RegisterActionControllerToGroup(groups.AuthGroup, "/notifications", controllers.NotificationController)
-
-	// filter
-	svc.RegisterActionControllerToGroup(groups.AuthGroup, "/filters", controllers.FilterController)
-
-	// data sources
-	svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/data-sources", controllers.DataSourceController)
-
-	// environments
-	svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/environments", controllers.EnvironmentController)
-}
-
-func registerRoutesFilterGroup(svc *RouterService, groups *RouterGroups) {
-	// filer
-	svc.RegisterActionControllerToGroup(groups.FilerGroup, "", controllers.FilerController)
-}

core/routes/router_test.go

@@ -1,26 +0,0 @@
-package routes
-
-import (
-	"github.com/gin-gonic/gin"
-	"github.com/stretchr/testify/require"
-	"net/http"
-	"testing"
-	"time"
-)
-
-func TestInitRoutes(t *testing.T) {
-	app := gin.New()
-	err := InitRoutes(app)
-	require.Nil(t, err)
-
-	srv := &http.Server{
-		Handler: app,
-		Addr:    "localhost:8000",
-	}
-	go func() {
-		err = srv.ListenAndServe()
-		require.Nil(t, err)
-	}()
-
-	time.Sleep(5 * time.Second)
-}