diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 24cc31c5..00000000 --- a/Jenkinsfile +++ /dev/null @@ -1,56 +0,0 @@ -pipeline { - agent { - node { - label 'crawlab' - } - } - - stages { - stage('Setup') { - steps { - echo "Running Setup..." - script { - if (env.GIT_BRANCH == 'develop') { - env.TAG = 'develop' - env.DOCKERFILE = 'Dockerfile.local' - } else if (env.GIT_BRANCH == 'master') { - env.TAG = 'master' - env.DOCKERFILE = 'Dockerfile.local' - } - } - } - } - stage('Build') { - steps { - echo "Building..." - sh """ - docker build -t tikazyq/crawlab:${ENV:TAG} -f ${ENV:DOCKERFILE} . - """ - } - } - stage('Test') { - steps { - echo 'Testing..' - } - } - stage('Deploy') { - steps { - echo 'Deploying....' - sh """ - # 重启docker compose - cd ./jenkins/${ENV:GIT_BRANCH} - docker-compose down | true - docker-compose up -d | true - """ - } - } - stage('Cleanup') { - steps { - echo 'Cleanup...' - sh """ - docker rmi -f `docker images | grep '' | grep -v IMAGE | awk '{ print \$3 }' | xargs` - """ - } - } - } -} \ No newline at end of file diff --git a/core/apps/api.go b/core/apps/api.go deleted file mode 100644 index 94e887c3..00000000 --- a/core/apps/api.go +++ /dev/null @@ -1,126 +0,0 @@ -package apps - -import ( - "context" - "errors" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/controllers" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/routes" - "github.com/gin-gonic/gin" - "github.com/spf13/viper" - "net" - "net/http" - "time" -) - -func init() { - // set gin mode - if viper.GetString("gin.mode") == "" { - gin.SetMode(gin.ReleaseMode) - } else { - gin.SetMode(viper.GetString("gin.mode")) - } -} - -type Api struct { - // dependencies - interfaces.WithConfigPath - - // internals - app *gin.Engine - ln net.Listener - srv *http.Server - ready bool -} - -func (app *Api) Init() { - // initialize controllers - _ = initModule("controllers", controllers.InitControllers) - - // initialize middlewares - _ = app.initModuleWithApp("middlewares", middlewares.InitMiddlewares) - - // initialize routes - _ = app.initModuleWithApp("routes", routes.InitRoutes) -} - -func (app *Api) Start() { - // address - host := viper.GetString("server.host") - port := viper.GetString("server.port") - address := net.JoinHostPort(host, port) - - // http server - app.srv = &http.Server{ - Handler: app.app, - Addr: address, - } - - // listen - var err error - app.ln, err = net.Listen("tcp", address) - if err != nil { - panic(err) - } - app.ready = true - - // serve - if err := http.Serve(app.ln, app.app); err != nil { - if !errors.Is(err, http.ErrServerClosed) { - log.Error("run server error:" + err.Error()) - } else { - log.Info("server graceful down") - } - } -} - -func (app *Api) Wait() { - DefaultWait() -} - -func (app *Api) Stop() { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - if err := app.srv.Shutdown(ctx); err != nil { - log.Error("run server error:" + err.Error()) - } -} - -func (app *Api) GetGinEngine() *gin.Engine { - return app.app -} - -func (app *Api) GetHttpServer() *http.Server { - return app.srv -} - -func (app *Api) Ready() (ok bool) { - return app.ready -} - -func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) { - return initModule(name, func() error { - return fn(app.app) - }) -} - -func NewApi() *Api { - api := &Api{ - app: gin.New(), - } - api.Init() - return api -} - -var api *Api - -func GetApi() *Api { - if api != nil { - return api - } - api = NewApi() - return api -} diff --git a/core/cmd/server.go b/core/cmd/server.go index 40f976ae..afa7cd42 100644 --- a/core/cmd/server.go +++ b/core/cmd/server.go @@ -16,7 +16,6 @@ var serverCmd = &cobra.Command{ Long: `Start Crawlab node server that can serve as API, task scheduler, task runner, etc.`, Run: func(cmd *cobra.Command, args []string) { // app - //svr := apps.GetServer(opts...) svr := apps.GetServerV2() // start diff --git a/core/config/default_config.go b/core/config/default_config.go index 8f0e7ea7..e2806e67 100644 --- a/core/config/default_config.go +++ b/core/config/default_config.go @@ -1,9 +1,8 @@ package config var DefaultConfigYaml = ` -info: - version: v0.6.3 - edition: global.edition.community +version: v0.6.3 +edition: global.edition.community mongo: host: localhost port: 27017 diff --git a/core/controllers/data_collection.go b/core/controllers/data_collection.go deleted file mode 100644 index cf9ef9a9..00000000 --- a/core/controllers/data_collection.go +++ /dev/null @@ -1,103 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" -) - -var DataCollectionController *dataCollectionController - -func getDataCollectionActions() []Action { - ctx := newDataCollectionContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/:id/indexes", - HandlerFunc: ctx.postIndexes, - }, - } -} - -type dataCollectionController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *dataCollectionContext -} - -type dataCollectionContext struct { - modelSvc service.ModelService - resultSvc interfaces.ResultService -} - -func (ctx *dataCollectionContext) postIndexes(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - dc, err := ctx.modelSvc.GetDataCollectionById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - for _, f := range dc.Fields { - if err := mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{ - Keys: f.Key, - }); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - HandleSuccess(c) -} - -var _dataCollectionCtx *dataCollectionContext - -func newDataCollectionContext() *dataCollectionContext { - if _dataCollectionCtx != nil { - return _dataCollectionCtx - } - - // context - ctx := &dataCollectionContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - ) { - ctx.modelSvc = modelSvc - }); err != nil { - panic(err) - } - - _dataCollectionCtx = ctx - - return ctx -} - -func newDataCollectionController() *dataCollectionController { - actions := getDataCollectionActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions) - d := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions) - ctx := newDataCollectionContext() - - return &dataCollectionController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/data_source.go b/core/controllers/data_source.go deleted file mode 100644 index a4b90565..00000000 --- a/core/controllers/data_source.go +++ /dev/null @@ -1,148 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/ds" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - interfaces2 "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" -) - -var DataSourceController *dataSourceController - -func getDataSourceActions() []Action { - ctx := newDataSourceContext() - return []Action{ - { - Path: "/:id/change-password", - Method: http.MethodPost, - HandlerFunc: ctx.changePassword, - }, - } -} - -type dataSourceController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *dataSourceContext -} - -func (ctr *dataSourceController) Post(c *gin.Context) { - // data source - var _ds models.DataSource - if err := c.ShouldBindJSON(&_ds); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // add data source to db - if err := mongo.RunTransaction(func(ctx mongo2.SessionContext) error { - if err := delegate.NewModelDelegate(&_ds).Add(); err != nil { - return trace.TraceError(err) - } - pwd, err := utils.EncryptAES(_ds.Password) - if err != nil { - return trace.TraceError(err) - } - p := models.Password{Id: _ds.Id, Password: pwd} - if err := delegate.NewModelDelegate(&p).Add(); err != nil { - return trace.TraceError(err) - } - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // check data source status - go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }() - - HandleSuccess(c) -} - -func (ctr *dataSourceController) Put(c *gin.Context) { - // data source - var _ds models.DataSource - if err := c.ShouldBindJSON(&_ds); err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := delegate.NewModelDelegate(&_ds).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // check data source status - go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }() -} - -type dataSourceContext struct { - dsSvc interfaces.DataSourceService -} - -var _dataSourceCtx *dataSourceContext - -func newDataSourceContext() *dataSourceContext { - if _dataSourceCtx != nil { - return _dataSourceCtx - } - dsSvc, err := ds.GetDataSourceService() - if err != nil { - panic(err) - } - _dataSourceCtx = &dataSourceContext{ - dsSvc: dsSvc, - } - return _dataSourceCtx -} - -func (ctx *dataSourceContext) changePassword(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - var payload map[string]string - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - password, ok := payload["password"] - if !ok { - HandleErrorBadRequest(c, errors.ErrorDataSourceMissingRequiredFields) - return - } - if err := ctx.dsSvc.ChangePassword(id, password); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func newDataSourceController() *dataSourceController { - actions := getDataSourceActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions) - d := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions) - ctx := newDataSourceContext() - - return &dataSourceController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/demo.go b/core/controllers/demo.go deleted file mode 100644 index 7a09d988..00000000 --- a/core/controllers/demo.go +++ /dev/null @@ -1,73 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "net/http" -) - -func getDemoActions() []Action { - ctx := newDemoContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/import", - HandlerFunc: ctx.import_, - }, - { - Method: http.MethodGet, - Path: "/reimport", - HandlerFunc: ctx.reimport, - }, - { - Method: http.MethodGet, - Path: "/cleanup", - HandlerFunc: ctx.cleanup, - }, - } -} - -type demoContext struct { -} - -func (ctx *demoContext) import_(c *gin.Context) { - if err := utils.ImportDemo(); err != nil { - trace.PrintError(err) - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *demoContext) reimport(c *gin.Context) { - if err := utils.ReimportDemo(); err != nil { - trace.PrintError(err) - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *demoContext) cleanup(c *gin.Context) { - if err := utils.ReimportDemo(); err != nil { - trace.PrintError(err) - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -var _demoCtx *demoContext - -func newDemoContext() *demoContext { - if _demoCtx != nil { - return _demoCtx - } - - _demoCtx = &demoContext{} - - return _demoCtx -} - -var DemoController ActionController diff --git a/core/controllers/export.go b/core/controllers/export.go deleted file mode 100644 index b99ffcb9..00000000 --- a/core/controllers/export.go +++ /dev/null @@ -1,127 +0,0 @@ -package controllers - -import ( - "errors" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/export" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/gin-gonic/gin" - "net/http" -) - -var ExportController ActionController - -func getExportActions() []Action { - ctx := newExportContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/:type", - HandlerFunc: ctx.postExport, - }, - { - Method: http.MethodGet, - Path: "/:type/:id", - HandlerFunc: ctx.getExport, - }, - { - Method: http.MethodGet, - Path: "/:type/:id/download", - HandlerFunc: ctx.getExportDownload, - }, - } -} - -type exportContext struct { - csvSvc interfaces.ExportService - jsonSvc interfaces.ExportService -} - -func (ctx *exportContext) postExport(c *gin.Context) { - exportType := c.Param("type") - exportTarget := c.Query("target") - exportFilter, _ := GetFilter(c) - - var exportId string - var err error - switch exportType { - case constants.ExportTypeCsv: - exportId, err = ctx.csvSvc.Export(exportType, exportTarget, exportFilter) - case constants.ExportTypeJson: - exportId, err = ctx.jsonSvc.Export(exportType, exportTarget, exportFilter) - default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - return - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, exportId) -} - -func (ctx *exportContext) getExport(c *gin.Context) { - exportType := c.Param("type") - exportId := c.Param("id") - - var exp interfaces.Export - var err error - switch exportType { - case constants.ExportTypeCsv: - exp, err = ctx.csvSvc.GetExport(exportId) - case constants.ExportTypeJson: - exp, err = ctx.jsonSvc.GetExport(exportId) - default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, exp) -} - -func (ctx *exportContext) getExportDownload(c *gin.Context) { - exportType := c.Param("type") - exportId := c.Param("id") - - var exp interfaces.Export - var err error - switch exportType { - case constants.ExportTypeCsv: - exp, err = ctx.csvSvc.GetExport(exportId) - case constants.ExportTypeJson: - exp, err = ctx.jsonSvc.GetExport(exportId) - default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - switch exportType { - case constants.ExportTypeCsv: - c.Header("Content-Type", "text/csv") - case constants.ExportTypeJson: - c.Header("Content-Type", "text/plain") - default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", exp.GetDownloadPath())) - c.File(exp.GetDownloadPath()) -} - -func newExportContext() *exportContext { - return &exportContext{ - csvSvc: export.GetCsvService(), - jsonSvc: export.GetJsonService(), - } -} diff --git a/core/controllers/filer.go b/core/controllers/filer.go deleted file mode 100644 index 4af5d142..00000000 --- a/core/controllers/filer.go +++ /dev/null @@ -1,130 +0,0 @@ -package controllers - -import ( - "bufio" - "fmt" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/gin-gonic/gin" - "github.com/imroc/req" - "github.com/spf13/viper" - "net/http" - "strings" -) - -var FilerController ActionController - -func getFilerActions() []Action { - filerCtx := newFilerContext() - return []Action{ - { - Method: http.MethodGet, - Path: "*path", - HandlerFunc: filerCtx.do, - }, - { - Method: http.MethodPost, - Path: "*path", - HandlerFunc: filerCtx.do, - }, - { - Method: http.MethodPut, - Path: "*path", - HandlerFunc: filerCtx.do, - }, - { - Method: http.MethodDelete, - Path: "*path", - HandlerFunc: filerCtx.do, - }, - } -} - -type filerContext struct { - endpoint string -} - -func (ctx *filerContext) do(c *gin.Context) { - // request path - requestPath := strings.Replace(c.Request.URL.Path, "/filer", "", 1) - - // request url - requestUrl := fmt.Sprintf("%s%s", ctx.endpoint, requestPath) - if c.Request.URL.RawQuery != "" { - requestUrl += "?" + c.Request.URL.RawQuery - } - - // request body - bufR := bufio.NewScanner(c.Request.Body) - requestBody := req.BodyJSON(bufR.Bytes()) - - // request file uploads - var requestFileUploads []req.FileUpload - form, err := c.MultipartForm() - if err == nil { - for k, v := range form.File { - for _, fh := range v { - f, err := fh.Open() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - requestFileUploads = append(requestFileUploads, req.FileUpload{ - FileName: fh.Filename, - FieldName: k, - File: f, - }) - } - } - } - - // request header - requestHeader := req.Header{} - for k, v := range c.Request.Header { - if len(v) > 0 { - requestHeader[k] = v[0] - } - } - - // perform request - res, err := req.Do(c.Request.Method, requestUrl, requestHeader, requestBody, requestFileUploads) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // status code check - statusCode := res.Response().StatusCode - if statusCode == http.StatusNotFound { - HandleErrorNotFoundNoPrint(c, errors.ErrorControllerFilerNotFound) - return - } - - // response - for k, v := range res.Response().Header { - if len(v) > 0 { - c.Header(k, v[0]) - } - } - _, _ = c.Writer.Write(res.Bytes()) - c.AbortWithStatus(statusCode) -} - -var _filerCtx *filerContext - -func newFilerContext() *filerContext { - if _filerCtx != nil { - return _filerCtx - } - - ctx := &filerContext{ - endpoint: "http://localhost:8888", - } - - if viper.GetString("fs.filer.proxy") != "" { - ctx.endpoint = viper.GetString("fs.filer.proxy") - } - - _filerCtx = ctx - - return ctx -} diff --git a/core/controllers/filter.go b/core/controllers/filter.go deleted file mode 100644 index b609f6e0..00000000 --- a/core/controllers/filter.go +++ /dev/null @@ -1,100 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" -) - -var FilterController ActionController - -func getFilterActions() []Action { - ctx := newFilterContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/:col", - HandlerFunc: ctx.getColFieldOptions, - }, - { - Method: http.MethodGet, - Path: "/:col/:value", - HandlerFunc: ctx.getColFieldOptions, - }, - { - Method: http.MethodGet, - Path: "/:col/:value/:label", - HandlerFunc: ctx.getColFieldOptions, - }, - } -} - -type filterContext struct { -} - -func (ctx *filterContext) getColFieldOptions(c *gin.Context) { - colName := c.Param("col") - value := c.Param("value") - if value == "" { - value = "_id" - } - label := c.Param("label") - if label == "" { - label = "name" - } - query := MustGetFilterQuery(c) - pipelines := mongo2.Pipeline{} - if query != nil { - pipelines = append(pipelines, bson.D{{"$match", query}}) - } - pipelines = append( - pipelines, - bson.D{ - { - "$group", - bson.M{ - "_id": bson.M{ - "value": "$" + value, - "label": "$" + label, - }, - }, - }, - }, - ) - pipelines = append( - pipelines, - bson.D{ - { - "$project", - bson.M{ - "value": "$_id.value", - "label": bson.M{"$toString": "$_id.label"}, - }, - }, - }, - ) - pipelines = append( - pipelines, - bson.D{ - { - "$sort", - bson.D{ - {"value", 1}, - }, - }, - }, - ) - var options []entity.FilterSelectOption - if err := mongo.GetMongoCol(colName).Aggregate(pipelines, nil).All(&options); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, options) -} - -func newFilterContext() *filterContext { - return &filterContext{} -} diff --git a/core/controllers/git.go b/core/controllers/git.go deleted file mode 100644 index 128323d4..00000000 --- a/core/controllers/git.go +++ /dev/null @@ -1,3 +0,0 @@ -package controllers - -var GitController ListController diff --git a/core/controllers/init.go b/core/controllers/init.go deleted file mode 100644 index e927b9c3..00000000 --- a/core/controllers/init.go +++ /dev/null @@ -1,42 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" -) - -func InitControllers() (err error) { - modelSvc, err := service.GetService() - if err != nil { - return err - } - - NodeController = newNodeController() - ProjectController = newProjectController() - SpiderController = newSpiderController() - TaskController = newTaskController() - UserController = newUserController() - TagController = NewListControllerDelegate(ControllerIdTag, modelSvc.GetBaseService(interfaces.ModelIdTag)) - SettingController = newSettingController() - LoginController = NewActionControllerDelegate(ControllerIdLogin, getLoginActions()) - DataCollectionController = newDataCollectionController() - ResultController = NewActionControllerDelegate(ControllerIdResult, getResultActions()) - ScheduleController = newScheduleController() - StatsController = NewActionControllerDelegate(ControllerIdStats, getStatsActions()) - TokenController = newTokenController() - FilerController = NewActionControllerDelegate(ControllerIdFiler, getFilerActions()) - GitController = NewListControllerDelegate(ControllerIdGit, modelSvc.GetBaseService(interfaces.ModelIdGit)) - VersionController = NewActionControllerDelegate(ControllerIdVersion, getVersionActions()) - SystemInfoController = NewActionControllerDelegate(ControllerIdSystemInfo, getSystemInfoActions()) - DemoController = NewActionControllerDelegate(ControllerIdDemo, getDemoActions()) - RoleController = NewListControllerDelegate(ControllerIdRole, modelSvc.GetBaseService(interfaces.ModelIdRole)) - PermissionController = NewListControllerDelegate(ControllerIdPermission, modelSvc.GetBaseService(interfaces.ModelIdPermission)) - ExportController = NewActionControllerDelegate(ControllerIdExport, getExportActions()) - NotificationController = NewActionControllerDelegate(ControllerIdNotification, getNotificationActions()) - FilterController = NewActionControllerDelegate(ControllerIdFilter, getFilterActions()) - SyncController = NewActionControllerDelegate(ControllerIdSync, getSyncActions()) - DataSourceController = newDataSourceController() - EnvironmentController = newEnvironmentController() - - return nil -} diff --git a/core/controllers/login.go b/core/controllers/login.go deleted file mode 100644 index 42926fd5..00000000 --- a/core/controllers/login.go +++ /dev/null @@ -1,64 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/gin-gonic/gin" - "net/http" -) - -var LoginController ActionController - -func getLoginActions() []Action { - loginCtx := newLoginContext() - return []Action{ - {Method: http.MethodPost, Path: "/login", HandlerFunc: loginCtx.login}, - {Method: http.MethodPost, Path: "/logout", HandlerFunc: loginCtx.logout}, - } -} - -type loginContext struct { - userSvc interfaces.UserService -} - -func (ctx *loginContext) login(c *gin.Context) { - var u models.User - if err := c.ShouldBindJSON(&u); err != nil { - HandleErrorBadRequest(c, err) - return - } - token, loggedInUser, err := ctx.userSvc.Login(&interfaces.UserLoginOptions{ - Username: u.Username, - Password: u.Password, - }) - if err != nil { - HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) - return - } - c.Set(constants.UserContextKey, loggedInUser) - HandleSuccessWithData(c, token) -} - -func (ctx *loginContext) logout(c *gin.Context) { - c.Set(constants.UserContextKey, nil) - HandleSuccess(c) -} - -func newLoginContext() *loginContext { - // context - ctx := &loginContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - userSvc interfaces.UserService, - ) { - ctx.userSvc = userSvc - }); err != nil { - panic(err) - } - - return ctx -} diff --git a/core/controllers/node.go b/core/controllers/node.go deleted file mode 100644 index 236ebafc..00000000 --- a/core/controllers/node.go +++ /dev/null @@ -1,94 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "github.com/google/uuid" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -var NodeController *nodeController - -type nodeController struct { - ListControllerDelegate -} - -func (ctr *nodeController) Post(c *gin.Context) { - var n models.Node - if err := c.ShouldBindJSON(&n); err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := ctr._post(c, &n); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctr *nodeController) PostList(c *gin.Context) { - // bind - var docs []models.Node - if err := c.ShouldBindJSON(&docs); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // success ids - var ids []primitive.ObjectID - - // iterate nodes - for _, n := range docs { - if err := ctr._post(c, &n); err != nil { - trace.PrintError(err) - continue - } - ids = append(ids, n.Id) - } - - // success - HandleSuccessWithData(c, docs) -} - -func (ctr *nodeController) _post(c *gin.Context, n *models.Node) (err error) { - // set default key - if n.Key == "" { - id, err := uuid.NewUUID() - if err != nil { - return trace.TraceError(err) - } - n.Key = id.String() - } - - // set default status - if n.Status == "" { - n.Status = constants.NodeStatusUnregistered - } - - // add - if err := delegate.NewModelDelegate(n, GetUserFromContext(c)).Add(); err != nil { - return trace.TraceError(err) - } - - return nil -} - -func newNodeController() *nodeController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListControllerDelegate(ControllerIdNode, modelSvc.GetBaseService(interfaces.ModelIdNode)) - - return &nodeController{ - ListControllerDelegate: *ctr, - } -} diff --git a/core/controllers/notification.go b/core/controllers/notification.go deleted file mode 100644 index 31b96c04..00000000 --- a/core/controllers/notification.go +++ /dev/null @@ -1,158 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/notification" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" -) - -var NotificationController ActionController - -func getNotificationActions() []Action { - ctx := newNotificationContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/settings", - HandlerFunc: ctx.GetSettingList, - }, - { - Method: http.MethodGet, - Path: "/settings/:id", - HandlerFunc: ctx.GetSetting, - }, - { - Method: http.MethodPost, - Path: "/settings", - HandlerFunc: ctx.PostSetting, - }, - { - Method: http.MethodPut, - Path: "/settings/:id", - HandlerFunc: ctx.PutSetting, - }, - { - Method: http.MethodDelete, - Path: "/settings/:id", - HandlerFunc: ctx.DeleteSetting, - }, - { - Method: http.MethodPost, - Path: "/settings/:id/enable", - HandlerFunc: ctx.EnableSetting, - }, - { - Method: http.MethodPost, - Path: "/settings/:id/disable", - HandlerFunc: ctx.DisableSetting, - }, - } -} - -type notificationContext struct { - svc *notification.Service -} - -func (ctx *notificationContext) GetSettingList(c *gin.Context) { - query := MustGetFilterQuery(c) - pagination := MustGetPagination(c) - sort := MustGetSortOption(c) - res, total, err := ctx.svc.GetSettingList(query, pagination, sort) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithListData(c, res, total) -} - -func (ctx *notificationContext) GetSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - res, err := ctx.svc.GetSetting(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, res) -} - -func (ctx *notificationContext) PostSetting(c *gin.Context) { - var s notification.NotificationSetting - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.PosSetting(&s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *notificationContext) PutSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - var s notification.NotificationSetting - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.PutSetting(id, s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *notificationContext) DeleteSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.DeleteSetting(id); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *notificationContext) EnableSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.EnableSetting(id); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *notificationContext) DisableSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.DisableSetting(id); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func newNotificationContext() *notificationContext { - ctx := ¬ificationContext{ - svc: notification.GetService(), - } - return ctx -} diff --git a/core/controllers/permission.go b/core/controllers/permission.go deleted file mode 100644 index 42be45dc..00000000 --- a/core/controllers/permission.go +++ /dev/null @@ -1,3 +0,0 @@ -package controllers - -var PermissionController ListController diff --git a/core/controllers/project.go b/core/controllers/project.go deleted file mode 100644 index fe13a9e8..00000000 --- a/core/controllers/project.go +++ /dev/null @@ -1,103 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -var ProjectController *projectController - -type projectController struct { - ListControllerDelegate -} - -func (ctr *projectController) GetList(c *gin.Context) { - // get all if query field "all" is set true - all := MustGetFilterAll(c) - if all { - ctr.getAll(c) - return - } - - // get list - list, total, err := ctr.getList(c) - if err != nil { - return - } - data := list.GetModels() - - // check empty list - if len(list.GetModels()) == 0 { - HandleSuccessWithListData(c, nil, 0) - return - } - - // project ids - var ids []primitive.ObjectID - - // count cache - cache := map[primitive.ObjectID]int{} - - // iterate - for _, d := range data { - p, ok := d.(*models.Project) - if !ok { - HandleErrorInternalServerError(c, errors.ErrorControllerInvalidType) - return - } - ids = append(ids, p.Id) - cache[p.Id] = 0 - } - - // spiders - modelSvc, err := service.NewService() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - spiders, err := modelSvc.GetSpiderList(bson.M{ - "project_id": bson.M{ - "$in": ids, - }, - }, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - for _, s := range spiders { - _, ok := cache[s.ProjectId] - if !ok { - HandleErrorInternalServerError(c, errors.ErrorControllerMissingInCache) - return - } - cache[s.ProjectId]++ - } - - // assign - var projects []models.Project - for _, d := range data { - p := d.(*models.Project) - p.Spiders = cache[p.Id] - projects = append(projects, *p) - } - - HandleSuccessWithListData(c, projects, total) -} - -func newProjectController() *projectController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListControllerDelegate(ControllerIdProject, modelSvc.GetBaseService(interfaces.ModelIdProject)) - - return &projectController{ - ListControllerDelegate: *ctr, - } -} diff --git a/core/controllers/result.go b/core/controllers/result.go deleted file mode 100644 index 0324a417..00000000 --- a/core/controllers/result.go +++ /dev/null @@ -1,150 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/result" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" -) - -var ResultController ActionController - -func getResultActions() []Action { - var resultCtx = newResultContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/:id", - HandlerFunc: resultCtx.getList, - }, - } -} - -type resultContext struct { - modelSvc service.ModelService -} - -func (ctx *resultContext) getList(c *gin.Context) { - // data collection id - dcId, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // data source id - var dsId primitive.ObjectID - dsIdStr := c.Query("data_source_id") - if dsIdStr != "" { - dsId, err = primitive.ObjectIDFromHex(dsIdStr) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - } - - // data collection - dc, err := ctx.modelSvc.GetDataCollectionById(dcId) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // data source - ds, err := ctx.modelSvc.GetDataSourceById(dsId) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - ds = &models.DataSource{} - } else { - HandleErrorInternalServerError(c, err) - return - } - } - - // spider - sq := bson.M{ - "col_id": dc.Id, - "data_source_id": ds.Id, - } - s, err := ctx.modelSvc.GetSpider(sq, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // service - svc, err := result.GetResultService(s.Id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // params - pagination := MustGetPagination(c) - query := ctx.getListQuery(c) - - // get results - data, err := svc.List(query, &generic.ListOptions{ - Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}}, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, - }) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - HandleSuccessWithListData(c, nil, 0) - return - } - HandleErrorInternalServerError(c, err) - return - } - - // validate results - if len(data) == 0 { - HandleSuccessWithListData(c, nil, 0) - return - } - - // total count - total, err := svc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // response - HandleSuccessWithListData(c, data, total) -} - -func (ctx *resultContext) getListQuery(c *gin.Context) (q generic.ListQuery) { - f, err := GetFilter(c) - if err != nil { - return q - } - for _, cond := range f.Conditions { - q = append(q, generic.ListQueryCondition{ - Key: cond.Key, - Op: cond.Op, - Value: utils.NormalizeObjectId(cond.Value), - }) - } - return q -} - -func newResultContext() *resultContext { - // context - ctx := &resultContext{} - - var err error - ctx.modelSvc, err = service.NewService() - if err != nil { - panic(err) - } - - return ctx -} diff --git a/core/controllers/role.go b/core/controllers/role.go deleted file mode 100644 index d4a286e5..00000000 --- a/core/controllers/role.go +++ /dev/null @@ -1,3 +0,0 @@ -package controllers - -var RoleController ListController diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index 02dfb3b5..809a2f79 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -357,13 +357,6 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetSystemInfo, }, }) - RegisterActions(groups.AnonymousGroup, "/version", []Action{ - { - Method: http.MethodGet, - Path: "", - HandlerFunc: GetVersion, - }, - }) RegisterActions(groups.AnonymousGroup, "/", []Action{ { Method: http.MethodPost, diff --git a/core/controllers/schedule.go b/core/controllers/schedule.go deleted file mode 100644 index 71ba2b67..00000000 --- a/core/controllers/schedule.go +++ /dev/null @@ -1,221 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" -) - -var ScheduleController *scheduleController - -func getScheduleActions() []Action { - scheduleCtx := newScheduleContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/:id/enable", - HandlerFunc: scheduleCtx.enable, - }, - { - Method: http.MethodPost, - Path: "/:id/disable", - HandlerFunc: scheduleCtx.disable, - }, - } -} - -type scheduleController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *scheduleContext -} - -func (ctr *scheduleController) Post(c *gin.Context) { - var s models.Schedule - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := delegate.NewModelDelegate(&s, GetUserFromContext(c)).Add(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - if s.Enabled { - if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - HandleSuccessWithData(c, s) -} - -func (ctr *scheduleController) Put(c *gin.Context) { - id := c.Param("id") - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - var s models.Schedule - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } - if s.GetId() != oid { - HandleErrorBadRequest(c, errors.ErrorHttpBadRequest) - return - } - if err := delegate.NewModelDelegate(&s).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - if s.Enabled { - if err := ctr.ctx.scheduleSvc.Disable(&s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - HandleSuccessWithData(c, s) -} - -func (ctr *scheduleController) Delete(c *gin.Context) { - id := c.Param("id") - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - s, err := ctr.ctx.modelSvc.GetScheduleById(oid) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := ctr.ctx.scheduleSvc.Disable(s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := delegate.NewModelDelegate(s, GetUserFromContext(c)).Delete(); err != nil { - HandleErrorInternalServerError(c, err) - return - } -} - -func (ctr *scheduleController) DeleteList(c *gin.Context) { - payload, err := NewJsonBinder(interfaces.ModelIdSchedule).BindBatchRequestPayload(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - for _, id := range payload.Ids { - s, err := ctr.ctx.modelSvc.GetScheduleById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := ctr.ctx.scheduleSvc.Disable(s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdSchedule).DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *scheduleContext) enable(c *gin.Context) { - s, err := ctx._getSchedule(c) - if err != nil { - return - } - if err := ctx.scheduleSvc.Enable(s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *scheduleContext) disable(c *gin.Context) { - s, err := ctx._getSchedule(c) - if err != nil { - return - } - if err := ctx.scheduleSvc.Disable(s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *scheduleContext) _getSchedule(c *gin.Context) (s *models.Schedule, err error) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - s, err = ctx.modelSvc.GetScheduleById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - return s, nil -} - -type scheduleContext struct { - modelSvc service.ModelService - scheduleSvc interfaces.ScheduleService -} - -func newScheduleContext() *scheduleContext { - // context - ctx := &scheduleContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - scheduleSvc interfaces.ScheduleService, - ) { - ctx.modelSvc = modelSvc - ctx.scheduleSvc = scheduleSvc - }); err != nil { - panic(err) - } - - return ctx -} - -func newScheduleController() *scheduleController { - actions := getScheduleActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions) - d := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions) - ctx := newScheduleContext() - - return &scheduleController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/setting.go b/core/controllers/setting.go deleted file mode 100644 index 068e2379..00000000 --- a/core/controllers/setting.go +++ /dev/null @@ -1,84 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/gin-gonic/gin" -) - -var SettingController *settingController - -type settingController struct { - ListControllerDelegate -} - -func (ctr *settingController) Get(c *gin.Context) { - // key - key := c.Param("id") - - // model service - modelSvc, err := service.NewService() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // setting - s, err := modelSvc.GetSettingByKey(key, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, s) -} - -func (ctr *settingController) Put(c *gin.Context) { - // key - key := c.Param("id") - - // settings - var s models.Setting - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // model service - modelSvc, err := service.NewService() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // setting - _s, err := modelSvc.GetSettingByKey(key, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // save - _s.Value = s.Value - if err := delegate.NewModelDelegate(_s).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func newSettingController() *settingController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListControllerDelegate(ControllerIdSetting, modelSvc.GetBaseService(interfaces.ModelIdSetting)) - - return &settingController{ - ListControllerDelegate: *ctr, - } -} diff --git a/core/controllers/spider.go b/core/controllers/spider.go deleted file mode 100644 index f05da372..00000000 --- a/core/controllers/spider.go +++ /dev/null @@ -1,1333 +0,0 @@ -package controllers - -import ( - "bytes" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - fs2 "github.com/crawlab-team/crawlab/core/fs" - "github.com/crawlab-team/crawlab/core/interfaces" - delegate2 "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "github.com/crawlab-team/crawlab/vcs" - "github.com/gin-gonic/gin" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/config" - "github.com/spf13/viper" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "io" - "math" - "net/http" - "os" - "path/filepath" - "strings" -) - -var SpiderController *spiderController - -func getSpiderActions() []Action { - ctx := newSpiderContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/:id/files/list", - HandlerFunc: ctx.listDir, - }, - { - Method: http.MethodGet, - Path: "/:id/files/get", - HandlerFunc: ctx.getFile, - }, - { - Method: http.MethodGet, - Path: "/:id/files/info", - HandlerFunc: ctx.getFileInfo, - }, - { - Method: http.MethodPost, - Path: "/:id/files/save", - HandlerFunc: ctx.saveFile, - }, - { - Method: http.MethodPost, - Path: "/:id/files/save/dir", - HandlerFunc: ctx.saveDir, - }, - { - Method: http.MethodPost, - Path: "/:id/files/rename", - HandlerFunc: ctx.renameFile, - }, - { - Method: http.MethodPost, - Path: "/:id/files/delete", - HandlerFunc: ctx.deleteFile, - }, - { - Method: http.MethodPost, - Path: "/:id/files/copy", - HandlerFunc: ctx.copyFile, - }, - { - Path: "/:id/files/export", - Method: http.MethodPost, - HandlerFunc: ctx.postExport, - }, - { - Method: http.MethodPost, - Path: "/:id/run", - HandlerFunc: ctx.run, - }, - { - Method: http.MethodGet, - Path: "/:id/git", - HandlerFunc: ctx.getGit, - }, - { - Method: http.MethodGet, - Path: "/:id/git/remote-refs", - HandlerFunc: ctx.getGitRemoteRefs, - }, - { - Method: http.MethodPost, - Path: "/:id/git/checkout", - HandlerFunc: ctx.gitCheckout, - }, - { - Method: http.MethodPost, - Path: "/:id/git/pull", - HandlerFunc: ctx.gitPull, - }, - { - Method: http.MethodPost, - Path: "/:id/git/commit", - HandlerFunc: ctx.gitCommit, - }, - //{ - // Method: http.MethodPost, - // Path: "/:id/clone", - // HandlerFunc: ctx.clone, - //}, - { - Path: "/:id/data-source", - Method: http.MethodGet, - HandlerFunc: ctx.getDataSource, - }, - { - Path: "/:id/data-source/:ds_id", - Method: http.MethodPost, - HandlerFunc: ctx.postDataSource, - }, - } -} - -type spiderController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *spiderContext -} - -func (ctr *spiderController) Get(c *gin.Context) { - ctr.ctx._get(c) -} - -func (ctr *spiderController) Post(c *gin.Context) { - s, err := ctr.ctx._post(c) - if err != nil { - return - } - HandleSuccessWithData(c, s) -} - -func (ctr *spiderController) Put(c *gin.Context) { - s, err := ctr.ctx._put(c) - if err != nil { - return - } - HandleSuccessWithData(c, s) -} - -func (ctr *spiderController) Delete(c *gin.Context) { - if err := ctr.ctx._delete(c); err != nil { - return - } - HandleSuccess(c) -} - -func (ctr *spiderController) GetList(c *gin.Context) { - withStats := c.Query("stats") - if withStats == "" { - ctr.d.GetList(c) - return - } - ctr.ctx._getListWithStats(c) -} - -func (ctr *spiderController) DeleteList(c *gin.Context) { - if err := ctr.ctx._deleteList(c); err != nil { - return - } - HandleSuccess(c) -} - -type spiderContext struct { - modelSvc service.ModelService - modelSpiderSvc interfaces.ModelBaseService - modelSpiderStatSvc interfaces.ModelBaseService - modelTaskSvc interfaces.ModelBaseService - modelTaskStatSvc interfaces.ModelBaseService - adminSvc interfaces.SpiderAdminService -} - -func (ctx *spiderContext) listDir(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodGet) - if err != nil { - return - } - - files, err := fsSvc.List(payload.Path) - if err != nil { - if err.Error() != "response status code: 404" { - HandleErrorInternalServerError(c, err) - return - } - } - - HandleSuccessWithData(c, files) -} - -func (ctx *spiderContext) getFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodGet) - if err != nil { - return - } - - data, err := fsSvc.GetFile(payload.Path) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - data = utils.TrimFileData(data) - - HandleSuccessWithData(c, string(data)) -} - -func (ctx *spiderContext) getFileInfo(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodGet) - if err != nil { - return - } - - info, err := fsSvc.GetFileInfo(payload.Path) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, info) -} - -func (ctx *spiderContext) saveFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.Save(payload.Path, []byte(payload.Data)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) saveDir(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.CreateDir(payload.Path); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) renameFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.Rename(payload.Path, payload.NewPath); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) deleteFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.Delete(payload.Path); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) copyFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.Copy(payload.Path, payload.NewPath); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) run(c *gin.Context) { - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // options - var opts interfaces.SpiderRunOptions - if err := c.ShouldBindJSON(&opts); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // user - if u := GetUserFromContext(c); u != nil { - opts.UserId = u.GetId() - } - - // schedule - taskIds, err := ctx.adminSvc.Schedule(id, &opts) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, taskIds) -} - -func (ctx *spiderContext) getGit(c *gin.Context) { - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // git client - gitClient, err := ctx._getGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // current branch - currentBranch, err := gitClient.GetCurrentBranch() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // branches - branches, err := gitClient.GetBranches() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if branches == nil || len(branches) == 0 && currentBranch != "" { - branches = []vcs.GitRef{{Name: currentBranch}} - } - - // changes - changes, err := gitClient.GetStatus() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // logs - logs, err := gitClient.GetLogsWithRefs() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // ignore - ignore, err := ctx._getGitIgnore(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // git - _git, err := ctx.modelSvc.GetGitById(id) - if err != nil { - if err.Error() != mongo2.ErrNoDocuments.Error() { - HandleErrorInternalServerError(c, err) - return - } - } - - // response - res := bson.M{ - "current_branch": currentBranch, - "branches": branches, - "changes": changes, - "logs": logs, - "ignore": ignore, - "git": _git, - } - - HandleSuccessWithData(c, res) -} - -func (ctx *spiderContext) getGitRemoteRefs(c *gin.Context) { - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // remote name - remoteName := c.Query("remote") - if remoteName == "" { - remoteName = vcs.GitRemoteNameOrigin - } - - // git client - gitClient, err := ctx._getGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // refs - refs, err := gitClient.GetRemoteRefs(remoteName) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, refs) -} - -func (ctx *spiderContext) gitCheckout(c *gin.Context) { - // payload - var payload entity.GitPayload - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // git client - gitClient, err := ctx._getGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // branch to pull - var branch string - if payload.Branch == "" { - // by default current branch - branch, err = gitClient.GetCurrentBranch() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - } else { - // payload branch - branch = payload.Branch - } - - // checkout - if err := ctx._gitCheckout(gitClient, constants.GitRemoteNameOrigin, branch); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) gitPull(c *gin.Context) { - // payload - var payload entity.GitPayload - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // git - g, err := ctx.modelSvc.GetGitById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // attempt to sync git - _ = ctx.adminSvc.SyncGitOne(g) - - HandleSuccess(c) -} - -func (ctx *spiderContext) gitCommit(c *gin.Context) { - // payload - var payload entity.GitPayload - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // git client - gitClient, err := ctx._getGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // add - for _, p := range payload.Paths { - if err := gitClient.Add(p); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - // commit - if err := gitClient.Commit(payload.CommitMessage); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // push - if err := gitClient.Push( - vcs.WithRemoteNamePush(vcs.GitRemoteNameOrigin), - ); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) getDataSource(c *gin.Context) { - // spider id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider - s, err := ctx.modelSvc.GetSpiderById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // data source - ds, err := ctx.modelSvc.GetDataSourceById(s.DataSourceId) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - HandleSuccess(c) - return - } - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, ds) -} - -func (ctx *spiderContext) postDataSource(c *gin.Context) { - // spider id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // data source id - dsId, err := primitive.ObjectIDFromHex(c.Param("ds_id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider - s, err := ctx.modelSvc.GetSpiderById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // data source - if !dsId.IsZero() { - _, err = ctx.modelSvc.GetDataSourceById(dsId) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - // save data source id - s.DataSourceId = dsId - if err := delegate2.NewModelDelegate(s).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) postExport(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // zip file path - zipFilePath, err := ctx.adminSvc.Export(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // download - c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilePath)) - c.File(zipFilePath) -} - -func (ctx *spiderContext) _get(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - s, err := ctx.modelSvc.GetSpiderById(id) - if err == mongo2.ErrNoDocuments { - HandleErrorNotFound(c, err) - return - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // stat - s.Stat, err = ctx.modelSvc.GetSpiderStatById(s.GetId()) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // data collection - if !s.ColId.IsZero() { - col, err := ctx.modelSvc.GetDataCollectionById(s.ColId) - if err != nil { - if err != mongo2.ErrNoDocuments { - HandleErrorInternalServerError(c, err) - return - } - } else { - s.ColName = col.Name - } - } - - HandleSuccessWithData(c, s) -} - -func (ctx *spiderContext) _post(c *gin.Context) (s *models.Spider, err error) { - // bind - s = &models.Spider{} - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return nil, err - } - - // upsert data collection - if err := ctx._upsertDataCollection(c, s); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - // add - if err := delegate2.NewModelDelegate(s, GetUserFromContext(c)).Add(); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - // add stat - st := &models.SpiderStat{ - Id: s.GetId(), - } - if err := delegate2.NewModelDelegate(st, GetUserFromContext(c)).Add(); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - return s, nil -} - -func (ctx *spiderContext) _put(c *gin.Context) (s *models.Spider, err error) { - // bind - s = &models.Spider{} - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return nil, err - } - - // upsert data collection - if err := ctx._upsertDataCollection(c, s); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - // save - if err := delegate2.NewModelDelegate(s, GetUserFromContext(c)).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - return s, nil -} - -func (ctx *spiderContext) _delete(c *gin.Context) (err error) { - id := c.Param("id") - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { - // delete spider - s, err := ctx.modelSvc.GetSpiderById(oid) - if err != nil { - return err - } - if err := delegate2.NewModelDelegate(s, GetUserFromContext(c)).Delete(); err != nil { - return err - } - - // delete spider stat - ss, err := ctx.modelSvc.GetSpiderStatById(oid) - if err != nil { - return err - } - if err := delegate2.NewModelDelegate(ss, GetUserFromContext(c)).Delete(); err != nil { - return err - } - - // related tasks - tasks, err := ctx.modelSvc.GetTaskList(bson.M{"spider_id": oid}, nil) - if err != nil { - return err - } - - // task ids - var taskIds []primitive.ObjectID - for _, t := range tasks { - taskIds = append(taskIds, t.Id) - } - - // delete related tasks - if err := ctx.modelTaskSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { - return err - } - - // delete related task stats - if err := ctx.modelTaskStatSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { - return err - } - - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return err - } - - return nil -} - -func (ctx *spiderContext) _getListWithStats(c *gin.Context) { - // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - - // get list - l, err := ctx.modelSpiderSvc.GetList(query, &mongo.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, - }) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - HandleErrorNotFound(c, err) - } else { - HandleErrorInternalServerError(c, err) - } - return - } - - // check empty list - if len(l.GetModels()) == 0 { - HandleSuccessWithListData(c, nil, 0) - return - } - - // ids - var ids []primitive.ObjectID - for _, d := range l.GetModels() { - s := d.(*models.Spider) - ids = append(ids, s.GetId()) - } - - // total count - total, err := ctx.modelSpiderSvc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // stat list - query = bson.M{ - "_id": bson.M{ - "$in": ids, - }, - } - stats, err := ctx.modelSvc.GetSpiderStatList(query, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // cache stat list to dict - dict := map[primitive.ObjectID]models.SpiderStat{} - var tids []primitive.ObjectID - for _, st := range stats { - if st.Tasks > 0 { - taskCount := int64(st.Tasks) - st.AverageWaitDuration = int64(math.Round(float64(st.WaitDuration) / float64(taskCount))) - st.AverageRuntimeDuration = int64(math.Round(float64(st.RuntimeDuration) / float64(taskCount))) - st.AverageTotalDuration = int64(math.Round(float64(st.TotalDuration) / float64(taskCount))) - } - dict[st.GetId()] = st - - if !st.LastTaskId.IsZero() { - tids = append(tids, st.LastTaskId) - } - } - - // task list and stats - var tasks []models.Task - dictTask := map[primitive.ObjectID]models.Task{} - dictTaskStat := map[primitive.ObjectID]models.TaskStat{} - if len(tids) > 0 { - // task list - queryTask := bson.M{ - "_id": bson.M{ - "$in": tids, - }, - } - tasks, err = ctx.modelSvc.GetTaskList(queryTask, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // task stats list - taskStats, err := ctx.modelSvc.GetTaskStatList(queryTask, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // cache task stats to dict - for _, st := range taskStats { - dictTaskStat[st.GetId()] = st - } - - // cache task list to dict - for _, t := range tasks { - st, ok := dictTaskStat[t.GetId()] - if ok { - t.Stat = &st - } - dictTask[t.GetSpiderId()] = t - } - } - - // iterate list again - var data []interface{} - for _, d := range l.GetModels() { - s := d.(*models.Spider) - - // spider stat - st, ok := dict[s.GetId()] - if ok { - s.Stat = &st - - // last task - t, ok := dictTask[s.GetId()] - if ok { - s.Stat.LastTask = &t - } - } - - // add to list - data = append(data, *s) - } - - // response - HandleSuccessWithListData(c, data, total) -} - -func (ctx *spiderContext) _deleteList(c *gin.Context) (err error) { - payload, err := NewJsonBinder(ControllerIdSpider).BindBatchRequestPayload(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { - // delete spiders - if err := ctx.modelSpiderSvc.DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - return err - } - - // delete spider stats - if err := ctx.modelSpiderStatSvc.DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - return err - } - - // related tasks - tasks, err := ctx.modelSvc.GetTaskList(bson.M{"spider_id": bson.M{"$in": payload.Ids}}, nil) - if err != nil { - return err - } - - // task ids - var taskIds []primitive.ObjectID - for _, t := range tasks { - taskIds = append(taskIds, t.Id) - } - - // delete related tasks - if err := ctx.modelTaskSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { - return err - } - - // delete related task stats - if err := ctx.modelTaskStatSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { - return err - } - - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return err - } - - return nil -} - -func (ctx *spiderContext) _processFileRequest(c *gin.Context, method string) (id primitive.ObjectID, payload entity.FileRequestPayload, fsSvc interfaces.FsServiceV2, err error) { - // id - id, err = primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // payload - contentType := c.GetHeader("Content-Type") - if strings.HasPrefix(contentType, "multipart/form-data") { - // multipart/form-data - payload, err = ctx._getFileRequestMultipartPayload(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - } else { - // query or application/json - switch method { - case http.MethodGet: - err = c.ShouldBindQuery(&payload) - default: - err = c.ShouldBindJSON(&payload) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - // fs service - workspacePath := viper.GetString("workspace") - fsSvc = fs2.NewFsServiceV2(filepath.Join(workspacePath, id.Hex())) - - return -} - -func (ctx *spiderContext) _getFileRequestMultipartPayload(c *gin.Context) (payload entity.FileRequestPayload, err error) { - fh, err := c.FormFile("file") - if err != nil { - return - } - f, err := fh.Open() - if err != nil { - return - } - buf := bytes.NewBuffer(nil) - if _, err = io.Copy(buf, f); err != nil { - return - } - payload.Path = c.PostForm("path") - payload.Data = buf.String() - return -} - -func (ctx *spiderContext) _processActionRequest(c *gin.Context) (id primitive.ObjectID, err error) { - // id - id, err = primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - return -} - -func (ctx *spiderContext) _upsertDataCollection(c *gin.Context, s *models.Spider) (err error) { - if s.ColId.IsZero() { - // validate - if s.ColName == "" { - return trace.TraceError(errors.ErrorControllerMissingRequestFields) - } - // no id - dc, err := ctx.modelSvc.GetDataCollectionByName(s.ColName, nil) - if err != nil { - if err == mongo2.ErrNoDocuments { - // not exists, add new - dc = &models.DataCollection{Name: s.ColName} - if err := delegate2.NewModelDelegate(dc, GetUserFromContext(c)).Add(); err != nil { - return err - } - } else { - // error - return err - } - } - s.ColId = dc.Id - - // create index - _ = mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{constants.TaskKey: 1}}) - _ = mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{constants.HashKey: 1}}) - } else { - // with id - dc, err := ctx.modelSvc.GetDataCollectionById(s.ColId) - if err != nil { - return err - } - s.ColId = dc.Id - } - return nil -} - -func (ctx *spiderContext) _getGitIgnore(id primitive.ObjectID) (ignore []string, err error) { - workspacePath := viper.GetString("workspace") - filePath := filepath.Join(workspacePath, id.Hex(), ".gitignore") - if !utils.Exists(filePath) { - return nil, nil - } - data, err := os.ReadFile(filePath) - if err != nil { - return nil, trace.TraceError(err) - } - ignore = strings.Split(string(data), "\n") - return ignore, nil -} - -func (ctx *spiderContext) _gitCheckout(gitClient *vcs.GitClient, remote, branch string) (err error) { - if err := gitClient.CheckoutBranch(branch, vcs.WithBranch(branch)); err != nil { - return trace.TraceError(err) - } - - // pull - return ctx._gitPull(gitClient, remote, branch) -} - -func (ctx *spiderContext) _gitPull(gitClient *vcs.GitClient, remote, branch string) (err error) { - // pull - if err := gitClient.Pull( - vcs.WithRemoteNamePull(remote), - vcs.WithBranchNamePull(branch), - ); err != nil { - return trace.TraceError(err) - } - - // reset - if err := gitClient.Reset(); err != nil { - return trace.TraceError(err) - } - - return nil -} - -func (ctx *spiderContext) _getGitClient(id primitive.ObjectID) (gitClient *vcs.GitClient, err error) { - // git - g, err := ctx.modelSvc.GetGitById(id) - if err != nil { - if err != mongo2.ErrNoDocuments { - return nil, trace.TraceError(err) - } - return nil, nil - } - - // git client - workspacePath := viper.GetString("workspace") - gitClient, err = vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, id.Hex()))) - if err != nil { - return nil, err - } - - // set auth - utils.InitGitClientAuth(g, gitClient) - - // remote name - remoteName := vcs.GitRemoteNameOrigin - - // update remote - r, err := gitClient.GetRemote(remoteName) - if err == git.ErrRemoteNotFound { - // remote not exists, create - if _, err := gitClient.CreateRemote(&config.RemoteConfig{ - Name: remoteName, - URLs: []string{g.Url}, - }); err != nil { - return nil, trace.TraceError(err) - } - } else if err == nil { - // remote exists, update if different - if g.Url != r.Config().URLs[0] { - if err := gitClient.DeleteRemote(remoteName); err != nil { - return nil, trace.TraceError(err) - } - if _, err := gitClient.CreateRemote(&config.RemoteConfig{ - Name: remoteName, - URLs: []string{g.Url}, - }); err != nil { - return nil, trace.TraceError(err) - } - } - gitClient.SetRemoteUrl(g.Url) - } else { - // error - return nil, trace.TraceError(err) - } - - // check if head reference exists - _, err = gitClient.GetRepository().Head() - if err == nil { - return gitClient, nil - } - - // align master/main branch - ctx._alignBranch(gitClient) - - return gitClient, nil -} - -func (ctx *spiderContext) _alignBranch(gitClient *vcs.GitClient) { - // current branch - currentBranch, err := gitClient.GetCurrentBranch() - if err != nil { - trace.PrintError(err) - return - } - - // skip if current branch is not master - if currentBranch != vcs.GitBranchNameMaster { - return - } - - // remote refs - refs, err := gitClient.GetRemoteRefs(vcs.GitRemoteNameOrigin) - if err != nil { - trace.PrintError(err) - return - } - - // main branch - defaultRemoteBranch, err := ctx._getDefaultRemoteBranch(refs) - if err != nil || defaultRemoteBranch == "" { - return - } - - // move branch - if err := gitClient.MoveBranch(vcs.GitBranchNameMaster, defaultRemoteBranch); err != nil { - trace.PrintError(err) - } -} - -func (ctx *spiderContext) _getDefaultRemoteBranch(refs []vcs.GitRef) (defaultRemoteBranchName string, err error) { - // remote branch name - for _, r := range refs { - if r.Type != vcs.GitRefTypeBranch { - continue - } - - if r.Name == vcs.GitBranchNameMain { - defaultRemoteBranchName = r.Name - break - } - - if r.Name == vcs.GitBranchNameMaster { - defaultRemoteBranchName = r.Name - continue - } - - if defaultRemoteBranchName == "" { - defaultRemoteBranchName = r.Name - continue - } - } - - return defaultRemoteBranchName, nil -} - -var _spiderCtx *spiderContext - -func newSpiderContext() *spiderContext { - if _spiderCtx != nil { - return _spiderCtx - } - - // context - ctx := &spiderContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - adminSvc interfaces.SpiderAdminService, - ) { - ctx.modelSvc = modelSvc - ctx.adminSvc = adminSvc - }); err != nil { - panic(err) - } - - // model spider service - ctx.modelSpiderSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdSpider) - - // model spider stat service - ctx.modelSpiderStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdSpiderStat) - - // model task service - ctx.modelTaskSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTask) - - // model task stat service - ctx.modelTaskStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTaskStat) - - _spiderCtx = ctx - - return ctx -} - -func newSpiderController() *spiderController { - actions := getSpiderActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdSpider, modelSvc.GetBaseService(interfaces.ModelIdSpider), actions) - d := NewListPostActionControllerDelegate(ControllerIdSpider, modelSvc.GetBaseService(interfaces.ModelIdSpider), actions) - ctx := newSpiderContext() - - return &spiderController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/stats.go b/core/controllers/stats.go deleted file mode 100644 index 3d474956..00000000 --- a/core/controllers/stats.go +++ /dev/null @@ -1,87 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "net/http" - "time" -) - -var StatsController ActionController - -func getStatsActions() []Action { - statsCtx := newStatsContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/overview", - HandlerFunc: statsCtx.getOverview, - }, - { - Method: http.MethodGet, - Path: "/daily", - HandlerFunc: statsCtx.getDaily, - }, - { - Method: http.MethodGet, - Path: "/tasks", - HandlerFunc: statsCtx.getTasks, - }, - } -} - -type statsContext struct { - statsSvc interfaces.StatsService - defaultQuery bson.M -} - -func (svc *statsContext) getOverview(c *gin.Context) { - data, err := svc.statsSvc.GetOverviewStats(svc.defaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, data) -} - -func (svc *statsContext) getDaily(c *gin.Context) { - data, err := svc.statsSvc.GetDailyStats(svc.defaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, data) -} - -func (svc *statsContext) getTasks(c *gin.Context) { - data, err := svc.statsSvc.GetTaskStats(svc.defaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, data) -} - -func newStatsContext() *statsContext { - // context - ctx := &statsContext{ - defaultQuery: bson.M{ - "create_ts": bson.M{ - "$gte": time.Now().Add(-30 * 24 * time.Hour), - }, - }, - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - statsSvc interfaces.StatsService, - ) { - ctx.statsSvc = statsSvc - }); err != nil { - panic(err) - } - - return ctx -} diff --git a/core/controllers/sync.go b/core/controllers/sync.go deleted file mode 100644 index 5f9874c8..00000000 --- a/core/controllers/sync.go +++ /dev/null @@ -1,58 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/utils" - "github.com/gin-gonic/gin" - "github.com/spf13/viper" - "net/http" - "path/filepath" -) - -var SyncController ActionController - -func getSyncActions() []Action { - var ctx = newSyncContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/:id/scan", - HandlerFunc: ctx.scan, - }, - { - Method: http.MethodGet, - Path: "/:id/download", - HandlerFunc: ctx.download, - }, - } -} - -type syncContext struct { -} - -func (ctx *syncContext) scan(c *gin.Context) { - id := c.Param("id") - path := c.Query("path") - dir := ctx._getDir(id, path) - files, err := utils.ScanDirectory(dir) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - c.AbortWithStatusJSON(http.StatusOK, files) -} - -func (ctx *syncContext) download(c *gin.Context) { - id := c.Param("id") - filePath := c.Query("path") - dir := ctx._getDir(id, "") - c.File(filepath.Join(dir, filePath)) -} - -func (ctx *syncContext) _getDir(id string, path string) string { - workspacePath := viper.GetString("workspace") - return filepath.Join(workspacePath, id, path) -} - -func newSyncContext() syncContext { - return syncContext{} -} diff --git a/core/controllers/system_info.go b/core/controllers/system_info.go deleted file mode 100644 index a89edc15..00000000 --- a/core/controllers/system_info.go +++ /dev/null @@ -1,28 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/gin-gonic/gin" - "github.com/spf13/viper" - "net/http" -) - -func getSystemInfo(c *gin.Context) { - info := &entity.SystemInfo{ - Edition: viper.GetString("info.edition"), - Version: viper.GetString("info.version"), - } - HandleSuccessWithData(c, info) -} - -func getSystemInfoActions() []Action { - return []Action{ - { - Path: "", - Method: http.MethodGet, - HandlerFunc: getSystemInfo, - }, - } -} - -var SystemInfoController ActionController diff --git a/core/controllers/system_info_v2.go b/core/controllers/system_info_v2.go index fc01ea6f..9b4fad83 100644 --- a/core/controllers/system_info_v2.go +++ b/core/controllers/system_info_v2.go @@ -8,8 +8,8 @@ import ( func GetSystemInfo(c *gin.Context) { info := &entity.SystemInfo{ - Edition: viper.GetString("info.edition"), - Version: viper.GetString("info.version"), + Edition: viper.GetString("edition"), + Version: viper.GetString("version"), } HandleSuccessWithData(c, info) } diff --git a/core/controllers/tag.go b/core/controllers/tag.go deleted file mode 100644 index 3f74abf2..00000000 --- a/core/controllers/tag.go +++ /dev/null @@ -1,3 +0,0 @@ -package controllers - -var TagController ListController diff --git a/core/controllers/task.go b/core/controllers/task.go deleted file mode 100644 index 9a66b9ef..00000000 --- a/core/controllers/task.go +++ /dev/null @@ -1,534 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - delegate2 "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/result" - "github.com/crawlab-team/crawlab/core/task/log" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" - "strings" -) - -var TaskController *taskController - -func getTaskActions() []Action { - taskCtx := newTaskContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/run", - HandlerFunc: taskCtx.run, - }, - { - Method: http.MethodPost, - Path: "/:id/restart", - HandlerFunc: taskCtx.restart, - }, - { - Method: http.MethodPost, - Path: "/:id/cancel", - HandlerFunc: taskCtx.cancel, - }, - { - Method: http.MethodGet, - Path: "/:id/logs", - HandlerFunc: taskCtx.getLogs, - }, - { - Method: http.MethodGet, - Path: "/:id/data", - HandlerFunc: taskCtx.getData, - }, - } -} - -type taskController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *taskContext -} - -func (ctr *taskController) Get(c *gin.Context) { - ctr.ctx.getWithStatsSpider(c) -} - -func (ctr *taskController) Delete(c *gin.Context) { - if err := ctr.ctx._delete(c); err != nil { - return - } - HandleSuccess(c) -} - -func (ctr *taskController) GetList(c *gin.Context) { - withStats := c.Query("stats") - if withStats == "" { - ctr.d.GetList(c) - return - } - ctr.ctx.getListWithStats(c) -} - -func (ctr *taskController) DeleteList(c *gin.Context) { - if err := ctr.ctx._deleteList(c); err != nil { - return - } - HandleSuccess(c) -} - -type taskContext struct { - modelSvc service.ModelService - modelTaskSvc interfaces.ModelBaseService - modelTaskStatSvc interfaces.ModelBaseService - adminSvc interfaces.SpiderAdminService - schedulerSvc interfaces.TaskSchedulerService - l log.Driver -} - -func (ctx *taskContext) run(c *gin.Context) { - // task - var t models.Task - if err := c.ShouldBindJSON(&t); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // validate spider id - if t.GetSpiderId().IsZero() { - HandleErrorBadRequest(c, errors.ErrorTaskEmptySpiderId) - return - } - - // spider - s, err := ctx.modelSvc.GetSpiderById(t.GetSpiderId()) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // options - opts := &interfaces.SpiderRunOptions{ - Mode: t.Mode, - NodeIds: t.NodeIds, - Cmd: t.Cmd, - Param: t.Param, - Priority: t.Priority, - } - - // user - if u := GetUserFromContext(c); u != nil { - opts.UserId = u.GetId() - } - - // run - taskIds, err := ctx.adminSvc.Schedule(s.GetId(), opts) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, taskIds) -} - -func (ctx *taskContext) restart(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // task - t, err := ctx.modelSvc.GetTaskById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // options - opts := &interfaces.SpiderRunOptions{ - Mode: t.Mode, - NodeIds: t.NodeIds, - Cmd: t.Cmd, - Param: t.Param, - Priority: t.Priority, - } - - // user - if u := GetUserFromContext(c); u != nil { - opts.UserId = u.GetId() - } - - // run - taskIds, err := ctx.adminSvc.Schedule(t.SpiderId, opts) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, taskIds) -} - -func (ctx *taskContext) cancel(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // task - t, err := ctx.modelSvc.GetTaskById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // validate - if !utils.IsCancellable(t.Status) { - HandleErrorInternalServerError(c, errors.ErrorControllerNotCancellable) - return - } - - // cancel - if err := ctx.schedulerSvc.Cancel(id, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *taskContext) getLogs(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // pagination - p, err := GetPagination(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // logs - logs, err := ctx.l.Find(id.Hex(), "", (p.Page-1)*p.Size, p.Size) - if err != nil { - if strings.HasSuffix(err.Error(), "Status:404 Not Found") { - HandleSuccess(c) - return - } - HandleErrorInternalServerError(c, err) - return - } - total, err := ctx.l.Count(id.Hex(), "") - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithListData(c, logs, total) -} - -func (ctx *taskContext) getListWithStats(c *gin.Context) { - // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - - // get list - list, err := ctx.modelTaskSvc.GetList(query, &mongo.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, - }) - if err != nil { - if err == mongo2.ErrNoDocuments { - HandleErrorNotFound(c, err) - } else { - HandleErrorInternalServerError(c, err) - } - return - } - - // check empty list - if len(list.GetModels()) == 0 { - HandleSuccessWithListData(c, nil, 0) - return - } - - // ids - var ids []primitive.ObjectID - for _, d := range list.GetModels() { - t := d.(interfaces.Model) - ids = append(ids, t.GetId()) - } - - // total count - total, err := ctx.modelTaskSvc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // stat list - query = bson.M{ - "_id": bson.M{ - "$in": ids, - }, - } - stats, err := ctx.modelSvc.GetTaskStatList(query, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // cache stat list to dict - dict := map[primitive.ObjectID]models.TaskStat{} - for _, s := range stats { - dict[s.GetId()] = s - } - - // iterate list again - var data []interface{} - for _, d := range list.GetModels() { - t := d.(*models.Task) - s, ok := dict[t.GetId()] - if ok { - t.Stat = &s - } - data = append(data, *t) - } - - // response - HandleSuccessWithListData(c, data, total) -} - -func (ctx *taskContext) getWithStatsSpider(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // task - t, err := ctx.modelSvc.GetTaskById(id) - if err == mongo2.ErrNoDocuments { - HandleErrorNotFound(c, err) - return - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // spider - t.Spider, _ = ctx.modelSvc.GetSpiderById(t.SpiderId) - - // skip if task status is pending - if t.Status == constants.TaskStatusPending { - HandleSuccessWithData(c, t) - return - } - - // task stat - t.Stat, _ = ctx.modelSvc.GetTaskStatById(id) - - HandleSuccessWithData(c, t) -} - -func (ctx *taskContext) getData(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // pagination - p, err := GetPagination(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // task - t, err := ctx.modelSvc.GetTaskById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // result service - resultSvc, err := result.GetResultService(t.SpiderId) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // query - query := generic.ListQuery{ - generic.ListQueryCondition{ - Key: constants.TaskKey, - Op: generic.OpEqual, - Value: t.Id, - }, - } - - // list - data, err := resultSvc.List(query, &generic.ListOptions{ - Skip: (p.Page - 1) * p.Size, - Limit: p.Size, - Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}}, - }) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // total - total, err := resultSvc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithListData(c, data, total) -} - -func (ctx *taskContext) _delete(c *gin.Context) (err error) { - id := c.Param("id") - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { - // delete task - task, err := ctx.modelSvc.GetTaskById(oid) - if err != nil { - return err - } - if err := delegate2.NewModelDelegate(task, GetUserFromContext(c)).Delete(); err != nil { - return err - } - - // delete task stat - taskStat, err := ctx.modelSvc.GetTaskStatById(oid) - if err != nil { - return err - } - if err := delegate2.NewModelDelegate(taskStat, GetUserFromContext(c)).Delete(); err != nil { - return err - } - - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return err - } - - return nil -} - -func (ctx *taskContext) _deleteList(c *gin.Context) (err error) { - payload, err := NewJsonBinder(ControllerIdTask).BindBatchRequestPayload(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := mongo.RunTransaction(func(context mongo2.SessionContext) error { - // delete tasks - if err := ctx.modelTaskSvc.DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - return err - } - - // delete task stats - if err := ctx.modelTaskStatSvc.DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - return err - } - - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return err - } - - return nil -} - -func newTaskContext() *taskContext { - // context - ctx := &taskContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - adminSvc interfaces.SpiderAdminService, - schedulerSvc interfaces.TaskSchedulerService, - ) { - ctx.modelSvc = modelSvc - ctx.adminSvc = adminSvc - ctx.schedulerSvc = schedulerSvc - }); err != nil { - panic(err) - } - - // model task service - ctx.modelTaskSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTask) - - // model task stat service - ctx.modelTaskStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTaskStat) - - // log driver - l, err := log.GetLogDriver(log.DriverTypeFile) - if err != nil { - panic(err) - } - ctx.l = l - - return ctx -} - -func newTaskController() *taskController { - actions := getTaskActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdTask, modelSvc.GetBaseService(interfaces.ModelIdTask), actions) - d := NewListPostActionControllerDelegate(ControllerIdTask, modelSvc.GetBaseService(interfaces.ModelIdTask), actions) - ctx := newTaskContext() - - return &taskController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/test/base.go b/core/controllers/test/base.go deleted file mode 100644 index 1e3c2906..00000000 --- a/core/controllers/test/base.go +++ /dev/null @@ -1,104 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/controllers" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/routes" - "github.com/crawlab-team/crawlab/trace" - "github.com/gavv/httpexpect/v2" - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" - "go.uber.org/dig" - "net/http/httptest" - "testing" - "time" -) - -func init() { - var err error - T, err = NewTest() - if err != nil { - panic(err) - } -} - -type Test struct { - // dependencies - modelSvc service.ModelService - - // internals - app *gin.Engine - svr *httptest.Server - - // test data - TestUsername string - TestPassword string - TestToken string -} - -func (t *Test) Setup(t2 *testing.T) { - //if err := controllers.InitControllers(); err != nil { - // panic(err) - //} - //t2.Cleanup(t.Cleanup) -} - -func (t *Test) Cleanup() { - _ = t.modelSvc.DropAll() - time.Sleep(200 * time.Millisecond) -} - -func (t *Test) NewExpect(t2 *testing.T) (e *httpexpect.Expect) { - e = httpexpect.New(t2, t.svr.URL) - res := e.POST("/login").WithJSON(map[string]string{ - "username": t.TestUsername, - "password": t.TestPassword, - }).Expect().JSON().Object() - t.TestToken = res.Path("$.data").String().Raw() - require.NotEmpty(t2, t.TestToken) - return e -} - -func (t *Test) WithAuth(req *httpexpect.Request) *httpexpect.Request { - return req.WithHeader("Authorization", t.TestToken) -} - -var T *Test - -func NewTest() (res *Test, err error) { - // test - t := &Test{} - - // gin app - t.app = gin.New() - - // http test server - t.svr = httptest.NewServer(t.app) - - // init controllers - if err := controllers.InitControllers(); err != nil { - return nil, err - } - - // init routes - if err := routes.InitRoutes(t.app); err != nil { - return nil, err - } - - // dependency injection - c := dig.New() - if err := c.Provide(service.NewService); err != nil { - return nil, trace.TraceError(err) - } - if err := c.Invoke(func(modelSvc service.ModelService) { - t.modelSvc = modelSvc - }); err != nil { - return nil, trace.TraceError(err) - } - - // test data - t.TestUsername = "admin" - t.TestPassword = "admin" - - return t, nil -} diff --git a/core/controllers/test/export_test.go b/core/controllers/test/export_test.go deleted file mode 100644 index 7623feb6..00000000 --- a/core/controllers/test/export_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/spf13/viper" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "net/http" - "testing" - "time" -) - -func init() { - viper.Set("mongo.db", "crawlab_test") -} - -func TestExportController_Csv(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - // mongo collection - colName := "test_collection_for_export" - col := mongo.GetMongoCol(colName) - - // insert test data to mongo collection - for i := 0; i < 10; i++ { - _, err := col.Insert(bson.M{ - "field1": i + 1, - "field2": i + 2, - "field3": i + 3, - "field4": i + 4, - }) - require.Nil(t, err) - } - - // export from mongo collection - res := T.WithAuth(e.POST("/export/csv")). - WithQuery("target", colName). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - - // export id - exportId := res.Path("$.data").String().Raw() - - // poll export with export id - for i := 0; i < 10; i++ { - res = T.WithAuth(e.GET("/export/csv/" + exportId)).Expect().Status(http.StatusOK).JSON().Object() - status := res.Path("$.data.status").String().Raw() - if status == constants.TaskStatusFinished { - break - } - time.Sleep(1 * time.Second) - } - - // download exported csv file - csvFileBody := T.WithAuth(e.GET("/export/csv/" + exportId + "/download")).Expect().Status(http.StatusOK).Body() - csvFileBody.NotEmpty() -} diff --git a/core/controllers/test/filter_test.go b/core/controllers/test/filter_test.go deleted file mode 100644 index e7579e0f..00000000 --- a/core/controllers/test/filter_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package test - -import ( - "encoding/json" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/spf13/viper" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" - "testing" -) - -func init() { - viper.Set("mongo.db", "crawlab_test") -} - -func TestFilterController_GetColFieldOptions(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - // mongo collection - colName := "test_collection_for_filter" - field1 := "field1" - field2 := "field2" - value1 := "value1" - col := mongo.GetMongoCol(colName) - n := 10 - var ids []primitive.ObjectID - var names []string - for i := 0; i < n; i++ { - _id := primitive.NewObjectID() - ids = append(ids, _id) - name := fmt.Sprintf("name_%d", i) - names = append(names, name) - _, err := col.Insert(bson.M{field1: value1, field2: i % 2, "name": name, "_id": _id}) - require.Nil(t, err) - } - - // validate filter options field 1 - res := T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field1, field1))). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - res.Path("$.data").Array().Length().Equal(1) - res.Path("$.data").Array().Element(0).Path("$.value").Equal(value1) - - // validate filter options field 2 - res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - res.Path("$.data").Array().Length().Equal(2) - - // validate filter options with query - conditions := []entity.Condition{{field2, constants.FilterOpEqual, 0}} - conditionsJson, err := json.Marshal(conditions) - conditionsJsonStr := string(conditionsJson) - require.Nil(t, err) - res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))). - WithQuery(constants.FilterQueryFieldConditions, conditionsJsonStr). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - res.Path("$.data").Array().Length().Equal(1) - - // validate filter options (basic path) - res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s", colName))). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - res.Path("$.data").Array().Length().Equal(n) - for i := 0; i < n; i++ { - res.Path("$.data").Array().Element(i).Object().Value("value").Equal(ids[i]) - res.Path("$.data").Array().Element(i).Object().Value("label").Equal(names[i]) - } -} diff --git a/core/controllers/test/main_test.go b/core/controllers/test/main_test.go deleted file mode 100644 index 2b9bb76b..00000000 --- a/core/controllers/test/main_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/controllers" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/user" - "go.mongodb.org/mongo-driver/mongo" - "testing" -) - -func TestMain(m *testing.M) { - // init user - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - _, err = modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil) - if err != nil { - if err.Error() != mongo.ErrNoDocuments.Error() { - panic(err) - } - userSvc, err := user.GetUserService() - if err != nil { - panic(err) - } - if err := userSvc.Create(&interfaces.UserCreateOptions{ - Username: constants.DefaultAdminUsername, - Password: constants.DefaultAdminPassword, - Role: constants.RoleAdmin, - }); err != nil { - panic(err) - } - } - - if err := controllers.InitControllers(); err != nil { - panic(err) - } - - m.Run() - - T.Cleanup() -} diff --git a/core/controllers/test/project_test.go b/core/controllers/test/project_test.go deleted file mode 100644 index b0a1b2c6..00000000 --- a/core/controllers/test/project_test.go +++ /dev/null @@ -1,305 +0,0 @@ -package test - -import ( - "encoding/json" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" - "testing" - "time" -) - -func TestProjectController_Get(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - p := models.Project{ - Name: "test project", - } - res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data._id").NotNull() - id := res.Path("$.data._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - res = T.WithAuth(e.GET("/projects/" + id)).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data._id").NotNull() - res.Path("$.data.name").Equal("test project") -} - -func TestProjectController_Put(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - p := models.Project{ - Name: "old name", - Description: "old description", - } - - // add - res := T.WithAuth(e.POST("/projects")). - WithJSON(p). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data._id").NotNull() - id := res.Path("$.data._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - // change object - p.Id = oid - p.Name = "new name" - p.Description = "new description" - - // update - T.WithAuth(e.PUT("/projects/" + id)). - WithJSON(p). - Expect().Status(http.StatusOK) - - // check - res = T.WithAuth(e.GET("/projects/" + id)).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data._id").Equal(id) - res.Path("$.data.name").Equal("new name") - res.Path("$.data.description").Equal("new description") -} - -func TestProjectController_Post(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - p := models.Project{ - Name: "test project", - Description: "this is a test project", - } - - res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data._id").NotNull() - res.Path("$.data.name").Equal("test project") - res.Path("$.data.description").Equal("this is a test project") -} - -func TestProjectController_Delete(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - p := models.Project{ - Name: "test project", - Description: "this is a test project", - } - - // add - res := T.WithAuth(e.POST("/projects")). - WithJSON(p). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data._id").NotNull() - id := res.Path("$.data._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - // get - res = T.WithAuth(e.GET("/projects/" + id)). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data._id").NotNull() - id = res.Path("$.data._id").String().Raw() - oid, err = primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - // delete - T.WithAuth(e.DELETE("/projects/" + id)). - Expect().Status(http.StatusOK). - JSON().Object() - - // get - T.WithAuth(e.GET("/projects/" + id)). - Expect().Status(http.StatusNotFound) -} - -func TestProjectController_GetList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - n := 100 // total - bn := 10 // batch - - for i := 0; i < n; i++ { - p := models.Project{ - Name: fmt.Sprintf("test name %d", i+1), - } - obj := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() - obj.Path("$.data._id").NotNull() - } - - f := entity.Filter{ - //IsOr: false, - Conditions: []*entity.Condition{ - {Key: "name", Op: constants.FilterOpContains, Value: "test name"}, - }, - } - condBytes, err := json.Marshal(&f.Conditions) - require.Nil(t, err) - - pagination := entity.Pagination{ - Page: 1, - Size: bn, - } - - // get list with pagination - res := T.WithAuth(e.GET("/projects")). - WithQuery("conditions", string(condBytes)). - WithQueryObject(pagination). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").Array().Length().Equal(bn) - res.Path("$.total").Number().Equal(n) - - data := res.Path("$.data").Array() - for i := 0; i < bn; i++ { - obj := data.Element(i) - obj.Path("$.name").Equal(fmt.Sprintf("test name %d", i+1)) - } - -} - -func TestProjectController_PostList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - n := 10 - var docs []models.Project - for i := 0; i < n; i++ { - docs = append(docs, models.Project{ - Name: fmt.Sprintf("project %d", i+1), - Description: "this is a project", - }) - } - - T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK) - - res := T.WithAuth(e.GET("/projects")). - WithQueryObject(entity.Pagination{Page: 1, Size: 10}). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data").Array().Length().Equal(n) -} - -func TestProjectController_DeleteList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - n := 10 - var docs []models.Project - for i := 0; i < n; i++ { - docs = append(docs, models.Project{ - Name: fmt.Sprintf("project %d", i+1), - Description: "this is a project", - }) - } - - // add - res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK). - JSON().Object() - var ids []primitive.ObjectID - data := res.Path("$.data").Array() - for i := 0; i < n; i++ { - obj := data.Element(i) - id := obj.Path("$._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - ids = append(ids, oid) - } - - // delete - payload := entity.BatchRequestPayload{ - Ids: ids, - } - T.WithAuth(e.DELETE("/projects")). - WithJSON(payload). - Expect().Status(http.StatusOK) - - // check - for _, id := range ids { - T.WithAuth(e.GET("/projects/" + id.Hex())). - Expect().Status(http.StatusNotFound) - } - -} - -func TestProjectController_PutList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - // now - now := time.Now() - - n := 10 - var docs []models.Project - for i := 0; i < n; i++ { - docs = append(docs, models.Project{ - Name: "old name", - Description: "old description", - }) - } - - // add - res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK). - JSON().Object() - var ids []primitive.ObjectID - data := res.Path("$.data").Array() - for i := 0; i < n; i++ { - obj := data.Element(i) - id := obj.Path("$._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - ids = append(ids, oid) - } - - // wait for 100 millisecond - time.Sleep(100 * time.Millisecond) - - // update - p := models.Project{ - Name: "new name", - Description: "new description", - } - dataBytes, err := json.Marshal(&p) - require.Nil(t, err) - payload := entity.BatchRequestPayloadWithStringData{ - Ids: ids, - Data: string(dataBytes), - Fields: []string{ - "name", - "description", - }, - } - T.WithAuth(e.PUT("/projects")).WithJSON(payload).Expect().Status(http.StatusOK) - - // check response data - for i := 0; i < n; i++ { - res = T.WithAuth(e.GET("/projects/" + ids[i].Hex())).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data.name").Equal("new name") - res.Path("$.data.description").Equal("new description") - } - - // check artifacts - pl, err := T.modelSvc.GetProjectList(bson.M{"_id": bson.M{"$in": ids}}, nil) - require.Nil(t, err) - for _, p := range pl { - a, err := delegate.NewModelDelegate(&p).GetArtifact() - require.Nil(t, err) - require.True(t, a.GetSys().GetUpdateTs().After(now)) - require.True(t, a.GetSys().GetUpdateTs().After(a.GetSys().GetCreateTs())) - } -} diff --git a/core/controllers/test/spider_test.go b/core/controllers/test/spider_test.go deleted file mode 100644 index b8c4dc06..00000000 --- a/core/controllers/test/spider_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" - "testing" -) - -func TestSpiderController_Delete(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - s := models.Spider{ - Name: "test spider", - Description: "this is a test spider", - ColName: "test col name", - } - - // add spider - res := T.WithAuth(e.POST("/spiders")). - WithJSON(s). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data._id").NotNull() - id := res.Path("$.data._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - // add tasks - var taskIds []primitive.ObjectID - tasks := []models.Task{ - { - Id: primitive.NewObjectID(), - SpiderId: oid, - }, - { - Id: primitive.NewObjectID(), - SpiderId: oid, - }, - } - for _, task := range tasks { - // add task - err := delegate.NewModelDelegate(&task).Add() - require.Nil(t, err) - - // add task stat - err = delegate.NewModelDelegate(&models.TaskStat{ - Id: task.Id, - }).Add() - require.Nil(t, err) - - taskIds = append(taskIds, task.Id) - } - - // delete - T.WithAuth(e.DELETE("/spiders/" + id)). - Expect().Status(http.StatusOK) - - // get - T.WithAuth(e.GET("/spiders/" + id)). - Expect().Status(http.StatusNotFound) - - // get tasks - for _, task := range tasks { - T.WithAuth(e.GET("/tasks/" + task.Id.Hex())). - Expect().Status(http.StatusNotFound) - } - - // spider stat - modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat) - spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{ - "_id": oid, - }) - require.Nil(t, err) - require.Zero(t, spiderStatCount) - - // task stats - modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) - taskStatCount, err := modelTaskStatSvc.Count(bson.M{ - "_id": bson.M{ - "$in": taskIds, - }, - }) - require.Nil(t, err) - require.Zero(t, taskStatCount) -} - -func TestSpiderController_DeleteList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - spiders := []models.Spider{ - { - Id: primitive.NewObjectID(), - Name: "test spider 1", - Description: "this is a test spider 1", - ColName: "test col name 1", - }, - { - Id: primitive.NewObjectID(), - Name: "test spider 2", - Description: "this is a test spider 2", - ColName: "test col name 2", - }, - } - - // add spiders - for _, spider := range spiders { - T.WithAuth(e.POST("/spiders")). - WithJSON(spider). - Expect().Status(http.StatusOK) - } - - var spiderIds []primitive.ObjectID - var taskIds []primitive.ObjectID - for _, spider := range spiders { - // task id - taskId := primitive.NewObjectID() - - // add task - err := delegate.NewModelDelegate(&models.Task{ - Id: taskId, - SpiderId: spider.Id, - }).Add() - require.Nil(t, err) - - // add task stats - err = delegate.NewModelDelegate(&models.TaskStat{ - Id: taskId, - }).Add() - require.Nil(t, err) - - spiderIds = append(spiderIds, spider.Id) - taskIds = append(taskIds, taskId) - } - - // delete spiders - T.WithAuth(e.DELETE("/spiders")). - WithJSON(entity.BatchRequestPayload{ - Ids: spiderIds, - }).Expect().Status(http.StatusOK) - - // get spiders - for _, spider := range spiders { - // get - T.WithAuth(e.GET("/spiders/" + spider.Id.Hex())). - Expect().Status(http.StatusNotFound) - } - - // get tasks - for _, taskId := range taskIds { - T.WithAuth(e.GET("/tasks/" + taskId.Hex())). - Expect().Status(http.StatusNotFound) - } - - // spider stat - modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat) - spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{ - "_id": bson.M{ - "$in": spiderIds, - }, - }) - require.Nil(t, err) - require.Zero(t, spiderStatCount) - - // task stats - modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) - taskStatCount, err := modelTaskStatSvc.Count(bson.M{ - "_id": bson.M{ - "$in": taskIds, - }, - }) - require.Nil(t, err) - require.Zero(t, taskStatCount) -} diff --git a/core/controllers/test/task_test.go b/core/controllers/test/task_test.go deleted file mode 100644 index 4d81738b..00000000 --- a/core/controllers/test/task_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" - "testing" -) - -func TestTaskController_Delete(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - task := models.Task{ - Id: primitive.NewObjectID(), - } - - // add task - err := delegate.NewModelDelegate(&task).Add() - require.Nil(t, err) - - // add task stat - err = delegate.NewModelDelegate(&models.TaskStat{ - Id: task.Id, - }).Add() - require.Nil(t, err) - - // delete - T.WithAuth(e.DELETE("/tasks/" + task.Id.Hex())). - Expect().Status(http.StatusOK) - - // get - T.WithAuth(e.GET("/tasks/" + task.Id.Hex())). - Expect().Status(http.StatusNotFound) - - // task stats - modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) - taskStatCount, err := modelTaskStatSvc.Count(bson.M{ - "_id": task.Id, - }) - require.Nil(t, err) - require.Zero(t, taskStatCount) -} - -func TestTaskController_DeleteList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - tasks := []models.Task{ - { - Id: primitive.NewObjectID(), - }, - { - Id: primitive.NewObjectID(), - }, - } - - // add spiders - var taskIds []primitive.ObjectID - for _, task := range tasks { - // add task - err := delegate.NewModelDelegate(&task).Add() - require.Nil(t, err) - - // add task stat - err = delegate.NewModelDelegate(&models.TaskStat{ - Id: task.Id, - }).Add() - require.Nil(t, err) - - taskIds = append(taskIds, task.Id) - } - - // delete tasks - T.WithAuth(e.DELETE("/tasks")). - WithJSON(entity.BatchRequestPayload{ - Ids: taskIds, - }).Expect().Status(http.StatusOK) - - // get tasks - for _, task := range tasks { - // get - T.WithAuth(e.GET("/tasks/" + task.Id.Hex())). - Expect().Status(http.StatusNotFound) - } - - // task stats - modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) - taskStatCount, err := modelTaskStatSvc.Count(bson.M{ - "_id": bson.M{ - "$in": taskIds, - }, - }) - require.Nil(t, err) - require.Zero(t, taskStatCount) -} diff --git a/core/controllers/token.go b/core/controllers/token.go deleted file mode 100644 index f94f3b68..00000000 --- a/core/controllers/token.go +++ /dev/null @@ -1,84 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/gin-gonic/gin" -) - -var TokenController *tokenController - -var TokenActions []Action - -type tokenController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *tokenContext -} - -func (ctr *tokenController) Post(c *gin.Context) { - var err error - var t models.Token - if err := c.ShouldBindJSON(&t); err != nil { - HandleErrorBadRequest(c, err) - return - } - u, err := ctr.ctx.userSvc.GetCurrentUser(c) - if err != nil { - HandleErrorUnauthorized(c, err) - return - } - t.Token, err = ctr.ctx.userSvc.MakeToken(u) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := delegate.NewModelDelegate(&t, GetUserFromContext(c)).Add(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -type tokenContext struct { - modelSvc service.ModelService - userSvc interfaces.UserService -} - -func newTokenContext() *tokenContext { - // context - ctx := &tokenContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - userSvc interfaces.UserService, - ) { - ctx.modelSvc = modelSvc - ctx.userSvc = userSvc - }); err != nil { - panic(err) - } - - return ctx -} - -func newTokenController() *tokenController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdToken, modelSvc.GetBaseService(interfaces.ModelIdToken), TokenActions) - d := NewListPostActionControllerDelegate(ControllerIdToken, modelSvc.GetBaseService(interfaces.ModelIdToken), TokenActions) - ctx := newTokenContext() - - return &tokenController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/user.go b/core/controllers/user.go deleted file mode 100644 index 5c22a729..00000000 --- a/core/controllers/user.go +++ /dev/null @@ -1,244 +0,0 @@ -package controllers - -import ( - "encoding/json" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - delegate2 "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" -) - -var UserController *userController - -func getUserActions() []Action { - userCtx := newUserContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/:id/change-password", - HandlerFunc: userCtx.changePassword, - }, - { - Method: http.MethodGet, - Path: "/me", - HandlerFunc: userCtx.getMe, - }, - { - Method: http.MethodPut, - Path: "/me", - HandlerFunc: userCtx.putMe, - }, - } -} - -type userController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *userContext -} - -func (ctr *userController) Post(c *gin.Context) { - var u models.User - if err := c.ShouldBindJSON(&u); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctr.ctx.userSvc.Create(&interfaces.UserCreateOptions{ - Username: u.Username, - Password: u.Password, - Email: u.Email, - Role: u.Role, - }); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctr *userController) PostList(c *gin.Context) { - // users - var users []models.User - if err := c.ShouldBindJSON(&users); err != nil { - HandleErrorBadRequest(c, err) - return - } - - for _, u := range users { - if err := ctr.ctx.userSvc.Create(&interfaces.UserCreateOptions{ - Username: u.Username, - Password: u.Password, - Email: u.Email, - Role: u.Role, - }); err != nil { - trace.PrintError(err) - } - } - - HandleSuccess(c) -} - -func (ctr *userController) PutList(c *gin.Context) { - // payload - var payload entity.BatchRequestPayloadWithStringData - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // doc to update - var doc models.User - if err := json.Unmarshal([]byte(payload.Data), &doc); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // query - query := bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - } - - // update users - if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdUser).UpdateDoc(query, &doc, payload.Fields); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // update passwords - if utils.Contains(payload.Fields, "password") { - for _, id := range payload.Ids { - if err := ctr.ctx.userSvc.ChangePassword(id, doc.Password); err != nil { - trace.PrintError(err) - } - } - } - - HandleSuccess(c) -} - -type userContext struct { - modelSvc service.ModelService - userSvc interfaces.UserService -} - -func (ctx *userContext) changePassword(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - var payload map[string]string - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - password, ok := payload["password"] - if !ok { - HandleErrorBadRequest(c, errors.ErrorUserMissingRequiredFields) - return - } - if len(password) < 5 { - HandleErrorBadRequest(c, errors.ErrorUserInvalidPassword) - return - } - if err := ctx.userSvc.ChangePassword(id, password); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *userContext) getMe(c *gin.Context) { - u, err := ctx._getMe(c) - if err != nil { - HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) - return - } - HandleSuccessWithData(c, u) -} - -func (ctx *userContext) putMe(c *gin.Context) { - // current user - u, err := ctx._getMe(c) - if err != nil { - HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) - return - } - - // payload - doc, err := NewJsonBinder(ControllerIdUser).Bind(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if doc.GetId() != u.GetId() { - HandleErrorBadRequest(c, errors.ErrorHttpBadRequest) - return - } - - // save to db - if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, doc) -} - -func (ctx *userContext) _getMe(c *gin.Context) (u interfaces.User, err error) { - res, ok := c.Get(constants.UserContextKey) - if !ok { - return nil, trace.TraceError(errors.ErrorUserNotExistsInContext) - } - u, ok = res.(interfaces.User) - if !ok { - return nil, trace.TraceError(errors.ErrorUserInvalidType) - } - return u, nil -} - -func newUserContext() *userContext { - // context - ctx := &userContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - userSvc interfaces.UserService, - ) { - ctx.modelSvc = modelSvc - ctx.userSvc = userSvc - }); err != nil { - panic(err) - } - - return ctx -} - -func newUserController() *userController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdUser, modelSvc.GetBaseService(interfaces.ModelIdUser), getUserActions()) - d := NewListPostActionControllerDelegate(ControllerIdUser, modelSvc.GetBaseService(interfaces.ModelIdUser), getUserActions()) - ctx := newUserContext() - - return &userController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/version.go b/core/controllers/version.go deleted file mode 100644 index 83e3fe95..00000000 --- a/core/controllers/version.go +++ /dev/null @@ -1,23 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/config" - "github.com/gin-gonic/gin" - "net/http" -) - -func GetVersion(c *gin.Context) { - HandleSuccessWithData(c, config.GetVersion()) -} - -func getVersionActions() []Action { - return []Action{ - { - Method: http.MethodGet, - Path: "", - HandlerFunc: GetVersion, - }, - } -} - -var VersionController ActionController diff --git a/core/routes/group.go b/core/routes/group.go deleted file mode 100644 index cade1af4..00000000 --- a/core/routes/group.go +++ /dev/null @@ -1,20 +0,0 @@ -package routes - -import ( - "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/gin-gonic/gin" -) - -type RouterGroups struct { - AuthGroup *gin.RouterGroup - AnonymousGroup *gin.RouterGroup - FilerGroup *gin.RouterGroup -} - -func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) { - return &RouterGroups{ - AuthGroup: app.Group("/", middlewares.AuthorizationMiddleware()), - AnonymousGroup: app.Group("/"), - FilerGroup: app.Group("/filer", middlewares.FilerAuthorizationMiddleware()), - } -} diff --git a/core/routes/router.go b/core/routes/router.go deleted file mode 100644 index c82603bc..00000000 --- a/core/routes/router.go +++ /dev/null @@ -1,178 +0,0 @@ -package routes - -import ( - "fmt" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/controllers" - "github.com/gin-gonic/gin" - "net/http" - "path" -) - -type RouterServiceInterface interface { - RegisterControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListController) - RegisterHandlerToGroup(group *gin.RouterGroup, path string, method string, handler gin.HandlerFunc) -} - -type RouterService struct { - app *gin.Engine -} - -func NewRouterService(app *gin.Engine) (svc *RouterService) { - return &RouterService{ - app: app, - } -} - -func (svc *RouterService) RegisterControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.BasicController) { - group.GET(basePath, ctr.Get) - group.POST(basePath, ctr.Post) - group.PUT(basePath, ctr.Put) - group.DELETE(basePath, ctr.Delete) -} - -func (svc *RouterService) RegisterListControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListController) { - group.GET(basePath+"/:id", ctr.Get) - group.GET(basePath, ctr.GetList) - group.POST(basePath, ctr.Post) - group.POST(basePath+"/batch", ctr.PostList) - group.PUT(basePath+"/:id", ctr.Put) - group.PUT(basePath, ctr.PutList) - group.DELETE(basePath+"/:id", ctr.Delete) - group.DELETE(basePath, ctr.DeleteList) -} - -func (svc *RouterService) RegisterActionControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ActionController) { - for _, action := range ctr.Actions() { - routerPath := path.Join(basePath, action.Path) - switch action.Method { - case http.MethodGet: - group.GET(routerPath, action.HandlerFunc) - case http.MethodPost: - group.POST(routerPath, action.HandlerFunc) - case http.MethodPut: - group.PUT(routerPath, action.HandlerFunc) - case http.MethodDelete: - group.DELETE(routerPath, action.HandlerFunc) - } - } -} - -func (svc *RouterService) RegisterListActionControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListActionController) { - svc.RegisterListControllerToGroup(group, basePath, ctr) - svc.RegisterActionControllerToGroup(group, basePath, ctr) -} - -func (svc *RouterService) RegisterHandlerToGroup(group *gin.RouterGroup, path string, method string, handler gin.HandlerFunc) { - switch method { - case http.MethodGet: - group.GET(path, handler) - case http.MethodPost: - group.POST(path, handler) - case http.MethodPut: - group.PUT(path, handler) - case http.MethodDelete: - group.DELETE(path, handler) - default: - log.Warn(fmt.Sprintf("%s is not a valid http method", method)) - } -} - -func InitRoutes(app *gin.Engine) (err error) { - // routes groups - groups := NewRouterGroups(app) - - // router service - svc := NewRouterService(app) - - // register routes - registerRoutesAnonymousGroup(svc, groups) - registerRoutesAuthGroup(svc, groups) - registerRoutesFilterGroup(svc, groups) - - return nil -} - -func registerRoutesAnonymousGroup(svc *RouterService, groups *RouterGroups) { - // login - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/", controllers.LoginController) - - // version - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/version", controllers.VersionController) - - // system info - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/system-info", controllers.SystemInfoController) - - // demo - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/demo", controllers.DemoController) - - // sync - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/sync", controllers.SyncController) -} - -func registerRoutesAuthGroup(svc *RouterService, groups *RouterGroups) { - // node - svc.RegisterListControllerToGroup(groups.AuthGroup, "/nodes", controllers.NodeController) - - // project - svc.RegisterListControllerToGroup(groups.AuthGroup, "/projects", controllers.ProjectController) - - // user - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/users", controllers.UserController) - - // spider - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/spiders", controllers.SpiderController) - - // task - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/tasks", controllers.TaskController) - - // tag - svc.RegisterListControllerToGroup(groups.AuthGroup, "/tags", controllers.TagController) - - // setting - svc.RegisterListControllerToGroup(groups.AuthGroup, "/settings", controllers.SettingController) - - // data collection - svc.RegisterListControllerToGroup(groups.AuthGroup, "/data/collections", controllers.DataCollectionController) - - // result - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/results", controllers.ResultController) - - // schedule - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/schedules", controllers.ScheduleController) - - // stats - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/stats", controllers.StatsController) - - // token - svc.RegisterListControllerToGroup(groups.AuthGroup, "/tokens", controllers.TokenController) - - // git - svc.RegisterListControllerToGroup(groups.AuthGroup, "/gits", controllers.GitController) - - // role - svc.RegisterListControllerToGroup(groups.AuthGroup, "/roles", controllers.RoleController) - - // permission - svc.RegisterListControllerToGroup(groups.AuthGroup, "/permissions", controllers.PermissionController) - - // export - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/export", controllers.ExportController) - - // notification - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/notifications", controllers.NotificationController) - - // filter - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/filters", controllers.FilterController) - - // data sources - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/data-sources", controllers.DataSourceController) - - // environments - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/environments", controllers.EnvironmentController) -} - -func registerRoutesFilterGroup(svc *RouterService, groups *RouterGroups) { - // filer - svc.RegisterActionControllerToGroup(groups.FilerGroup, "", controllers.FilerController) -} diff --git a/core/routes/router_test.go b/core/routes/router_test.go deleted file mode 100644 index 7b72c852..00000000 --- a/core/routes/router_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package routes - -import ( - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" - "net/http" - "testing" - "time" -) - -func TestInitRoutes(t *testing.T) { - app := gin.New() - err := InitRoutes(app) - require.Nil(t, err) - - srv := &http.Server{ - Handler: app, - Addr: "localhost:8000", - } - go func() { - err = srv.ListenAndServe() - require.Nil(t, err) - }() - - time.Sleep(5 * time.Second) -}