mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-21 17:21:09 +01:00
refactor: code cleanup
This commit is contained in:
Marvin Zhang
@@ -1,103 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/container"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/crawlab-team/crawlab/db/mongo"
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
mongo2 "go.mongodb.org/mongo-driver/mongo"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
var DataCollectionController *dataCollectionController
|
||||
|
||||
func getDataCollectionActions() []Action {
|
||||
ctx := newDataCollectionContext()
|
||||
return []Action{
|
||||
{
|
||||
Method: http.MethodPost,
|
||||
Path: "/:id/indexes",
|
||||
HandlerFunc: ctx.postIndexes,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type dataCollectionController struct {
|
||||
ListActionControllerDelegate
|
||||
d ListActionControllerDelegate
|
||||
ctx *dataCollectionContext
|
||||
}
|
||||
|
||||
type dataCollectionContext struct {
|
||||
modelSvc service.ModelService
|
||||
resultSvc interfaces.ResultService
|
||||
}
|
||||
|
||||
func (ctx *dataCollectionContext) postIndexes(c *gin.Context) {
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
dc, err := ctx.modelSvc.GetDataCollectionById(id)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, f := range dc.Fields {
|
||||
if err := mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{
|
||||
Keys: f.Key,
|
||||
}); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
var _dataCollectionCtx *dataCollectionContext
|
||||
|
||||
func newDataCollectionContext() *dataCollectionContext {
|
||||
if _dataCollectionCtx != nil {
|
||||
return _dataCollectionCtx
|
||||
}
|
||||
|
||||
// context
|
||||
ctx := &dataCollectionContext{}
|
||||
|
||||
// dependency injection
|
||||
if err := container.GetContainer().Invoke(func(
|
||||
modelSvc service.ModelService,
|
||||
) {
|
||||
ctx.modelSvc = modelSvc
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
_dataCollectionCtx = ctx
|
||||
|
||||
return ctx
|
||||
}
|
||||
|
||||
func newDataCollectionController() *dataCollectionController {
|
||||
actions := getDataCollectionActions()
|
||||
modelSvc, err := service.GetService()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
ctr := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions)
|
||||
d := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions)
|
||||
ctx := newDataCollectionContext()
|
||||
|
||||
return &dataCollectionController{
|
||||
ListActionControllerDelegate: *ctr,
|
||||
d: *d,
|
||||
ctx: ctx,
|
||||
}
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/ds"
|
||||
"github.com/crawlab-team/crawlab/core/errors"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
interfaces2 "github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/crawlab-team/crawlab/core/models/delegate"
|
||||
"github.com/crawlab-team/crawlab/core/models/models"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/crawlab-team/crawlab/core/utils"
|
||||
"github.com/crawlab-team/crawlab/db/mongo"
|
||||
"github.com/crawlab-team/crawlab/trace"
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
mongo2 "go.mongodb.org/mongo-driver/mongo"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
var DataSourceController *dataSourceController
|
||||
|
||||
func getDataSourceActions() []Action {
|
||||
ctx := newDataSourceContext()
|
||||
return []Action{
|
||||
{
|
||||
Path: "/:id/change-password",
|
||||
Method: http.MethodPost,
|
||||
HandlerFunc: ctx.changePassword,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type dataSourceController struct {
|
||||
ListActionControllerDelegate
|
||||
d ListActionControllerDelegate
|
||||
ctx *dataSourceContext
|
||||
}
|
||||
|
||||
func (ctr *dataSourceController) Post(c *gin.Context) {
|
||||
// data source
|
||||
var _ds models.DataSource
|
||||
if err := c.ShouldBindJSON(&_ds); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// add data source to db
|
||||
if err := mongo.RunTransaction(func(ctx mongo2.SessionContext) error {
|
||||
if err := delegate.NewModelDelegate(&_ds).Add(); err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
pwd, err := utils.EncryptAES(_ds.Password)
|
||||
if err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
p := models.Password{Id: _ds.Id, Password: pwd}
|
||||
if err := delegate.NewModelDelegate(&p).Add(); err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// check data source status
|
||||
go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }()
|
||||
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctr *dataSourceController) Put(c *gin.Context) {
|
||||
// data source
|
||||
var _ds models.DataSource
|
||||
if err := c.ShouldBindJSON(&_ds); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := delegate.NewModelDelegate(&_ds).Save(); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// check data source status
|
||||
go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }()
|
||||
}
|
||||
|
||||
type dataSourceContext struct {
|
||||
dsSvc interfaces.DataSourceService
|
||||
}
|
||||
|
||||
var _dataSourceCtx *dataSourceContext
|
||||
|
||||
func newDataSourceContext() *dataSourceContext {
|
||||
if _dataSourceCtx != nil {
|
||||
return _dataSourceCtx
|
||||
}
|
||||
dsSvc, err := ds.GetDataSourceService()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
_dataSourceCtx = &dataSourceContext{
|
||||
dsSvc: dsSvc,
|
||||
}
|
||||
return _dataSourceCtx
|
||||
}
|
||||
|
||||
func (ctx *dataSourceContext) changePassword(c *gin.Context) {
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
var payload map[string]string
|
||||
if err := c.ShouldBindJSON(&payload); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
password, ok := payload["password"]
|
||||
if !ok {
|
||||
HandleErrorBadRequest(c, errors.ErrorDataSourceMissingRequiredFields)
|
||||
return
|
||||
}
|
||||
if err := ctx.dsSvc.ChangePassword(id, password); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func newDataSourceController() *dataSourceController {
|
||||
actions := getDataSourceActions()
|
||||
modelSvc, err := service.GetService()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
ctr := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions)
|
||||
d := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions)
|
||||
ctx := newDataSourceContext()
|
||||
|
||||
return &dataSourceController{
|
||||
ListActionControllerDelegate: *ctr,
|
||||
d: *d,
|
||||
ctx: ctx,
|
||||
}
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/utils"
|
||||
"github.com/crawlab-team/crawlab/trace"
|
||||
"github.com/gin-gonic/gin"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func getDemoActions() []Action {
|
||||
ctx := newDemoContext()
|
||||
return []Action{
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/import",
|
||||
HandlerFunc: ctx.import_,
|
||||
},
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/reimport",
|
||||
HandlerFunc: ctx.reimport,
|
||||
},
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/cleanup",
|
||||
HandlerFunc: ctx.cleanup,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type demoContext struct {
|
||||
}
|
||||
|
||||
func (ctx *demoContext) import_(c *gin.Context) {
|
||||
if err := utils.ImportDemo(); err != nil {
|
||||
trace.PrintError(err)
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctx *demoContext) reimport(c *gin.Context) {
|
||||
if err := utils.ReimportDemo(); err != nil {
|
||||
trace.PrintError(err)
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctx *demoContext) cleanup(c *gin.Context) {
|
||||
if err := utils.ReimportDemo(); err != nil {
|
||||
trace.PrintError(err)
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
var _demoCtx *demoContext
|
||||
|
||||
func newDemoContext() *demoContext {
|
||||
if _demoCtx != nil {
|
||||
return _demoCtx
|
||||
}
|
||||
|
||||
_demoCtx = &demoContext{}
|
||||
|
||||
return _demoCtx
|
||||
}
|
||||
|
||||
var DemoController ActionController
|
||||
@@ -1,127 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/crawlab-team/crawlab/core/constants"
|
||||
"github.com/crawlab-team/crawlab/core/export"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/gin-gonic/gin"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
var ExportController ActionController
|
||||
|
||||
func getExportActions() []Action {
|
||||
ctx := newExportContext()
|
||||
return []Action{
|
||||
{
|
||||
Method: http.MethodPost,
|
||||
Path: "/:type",
|
||||
HandlerFunc: ctx.postExport,
|
||||
},
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/:type/:id",
|
||||
HandlerFunc: ctx.getExport,
|
||||
},
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/:type/:id/download",
|
||||
HandlerFunc: ctx.getExportDownload,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type exportContext struct {
|
||||
csvSvc interfaces.ExportService
|
||||
jsonSvc interfaces.ExportService
|
||||
}
|
||||
|
||||
func (ctx *exportContext) postExport(c *gin.Context) {
|
||||
exportType := c.Param("type")
|
||||
exportTarget := c.Query("target")
|
||||
exportFilter, _ := GetFilter(c)
|
||||
|
||||
var exportId string
|
||||
var err error
|
||||
switch exportType {
|
||||
case constants.ExportTypeCsv:
|
||||
exportId, err = ctx.csvSvc.Export(exportType, exportTarget, exportFilter)
|
||||
case constants.ExportTypeJson:
|
||||
exportId, err = ctx.jsonSvc.Export(exportType, exportTarget, exportFilter)
|
||||
default:
|
||||
HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
HandleSuccessWithData(c, exportId)
|
||||
}
|
||||
|
||||
func (ctx *exportContext) getExport(c *gin.Context) {
|
||||
exportType := c.Param("type")
|
||||
exportId := c.Param("id")
|
||||
|
||||
var exp interfaces.Export
|
||||
var err error
|
||||
switch exportType {
|
||||
case constants.ExportTypeCsv:
|
||||
exp, err = ctx.csvSvc.GetExport(exportId)
|
||||
case constants.ExportTypeJson:
|
||||
exp, err = ctx.jsonSvc.GetExport(exportId)
|
||||
default:
|
||||
HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
|
||||
}
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
HandleSuccessWithData(c, exp)
|
||||
}
|
||||
|
||||
func (ctx *exportContext) getExportDownload(c *gin.Context) {
|
||||
exportType := c.Param("type")
|
||||
exportId := c.Param("id")
|
||||
|
||||
var exp interfaces.Export
|
||||
var err error
|
||||
switch exportType {
|
||||
case constants.ExportTypeCsv:
|
||||
exp, err = ctx.csvSvc.GetExport(exportId)
|
||||
case constants.ExportTypeJson:
|
||||
exp, err = ctx.jsonSvc.GetExport(exportId)
|
||||
default:
|
||||
HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
|
||||
}
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
switch exportType {
|
||||
case constants.ExportTypeCsv:
|
||||
c.Header("Content-Type", "text/csv")
|
||||
case constants.ExportTypeJson:
|
||||
c.Header("Content-Type", "text/plain")
|
||||
default:
|
||||
HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
|
||||
}
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", exp.GetDownloadPath()))
|
||||
c.File(exp.GetDownloadPath())
|
||||
}
|
||||
|
||||
func newExportContext() *exportContext {
|
||||
return &exportContext{
|
||||
csvSvc: export.GetCsvService(),
|
||||
jsonSvc: export.GetJsonService(),
|
||||
}
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"github.com/crawlab-team/crawlab/core/errors"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/imroc/req"
|
||||
"github.com/spf13/viper"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var FilerController ActionController
|
||||
|
||||
func getFilerActions() []Action {
|
||||
filerCtx := newFilerContext()
|
||||
return []Action{
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "*path",
|
||||
HandlerFunc: filerCtx.do,
|
||||
},
|
||||
{
|
||||
Method: http.MethodPost,
|
||||
Path: "*path",
|
||||
HandlerFunc: filerCtx.do,
|
||||
},
|
||||
{
|
||||
Method: http.MethodPut,
|
||||
Path: "*path",
|
||||
HandlerFunc: filerCtx.do,
|
||||
},
|
||||
{
|
||||
Method: http.MethodDelete,
|
||||
Path: "*path",
|
||||
HandlerFunc: filerCtx.do,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type filerContext struct {
|
||||
endpoint string
|
||||
}
|
||||
|
||||
func (ctx *filerContext) do(c *gin.Context) {
|
||||
// request path
|
||||
requestPath := strings.Replace(c.Request.URL.Path, "/filer", "", 1)
|
||||
|
||||
// request url
|
||||
requestUrl := fmt.Sprintf("%s%s", ctx.endpoint, requestPath)
|
||||
if c.Request.URL.RawQuery != "" {
|
||||
requestUrl += "?" + c.Request.URL.RawQuery
|
||||
}
|
||||
|
||||
// request body
|
||||
bufR := bufio.NewScanner(c.Request.Body)
|
||||
requestBody := req.BodyJSON(bufR.Bytes())
|
||||
|
||||
// request file uploads
|
||||
var requestFileUploads []req.FileUpload
|
||||
form, err := c.MultipartForm()
|
||||
if err == nil {
|
||||
for k, v := range form.File {
|
||||
for _, fh := range v {
|
||||
f, err := fh.Open()
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
requestFileUploads = append(requestFileUploads, req.FileUpload{
|
||||
FileName: fh.Filename,
|
||||
FieldName: k,
|
||||
File: f,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// request header
|
||||
requestHeader := req.Header{}
|
||||
for k, v := range c.Request.Header {
|
||||
if len(v) > 0 {
|
||||
requestHeader[k] = v[0]
|
||||
}
|
||||
}
|
||||
|
||||
// perform request
|
||||
res, err := req.Do(c.Request.Method, requestUrl, requestHeader, requestBody, requestFileUploads)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// status code check
|
||||
statusCode := res.Response().StatusCode
|
||||
if statusCode == http.StatusNotFound {
|
||||
HandleErrorNotFoundNoPrint(c, errors.ErrorControllerFilerNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// response
|
||||
for k, v := range res.Response().Header {
|
||||
if len(v) > 0 {
|
||||
c.Header(k, v[0])
|
||||
}
|
||||
}
|
||||
_, _ = c.Writer.Write(res.Bytes())
|
||||
c.AbortWithStatus(statusCode)
|
||||
}
|
||||
|
||||
var _filerCtx *filerContext
|
||||
|
||||
func newFilerContext() *filerContext {
|
||||
if _filerCtx != nil {
|
||||
return _filerCtx
|
||||
}
|
||||
|
||||
ctx := &filerContext{
|
||||
endpoint: "http://localhost:8888",
|
||||
}
|
||||
|
||||
if viper.GetString("fs.filer.proxy") != "" {
|
||||
ctx.endpoint = viper.GetString("fs.filer.proxy")
|
||||
}
|
||||
|
||||
_filerCtx = ctx
|
||||
|
||||
return ctx
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/entity"
|
||||
"github.com/crawlab-team/crawlab/db/mongo"
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
mongo2 "go.mongodb.org/mongo-driver/mongo"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
var FilterController ActionController
|
||||
|
||||
func getFilterActions() []Action {
|
||||
ctx := newFilterContext()
|
||||
return []Action{
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/:col",
|
||||
HandlerFunc: ctx.getColFieldOptions,
|
||||
},
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/:col/:value",
|
||||
HandlerFunc: ctx.getColFieldOptions,
|
||||
},
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/:col/:value/:label",
|
||||
HandlerFunc: ctx.getColFieldOptions,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type filterContext struct {
|
||||
}
|
||||
|
||||
func (ctx *filterContext) getColFieldOptions(c *gin.Context) {
|
||||
colName := c.Param("col")
|
||||
value := c.Param("value")
|
||||
if value == "" {
|
||||
value = "_id"
|
||||
}
|
||||
label := c.Param("label")
|
||||
if label == "" {
|
||||
label = "name"
|
||||
}
|
||||
query := MustGetFilterQuery(c)
|
||||
pipelines := mongo2.Pipeline{}
|
||||
if query != nil {
|
||||
pipelines = append(pipelines, bson.D{{"$match", query}})
|
||||
}
|
||||
pipelines = append(
|
||||
pipelines,
|
||||
bson.D{
|
||||
{
|
||||
"$group",
|
||||
bson.M{
|
||||
"_id": bson.M{
|
||||
"value": "$" + value,
|
||||
"label": "$" + label,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
pipelines = append(
|
||||
pipelines,
|
||||
bson.D{
|
||||
{
|
||||
"$project",
|
||||
bson.M{
|
||||
"value": "$_id.value",
|
||||
"label": bson.M{"$toString": "$_id.label"},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
pipelines = append(
|
||||
pipelines,
|
||||
bson.D{
|
||||
{
|
||||
"$sort",
|
||||
bson.D{
|
||||
{"value", 1},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
var options []entity.FilterSelectOption
|
||||
if err := mongo.GetMongoCol(colName).Aggregate(pipelines, nil).All(&options); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccessWithData(c, options)
|
||||
}
|
||||
|
||||
func newFilterContext() *filterContext {
|
||||
return &filterContext{}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
package controllers
|
||||
|
||||
var GitController ListController
|
||||
@@ -1,42 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
)
|
||||
|
||||
func InitControllers() (err error) {
|
||||
modelSvc, err := service.GetService()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
NodeController = newNodeController()
|
||||
ProjectController = newProjectController()
|
||||
SpiderController = newSpiderController()
|
||||
TaskController = newTaskController()
|
||||
UserController = newUserController()
|
||||
TagController = NewListControllerDelegate(ControllerIdTag, modelSvc.GetBaseService(interfaces.ModelIdTag))
|
||||
SettingController = newSettingController()
|
||||
LoginController = NewActionControllerDelegate(ControllerIdLogin, getLoginActions())
|
||||
DataCollectionController = newDataCollectionController()
|
||||
ResultController = NewActionControllerDelegate(ControllerIdResult, getResultActions())
|
||||
ScheduleController = newScheduleController()
|
||||
StatsController = NewActionControllerDelegate(ControllerIdStats, getStatsActions())
|
||||
TokenController = newTokenController()
|
||||
FilerController = NewActionControllerDelegate(ControllerIdFiler, getFilerActions())
|
||||
GitController = NewListControllerDelegate(ControllerIdGit, modelSvc.GetBaseService(interfaces.ModelIdGit))
|
||||
VersionController = NewActionControllerDelegate(ControllerIdVersion, getVersionActions())
|
||||
SystemInfoController = NewActionControllerDelegate(ControllerIdSystemInfo, getSystemInfoActions())
|
||||
DemoController = NewActionControllerDelegate(ControllerIdDemo, getDemoActions())
|
||||
RoleController = NewListControllerDelegate(ControllerIdRole, modelSvc.GetBaseService(interfaces.ModelIdRole))
|
||||
PermissionController = NewListControllerDelegate(ControllerIdPermission, modelSvc.GetBaseService(interfaces.ModelIdPermission))
|
||||
ExportController = NewActionControllerDelegate(ControllerIdExport, getExportActions())
|
||||
NotificationController = NewActionControllerDelegate(ControllerIdNotification, getNotificationActions())
|
||||
FilterController = NewActionControllerDelegate(ControllerIdFilter, getFilterActions())
|
||||
SyncController = NewActionControllerDelegate(ControllerIdSync, getSyncActions())
|
||||
DataSourceController = newDataSourceController()
|
||||
EnvironmentController = newEnvironmentController()
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/constants"
|
||||
"github.com/crawlab-team/crawlab/core/container"
|
||||
"github.com/crawlab-team/crawlab/core/errors"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/crawlab-team/crawlab/core/models/models"
|
||||
"github.com/gin-gonic/gin"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
var LoginController ActionController
|
||||
|
||||
func getLoginActions() []Action {
|
||||
loginCtx := newLoginContext()
|
||||
return []Action{
|
||||
{Method: http.MethodPost, Path: "/login", HandlerFunc: loginCtx.login},
|
||||
{Method: http.MethodPost, Path: "/logout", HandlerFunc: loginCtx.logout},
|
||||
}
|
||||
}
|
||||
|
||||
type loginContext struct {
|
||||
userSvc interfaces.UserService
|
||||
}
|
||||
|
||||
func (ctx *loginContext) login(c *gin.Context) {
|
||||
var u models.User
|
||||
if err := c.ShouldBindJSON(&u); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
token, loggedInUser, err := ctx.userSvc.Login(&interfaces.UserLoginOptions{
|
||||
Username: u.Username,
|
||||
Password: u.Password,
|
||||
})
|
||||
if err != nil {
|
||||
HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized)
|
||||
return
|
||||
}
|
||||
c.Set(constants.UserContextKey, loggedInUser)
|
||||
HandleSuccessWithData(c, token)
|
||||
}
|
||||
|
||||
func (ctx *loginContext) logout(c *gin.Context) {
|
||||
c.Set(constants.UserContextKey, nil)
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func newLoginContext() *loginContext {
|
||||
// context
|
||||
ctx := &loginContext{}
|
||||
|
||||
// dependency injection
|
||||
if err := container.GetContainer().Invoke(func(
|
||||
userSvc interfaces.UserService,
|
||||
) {
|
||||
ctx.userSvc = userSvc
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return ctx
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/constants"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/crawlab-team/crawlab/core/models/delegate"
|
||||
"github.com/crawlab-team/crawlab/core/models/models"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/crawlab-team/crawlab/trace"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
)
|
||||
|
||||
var NodeController *nodeController
|
||||
|
||||
type nodeController struct {
|
||||
ListControllerDelegate
|
||||
}
|
||||
|
||||
func (ctr *nodeController) Post(c *gin.Context) {
|
||||
var n models.Node
|
||||
if err := c.ShouldBindJSON(&n); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := ctr._post(c, &n); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctr *nodeController) PostList(c *gin.Context) {
|
||||
// bind
|
||||
var docs []models.Node
|
||||
if err := c.ShouldBindJSON(&docs); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// success ids
|
||||
var ids []primitive.ObjectID
|
||||
|
||||
// iterate nodes
|
||||
for _, n := range docs {
|
||||
if err := ctr._post(c, &n); err != nil {
|
||||
trace.PrintError(err)
|
||||
continue
|
||||
}
|
||||
ids = append(ids, n.Id)
|
||||
}
|
||||
|
||||
// success
|
||||
HandleSuccessWithData(c, docs)
|
||||
}
|
||||
|
||||
func (ctr *nodeController) _post(c *gin.Context, n *models.Node) (err error) {
|
||||
// set default key
|
||||
if n.Key == "" {
|
||||
id, err := uuid.NewUUID()
|
||||
if err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
n.Key = id.String()
|
||||
}
|
||||
|
||||
// set default status
|
||||
if n.Status == "" {
|
||||
n.Status = constants.NodeStatusUnregistered
|
||||
}
|
||||
|
||||
// add
|
||||
if err := delegate.NewModelDelegate(n, GetUserFromContext(c)).Add(); err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func newNodeController() *nodeController {
|
||||
modelSvc, err := service.GetService()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
ctr := NewListControllerDelegate(ControllerIdNode, modelSvc.GetBaseService(interfaces.ModelIdNode))
|
||||
|
||||
return &nodeController{
|
||||
ListControllerDelegate: *ctr,
|
||||
}
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/notification"
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
var NotificationController ActionController
|
||||
|
||||
func getNotificationActions() []Action {
|
||||
ctx := newNotificationContext()
|
||||
return []Action{
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/settings",
|
||||
HandlerFunc: ctx.GetSettingList,
|
||||
},
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/settings/:id",
|
||||
HandlerFunc: ctx.GetSetting,
|
||||
},
|
||||
{
|
||||
Method: http.MethodPost,
|
||||
Path: "/settings",
|
||||
HandlerFunc: ctx.PostSetting,
|
||||
},
|
||||
{
|
||||
Method: http.MethodPut,
|
||||
Path: "/settings/:id",
|
||||
HandlerFunc: ctx.PutSetting,
|
||||
},
|
||||
{
|
||||
Method: http.MethodDelete,
|
||||
Path: "/settings/:id",
|
||||
HandlerFunc: ctx.DeleteSetting,
|
||||
},
|
||||
{
|
||||
Method: http.MethodPost,
|
||||
Path: "/settings/:id/enable",
|
||||
HandlerFunc: ctx.EnableSetting,
|
||||
},
|
||||
{
|
||||
Method: http.MethodPost,
|
||||
Path: "/settings/:id/disable",
|
||||
HandlerFunc: ctx.DisableSetting,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type notificationContext struct {
|
||||
svc *notification.Service
|
||||
}
|
||||
|
||||
func (ctx *notificationContext) GetSettingList(c *gin.Context) {
|
||||
query := MustGetFilterQuery(c)
|
||||
pagination := MustGetPagination(c)
|
||||
sort := MustGetSortOption(c)
|
||||
res, total, err := ctx.svc.GetSettingList(query, pagination, sort)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccessWithListData(c, res, total)
|
||||
}
|
||||
|
||||
func (ctx *notificationContext) GetSetting(c *gin.Context) {
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
res, err := ctx.svc.GetSetting(id)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccessWithData(c, res)
|
||||
}
|
||||
|
||||
func (ctx *notificationContext) PostSetting(c *gin.Context) {
|
||||
var s notification.NotificationSetting
|
||||
if err := c.ShouldBindJSON(&s); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
if err := ctx.svc.PosSetting(&s); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctx *notificationContext) PutSetting(c *gin.Context) {
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
var s notification.NotificationSetting
|
||||
if err := c.ShouldBindJSON(&s); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
if err := ctx.svc.PutSetting(id, s); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctx *notificationContext) DeleteSetting(c *gin.Context) {
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
if err := ctx.svc.DeleteSetting(id); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctx *notificationContext) EnableSetting(c *gin.Context) {
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
if err := ctx.svc.EnableSetting(id); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctx *notificationContext) DisableSetting(c *gin.Context) {
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
if err := ctx.svc.DisableSetting(id); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func newNotificationContext() *notificationContext {
|
||||
ctx := ¬ificationContext{
|
||||
svc: notification.GetService(),
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
package controllers
|
||||
|
||||
var PermissionController ListController
|
||||
@@ -1,103 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/errors"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/crawlab-team/crawlab/core/models/models"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
)
|
||||
|
||||
var ProjectController *projectController
|
||||
|
||||
type projectController struct {
|
||||
ListControllerDelegate
|
||||
}
|
||||
|
||||
func (ctr *projectController) GetList(c *gin.Context) {
|
||||
// get all if query field "all" is set true
|
||||
all := MustGetFilterAll(c)
|
||||
if all {
|
||||
ctr.getAll(c)
|
||||
return
|
||||
}
|
||||
|
||||
// get list
|
||||
list, total, err := ctr.getList(c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
data := list.GetModels()
|
||||
|
||||
// check empty list
|
||||
if len(list.GetModels()) == 0 {
|
||||
HandleSuccessWithListData(c, nil, 0)
|
||||
return
|
||||
}
|
||||
|
||||
// project ids
|
||||
var ids []primitive.ObjectID
|
||||
|
||||
// count cache
|
||||
cache := map[primitive.ObjectID]int{}
|
||||
|
||||
// iterate
|
||||
for _, d := range data {
|
||||
p, ok := d.(*models.Project)
|
||||
if !ok {
|
||||
HandleErrorInternalServerError(c, errors.ErrorControllerInvalidType)
|
||||
return
|
||||
}
|
||||
ids = append(ids, p.Id)
|
||||
cache[p.Id] = 0
|
||||
}
|
||||
|
||||
// spiders
|
||||
modelSvc, err := service.NewService()
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
spiders, err := modelSvc.GetSpiderList(bson.M{
|
||||
"project_id": bson.M{
|
||||
"$in": ids,
|
||||
},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
for _, s := range spiders {
|
||||
_, ok := cache[s.ProjectId]
|
||||
if !ok {
|
||||
HandleErrorInternalServerError(c, errors.ErrorControllerMissingInCache)
|
||||
return
|
||||
}
|
||||
cache[s.ProjectId]++
|
||||
}
|
||||
|
||||
// assign
|
||||
var projects []models.Project
|
||||
for _, d := range data {
|
||||
p := d.(*models.Project)
|
||||
p.Spiders = cache[p.Id]
|
||||
projects = append(projects, *p)
|
||||
}
|
||||
|
||||
HandleSuccessWithListData(c, projects, total)
|
||||
}
|
||||
|
||||
func newProjectController() *projectController {
|
||||
modelSvc, err := service.GetService()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
ctr := NewListControllerDelegate(ControllerIdProject, modelSvc.GetBaseService(interfaces.ModelIdProject))
|
||||
|
||||
return &projectController{
|
||||
ListControllerDelegate: *ctr,
|
||||
}
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/models/models"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/crawlab-team/crawlab/core/result"
|
||||
"github.com/crawlab-team/crawlab/core/utils"
|
||||
"github.com/crawlab-team/crawlab/db/generic"
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
mongo2 "go.mongodb.org/mongo-driver/mongo"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
var ResultController ActionController
|
||||
|
||||
func getResultActions() []Action {
|
||||
var resultCtx = newResultContext()
|
||||
return []Action{
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/:id",
|
||||
HandlerFunc: resultCtx.getList,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type resultContext struct {
|
||||
modelSvc service.ModelService
|
||||
}
|
||||
|
||||
func (ctx *resultContext) getList(c *gin.Context) {
|
||||
// data collection id
|
||||
dcId, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// data source id
|
||||
var dsId primitive.ObjectID
|
||||
dsIdStr := c.Query("data_source_id")
|
||||
if dsIdStr != "" {
|
||||
dsId, err = primitive.ObjectIDFromHex(dsIdStr)
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// data collection
|
||||
dc, err := ctx.modelSvc.GetDataCollectionById(dcId)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// data source
|
||||
ds, err := ctx.modelSvc.GetDataSourceById(dsId)
|
||||
if err != nil {
|
||||
if err.Error() == mongo2.ErrNoDocuments.Error() {
|
||||
ds = &models.DataSource{}
|
||||
} else {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// spider
|
||||
sq := bson.M{
|
||||
"col_id": dc.Id,
|
||||
"data_source_id": ds.Id,
|
||||
}
|
||||
s, err := ctx.modelSvc.GetSpider(sq, nil)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// service
|
||||
svc, err := result.GetResultService(s.Id)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// params
|
||||
pagination := MustGetPagination(c)
|
||||
query := ctx.getListQuery(c)
|
||||
|
||||
// get results
|
||||
data, err := svc.List(query, &generic.ListOptions{
|
||||
Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}},
|
||||
Skip: pagination.Size * (pagination.Page - 1),
|
||||
Limit: pagination.Size,
|
||||
})
|
||||
if err != nil {
|
||||
if err.Error() == mongo2.ErrNoDocuments.Error() {
|
||||
HandleSuccessWithListData(c, nil, 0)
|
||||
return
|
||||
}
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// validate results
|
||||
if len(data) == 0 {
|
||||
HandleSuccessWithListData(c, nil, 0)
|
||||
return
|
||||
}
|
||||
|
||||
// total count
|
||||
total, err := svc.Count(query)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// response
|
||||
HandleSuccessWithListData(c, data, total)
|
||||
}
|
||||
|
||||
func (ctx *resultContext) getListQuery(c *gin.Context) (q generic.ListQuery) {
|
||||
f, err := GetFilter(c)
|
||||
if err != nil {
|
||||
return q
|
||||
}
|
||||
for _, cond := range f.Conditions {
|
||||
q = append(q, generic.ListQueryCondition{
|
||||
Key: cond.Key,
|
||||
Op: cond.Op,
|
||||
Value: utils.NormalizeObjectId(cond.Value),
|
||||
})
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
func newResultContext() *resultContext {
|
||||
// context
|
||||
ctx := &resultContext{}
|
||||
|
||||
var err error
|
||||
ctx.modelSvc, err = service.NewService()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return ctx
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
package controllers
|
||||
|
||||
var RoleController ListController
|
||||
@@ -357,13 +357,6 @@ func InitRoutes(app *gin.Engine) (err error) {
|
||||
HandlerFunc: GetSystemInfo,
|
||||
},
|
||||
})
|
||||
RegisterActions(groups.AnonymousGroup, "/version", []Action{
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "",
|
||||
HandlerFunc: GetVersion,
|
||||
},
|
||||
})
|
||||
RegisterActions(groups.AnonymousGroup, "/", []Action{
|
||||
{
|
||||
Method: http.MethodPost,
|
||||
|
||||
@@ -1,221 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/container"
|
||||
"github.com/crawlab-team/crawlab/core/errors"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/crawlab-team/crawlab/core/models/delegate"
|
||||
"github.com/crawlab-team/crawlab/core/models/models"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
var ScheduleController *scheduleController
|
||||
|
||||
func getScheduleActions() []Action {
|
||||
scheduleCtx := newScheduleContext()
|
||||
return []Action{
|
||||
{
|
||||
Method: http.MethodPost,
|
||||
Path: "/:id/enable",
|
||||
HandlerFunc: scheduleCtx.enable,
|
||||
},
|
||||
{
|
||||
Method: http.MethodPost,
|
||||
Path: "/:id/disable",
|
||||
HandlerFunc: scheduleCtx.disable,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type scheduleController struct {
|
||||
ListActionControllerDelegate
|
||||
d ListActionControllerDelegate
|
||||
ctx *scheduleContext
|
||||
}
|
||||
|
||||
func (ctr *scheduleController) Post(c *gin.Context) {
|
||||
var s models.Schedule
|
||||
if err := c.ShouldBindJSON(&s); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
if err := delegate.NewModelDelegate(&s, GetUserFromContext(c)).Add(); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
if s.Enabled {
|
||||
if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
HandleSuccessWithData(c, s)
|
||||
}
|
||||
|
||||
func (ctr *scheduleController) Put(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
oid, err := primitive.ObjectIDFromHex(id)
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
var s models.Schedule
|
||||
if err := c.ShouldBindJSON(&s); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
if s.GetId() != oid {
|
||||
HandleErrorBadRequest(c, errors.ErrorHttpBadRequest)
|
||||
return
|
||||
}
|
||||
if err := delegate.NewModelDelegate(&s).Save(); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
if s.Enabled {
|
||||
if err := ctr.ctx.scheduleSvc.Disable(&s, GetUserFromContext(c)); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
HandleSuccessWithData(c, s)
|
||||
}
|
||||
|
||||
func (ctr *scheduleController) Delete(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
oid, err := primitive.ObjectIDFromHex(id)
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
s, err := ctr.ctx.modelSvc.GetScheduleById(oid)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
if err := ctr.ctx.scheduleSvc.Disable(s); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
if err := delegate.NewModelDelegate(s, GetUserFromContext(c)).Delete(); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (ctr *scheduleController) DeleteList(c *gin.Context) {
|
||||
payload, err := NewJsonBinder(interfaces.ModelIdSchedule).BindBatchRequestPayload(c)
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
for _, id := range payload.Ids {
|
||||
s, err := ctr.ctx.modelSvc.GetScheduleById(id)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
if err := ctr.ctx.scheduleSvc.Disable(s); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdSchedule).DeleteList(bson.M{
|
||||
"_id": bson.M{
|
||||
"$in": payload.Ids,
|
||||
},
|
||||
}); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctx *scheduleContext) enable(c *gin.Context) {
|
||||
s, err := ctx._getSchedule(c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := ctx.scheduleSvc.Enable(s, GetUserFromContext(c)); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctx *scheduleContext) disable(c *gin.Context) {
|
||||
s, err := ctx._getSchedule(c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := ctx.scheduleSvc.Disable(s, GetUserFromContext(c)); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctx *scheduleContext) _getSchedule(c *gin.Context) (s *models.Schedule, err error) {
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
s, err = ctx.modelSvc.GetScheduleById(id)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
type scheduleContext struct {
|
||||
modelSvc service.ModelService
|
||||
scheduleSvc interfaces.ScheduleService
|
||||
}
|
||||
|
||||
func newScheduleContext() *scheduleContext {
|
||||
// context
|
||||
ctx := &scheduleContext{}
|
||||
|
||||
// dependency injection
|
||||
if err := container.GetContainer().Invoke(func(
|
||||
modelSvc service.ModelService,
|
||||
scheduleSvc interfaces.ScheduleService,
|
||||
) {
|
||||
ctx.modelSvc = modelSvc
|
||||
ctx.scheduleSvc = scheduleSvc
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return ctx
|
||||
}
|
||||
|
||||
func newScheduleController() *scheduleController {
|
||||
actions := getScheduleActions()
|
||||
modelSvc, err := service.GetService()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
ctr := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions)
|
||||
d := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions)
|
||||
ctx := newScheduleContext()
|
||||
|
||||
return &scheduleController{
|
||||
ListActionControllerDelegate: *ctr,
|
||||
d: *d,
|
||||
ctx: ctx,
|
||||
}
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/crawlab-team/crawlab/core/models/delegate"
|
||||
"github.com/crawlab-team/crawlab/core/models/models"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
var SettingController *settingController
|
||||
|
||||
type settingController struct {
|
||||
ListControllerDelegate
|
||||
}
|
||||
|
||||
func (ctr *settingController) Get(c *gin.Context) {
|
||||
// key
|
||||
key := c.Param("id")
|
||||
|
||||
// model service
|
||||
modelSvc, err := service.NewService()
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// setting
|
||||
s, err := modelSvc.GetSettingByKey(key, nil)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
HandleSuccessWithData(c, s)
|
||||
}
|
||||
|
||||
func (ctr *settingController) Put(c *gin.Context) {
|
||||
// key
|
||||
key := c.Param("id")
|
||||
|
||||
// settings
|
||||
var s models.Setting
|
||||
if err := c.ShouldBindJSON(&s); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// model service
|
||||
modelSvc, err := service.NewService()
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// setting
|
||||
_s, err := modelSvc.GetSettingByKey(key, nil)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// save
|
||||
_s.Value = s.Value
|
||||
if err := delegate.NewModelDelegate(_s).Save(); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func newSettingController() *settingController {
|
||||
modelSvc, err := service.GetService()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
ctr := NewListControllerDelegate(ControllerIdSetting, modelSvc.GetBaseService(interfaces.ModelIdSetting))
|
||||
|
||||
return &settingController{
|
||||
ListControllerDelegate: *ctr,
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,87 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/container"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
var StatsController ActionController
|
||||
|
||||
func getStatsActions() []Action {
|
||||
statsCtx := newStatsContext()
|
||||
return []Action{
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/overview",
|
||||
HandlerFunc: statsCtx.getOverview,
|
||||
},
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/daily",
|
||||
HandlerFunc: statsCtx.getDaily,
|
||||
},
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/tasks",
|
||||
HandlerFunc: statsCtx.getTasks,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type statsContext struct {
|
||||
statsSvc interfaces.StatsService
|
||||
defaultQuery bson.M
|
||||
}
|
||||
|
||||
func (svc *statsContext) getOverview(c *gin.Context) {
|
||||
data, err := svc.statsSvc.GetOverviewStats(svc.defaultQuery)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccessWithData(c, data)
|
||||
}
|
||||
|
||||
func (svc *statsContext) getDaily(c *gin.Context) {
|
||||
data, err := svc.statsSvc.GetDailyStats(svc.defaultQuery)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccessWithData(c, data)
|
||||
}
|
||||
|
||||
func (svc *statsContext) getTasks(c *gin.Context) {
|
||||
data, err := svc.statsSvc.GetTaskStats(svc.defaultQuery)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccessWithData(c, data)
|
||||
}
|
||||
|
||||
func newStatsContext() *statsContext {
|
||||
// context
|
||||
ctx := &statsContext{
|
||||
defaultQuery: bson.M{
|
||||
"create_ts": bson.M{
|
||||
"$gte": time.Now().Add(-30 * 24 * time.Hour),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// dependency injection
|
||||
if err := container.GetContainer().Invoke(func(
|
||||
statsSvc interfaces.StatsService,
|
||||
) {
|
||||
ctx.statsSvc = statsSvc
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return ctx
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/utils"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/spf13/viper"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
var SyncController ActionController
|
||||
|
||||
func getSyncActions() []Action {
|
||||
var ctx = newSyncContext()
|
||||
return []Action{
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/:id/scan",
|
||||
HandlerFunc: ctx.scan,
|
||||
},
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/:id/download",
|
||||
HandlerFunc: ctx.download,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type syncContext struct {
|
||||
}
|
||||
|
||||
func (ctx *syncContext) scan(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
path := c.Query("path")
|
||||
dir := ctx._getDir(id, path)
|
||||
files, err := utils.ScanDirectory(dir)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
c.AbortWithStatusJSON(http.StatusOK, files)
|
||||
}
|
||||
|
||||
func (ctx *syncContext) download(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
filePath := c.Query("path")
|
||||
dir := ctx._getDir(id, "")
|
||||
c.File(filepath.Join(dir, filePath))
|
||||
}
|
||||
|
||||
func (ctx *syncContext) _getDir(id string, path string) string {
|
||||
workspacePath := viper.GetString("workspace")
|
||||
return filepath.Join(workspacePath, id, path)
|
||||
}
|
||||
|
||||
func newSyncContext() syncContext {
|
||||
return syncContext{}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/entity"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/spf13/viper"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func getSystemInfo(c *gin.Context) {
|
||||
info := &entity.SystemInfo{
|
||||
Edition: viper.GetString("info.edition"),
|
||||
Version: viper.GetString("info.version"),
|
||||
}
|
||||
HandleSuccessWithData(c, info)
|
||||
}
|
||||
|
||||
func getSystemInfoActions() []Action {
|
||||
return []Action{
|
||||
{
|
||||
Path: "",
|
||||
Method: http.MethodGet,
|
||||
HandlerFunc: getSystemInfo,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
var SystemInfoController ActionController
|
||||
@@ -8,8 +8,8 @@ import (
|
||||
|
||||
func GetSystemInfo(c *gin.Context) {
|
||||
info := &entity.SystemInfo{
|
||||
Edition: viper.GetString("info.edition"),
|
||||
Version: viper.GetString("info.version"),
|
||||
Edition: viper.GetString("edition"),
|
||||
Version: viper.GetString("version"),
|
||||
}
|
||||
HandleSuccessWithData(c, info)
|
||||
}
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
package controllers
|
||||
|
||||
var TagController ListController
|
||||
@@ -1,534 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/constants"
|
||||
"github.com/crawlab-team/crawlab/core/container"
|
||||
"github.com/crawlab-team/crawlab/core/errors"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
delegate2 "github.com/crawlab-team/crawlab/core/models/delegate"
|
||||
"github.com/crawlab-team/crawlab/core/models/models"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/crawlab-team/crawlab/core/result"
|
||||
"github.com/crawlab-team/crawlab/core/task/log"
|
||||
"github.com/crawlab-team/crawlab/core/utils"
|
||||
"github.com/crawlab-team/crawlab/db/generic"
|
||||
"github.com/crawlab-team/crawlab/db/mongo"
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
mongo2 "go.mongodb.org/mongo-driver/mongo"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var TaskController *taskController
|
||||
|
||||
func getTaskActions() []Action {
|
||||
taskCtx := newTaskContext()
|
||||
return []Action{
|
||||
{
|
||||
Method: http.MethodPost,
|
||||
Path: "/run",
|
||||
HandlerFunc: taskCtx.run,
|
||||
},
|
||||
{
|
||||
Method: http.MethodPost,
|
||||
Path: "/:id/restart",
|
||||
HandlerFunc: taskCtx.restart,
|
||||
},
|
||||
{
|
||||
Method: http.MethodPost,
|
||||
Path: "/:id/cancel",
|
||||
HandlerFunc: taskCtx.cancel,
|
||||
},
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/:id/logs",
|
||||
HandlerFunc: taskCtx.getLogs,
|
||||
},
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/:id/data",
|
||||
HandlerFunc: taskCtx.getData,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type taskController struct {
|
||||
ListActionControllerDelegate
|
||||
d ListActionControllerDelegate
|
||||
ctx *taskContext
|
||||
}
|
||||
|
||||
func (ctr *taskController) Get(c *gin.Context) {
|
||||
ctr.ctx.getWithStatsSpider(c)
|
||||
}
|
||||
|
||||
func (ctr *taskController) Delete(c *gin.Context) {
|
||||
if err := ctr.ctx._delete(c); err != nil {
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctr *taskController) GetList(c *gin.Context) {
|
||||
withStats := c.Query("stats")
|
||||
if withStats == "" {
|
||||
ctr.d.GetList(c)
|
||||
return
|
||||
}
|
||||
ctr.ctx.getListWithStats(c)
|
||||
}
|
||||
|
||||
func (ctr *taskController) DeleteList(c *gin.Context) {
|
||||
if err := ctr.ctx._deleteList(c); err != nil {
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
type taskContext struct {
|
||||
modelSvc service.ModelService
|
||||
modelTaskSvc interfaces.ModelBaseService
|
||||
modelTaskStatSvc interfaces.ModelBaseService
|
||||
adminSvc interfaces.SpiderAdminService
|
||||
schedulerSvc interfaces.TaskSchedulerService
|
||||
l log.Driver
|
||||
}
|
||||
|
||||
func (ctx *taskContext) run(c *gin.Context) {
|
||||
// task
|
||||
var t models.Task
|
||||
if err := c.ShouldBindJSON(&t); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// validate spider id
|
||||
if t.GetSpiderId().IsZero() {
|
||||
HandleErrorBadRequest(c, errors.ErrorTaskEmptySpiderId)
|
||||
return
|
||||
}
|
||||
|
||||
// spider
|
||||
s, err := ctx.modelSvc.GetSpiderById(t.GetSpiderId())
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// options
|
||||
opts := &interfaces.SpiderRunOptions{
|
||||
Mode: t.Mode,
|
||||
NodeIds: t.NodeIds,
|
||||
Cmd: t.Cmd,
|
||||
Param: t.Param,
|
||||
Priority: t.Priority,
|
||||
}
|
||||
|
||||
// user
|
||||
if u := GetUserFromContext(c); u != nil {
|
||||
opts.UserId = u.GetId()
|
||||
}
|
||||
|
||||
// run
|
||||
taskIds, err := ctx.adminSvc.Schedule(s.GetId(), opts)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
HandleSuccessWithData(c, taskIds)
|
||||
}
|
||||
|
||||
func (ctx *taskContext) restart(c *gin.Context) {
|
||||
// id
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// task
|
||||
t, err := ctx.modelSvc.GetTaskById(id)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// options
|
||||
opts := &interfaces.SpiderRunOptions{
|
||||
Mode: t.Mode,
|
||||
NodeIds: t.NodeIds,
|
||||
Cmd: t.Cmd,
|
||||
Param: t.Param,
|
||||
Priority: t.Priority,
|
||||
}
|
||||
|
||||
// user
|
||||
if u := GetUserFromContext(c); u != nil {
|
||||
opts.UserId = u.GetId()
|
||||
}
|
||||
|
||||
// run
|
||||
taskIds, err := ctx.adminSvc.Schedule(t.SpiderId, opts)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
HandleSuccessWithData(c, taskIds)
|
||||
}
|
||||
|
||||
func (ctx *taskContext) cancel(c *gin.Context) {
|
||||
// id
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// task
|
||||
t, err := ctx.modelSvc.GetTaskById(id)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// validate
|
||||
if !utils.IsCancellable(t.Status) {
|
||||
HandleErrorInternalServerError(c, errors.ErrorControllerNotCancellable)
|
||||
return
|
||||
}
|
||||
|
||||
// cancel
|
||||
if err := ctx.schedulerSvc.Cancel(id, GetUserFromContext(c)); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctx *taskContext) getLogs(c *gin.Context) {
|
||||
// id
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// pagination
|
||||
p, err := GetPagination(c)
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// logs
|
||||
logs, err := ctx.l.Find(id.Hex(), "", (p.Page-1)*p.Size, p.Size)
|
||||
if err != nil {
|
||||
if strings.HasSuffix(err.Error(), "Status:404 Not Found") {
|
||||
HandleSuccess(c)
|
||||
return
|
||||
}
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
total, err := ctx.l.Count(id.Hex(), "")
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
HandleSuccessWithListData(c, logs, total)
|
||||
}
|
||||
|
||||
func (ctx *taskContext) getListWithStats(c *gin.Context) {
|
||||
// params
|
||||
pagination := MustGetPagination(c)
|
||||
query := MustGetFilterQuery(c)
|
||||
sort := MustGetSortOption(c)
|
||||
|
||||
// get list
|
||||
list, err := ctx.modelTaskSvc.GetList(query, &mongo.FindOptions{
|
||||
Sort: sort,
|
||||
Skip: pagination.Size * (pagination.Page - 1),
|
||||
Limit: pagination.Size,
|
||||
})
|
||||
if err != nil {
|
||||
if err == mongo2.ErrNoDocuments {
|
||||
HandleErrorNotFound(c, err)
|
||||
} else {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// check empty list
|
||||
if len(list.GetModels()) == 0 {
|
||||
HandleSuccessWithListData(c, nil, 0)
|
||||
return
|
||||
}
|
||||
|
||||
// ids
|
||||
var ids []primitive.ObjectID
|
||||
for _, d := range list.GetModels() {
|
||||
t := d.(interfaces.Model)
|
||||
ids = append(ids, t.GetId())
|
||||
}
|
||||
|
||||
// total count
|
||||
total, err := ctx.modelTaskSvc.Count(query)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// stat list
|
||||
query = bson.M{
|
||||
"_id": bson.M{
|
||||
"$in": ids,
|
||||
},
|
||||
}
|
||||
stats, err := ctx.modelSvc.GetTaskStatList(query, nil)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// cache stat list to dict
|
||||
dict := map[primitive.ObjectID]models.TaskStat{}
|
||||
for _, s := range stats {
|
||||
dict[s.GetId()] = s
|
||||
}
|
||||
|
||||
// iterate list again
|
||||
var data []interface{}
|
||||
for _, d := range list.GetModels() {
|
||||
t := d.(*models.Task)
|
||||
s, ok := dict[t.GetId()]
|
||||
if ok {
|
||||
t.Stat = &s
|
||||
}
|
||||
data = append(data, *t)
|
||||
}
|
||||
|
||||
// response
|
||||
HandleSuccessWithListData(c, data, total)
|
||||
}
|
||||
|
||||
func (ctx *taskContext) getWithStatsSpider(c *gin.Context) {
|
||||
// id
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// task
|
||||
t, err := ctx.modelSvc.GetTaskById(id)
|
||||
if err == mongo2.ErrNoDocuments {
|
||||
HandleErrorNotFound(c, err)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// spider
|
||||
t.Spider, _ = ctx.modelSvc.GetSpiderById(t.SpiderId)
|
||||
|
||||
// skip if task status is pending
|
||||
if t.Status == constants.TaskStatusPending {
|
||||
HandleSuccessWithData(c, t)
|
||||
return
|
||||
}
|
||||
|
||||
// task stat
|
||||
t.Stat, _ = ctx.modelSvc.GetTaskStatById(id)
|
||||
|
||||
HandleSuccessWithData(c, t)
|
||||
}
|
||||
|
||||
func (ctx *taskContext) getData(c *gin.Context) {
|
||||
// id
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// pagination
|
||||
p, err := GetPagination(c)
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// task
|
||||
t, err := ctx.modelSvc.GetTaskById(id)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// result service
|
||||
resultSvc, err := result.GetResultService(t.SpiderId)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// query
|
||||
query := generic.ListQuery{
|
||||
generic.ListQueryCondition{
|
||||
Key: constants.TaskKey,
|
||||
Op: generic.OpEqual,
|
||||
Value: t.Id,
|
||||
},
|
||||
}
|
||||
|
||||
// list
|
||||
data, err := resultSvc.List(query, &generic.ListOptions{
|
||||
Skip: (p.Page - 1) * p.Size,
|
||||
Limit: p.Size,
|
||||
Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}},
|
||||
})
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// total
|
||||
total, err := resultSvc.Count(query)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
HandleSuccessWithListData(c, data, total)
|
||||
}
|
||||
|
||||
func (ctx *taskContext) _delete(c *gin.Context) (err error) {
|
||||
id := c.Param("id")
|
||||
oid, err := primitive.ObjectIDFromHex(id)
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) {
|
||||
// delete task
|
||||
task, err := ctx.modelSvc.GetTaskById(oid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := delegate2.NewModelDelegate(task, GetUserFromContext(c)).Delete(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// delete task stat
|
||||
taskStat, err := ctx.modelSvc.GetTaskStatById(oid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := delegate2.NewModelDelegate(taskStat, GetUserFromContext(c)).Delete(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ctx *taskContext) _deleteList(c *gin.Context) (err error) {
|
||||
payload, err := NewJsonBinder(ControllerIdTask).BindBatchRequestPayload(c)
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := mongo.RunTransaction(func(context mongo2.SessionContext) error {
|
||||
// delete tasks
|
||||
if err := ctx.modelTaskSvc.DeleteList(bson.M{
|
||||
"_id": bson.M{
|
||||
"$in": payload.Ids,
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// delete task stats
|
||||
if err := ctx.modelTaskStatSvc.DeleteList(bson.M{
|
||||
"_id": bson.M{
|
||||
"$in": payload.Ids,
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func newTaskContext() *taskContext {
|
||||
// context
|
||||
ctx := &taskContext{}
|
||||
|
||||
// dependency injection
|
||||
if err := container.GetContainer().Invoke(func(
|
||||
modelSvc service.ModelService,
|
||||
adminSvc interfaces.SpiderAdminService,
|
||||
schedulerSvc interfaces.TaskSchedulerService,
|
||||
) {
|
||||
ctx.modelSvc = modelSvc
|
||||
ctx.adminSvc = adminSvc
|
||||
ctx.schedulerSvc = schedulerSvc
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// model task service
|
||||
ctx.modelTaskSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTask)
|
||||
|
||||
// model task stat service
|
||||
ctx.modelTaskStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTaskStat)
|
||||
|
||||
// log driver
|
||||
l, err := log.GetLogDriver(log.DriverTypeFile)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
ctx.l = l
|
||||
|
||||
return ctx
|
||||
}
|
||||
|
||||
func newTaskController() *taskController {
|
||||
actions := getTaskActions()
|
||||
modelSvc, err := service.GetService()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
ctr := NewListPostActionControllerDelegate(ControllerIdTask, modelSvc.GetBaseService(interfaces.ModelIdTask), actions)
|
||||
d := NewListPostActionControllerDelegate(ControllerIdTask, modelSvc.GetBaseService(interfaces.ModelIdTask), actions)
|
||||
ctx := newTaskContext()
|
||||
|
||||
return &taskController{
|
||||
ListActionControllerDelegate: *ctr,
|
||||
d: *d,
|
||||
ctx: ctx,
|
||||
}
|
||||
}
|
||||
@@ -1,104 +0,0 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/controllers"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/crawlab-team/crawlab/core/routes"
|
||||
"github.com/crawlab-team/crawlab/trace"
|
||||
"github.com/gavv/httpexpect/v2"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/dig"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var err error
|
||||
T, err = NewTest()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
type Test struct {
|
||||
// dependencies
|
||||
modelSvc service.ModelService
|
||||
|
||||
// internals
|
||||
app *gin.Engine
|
||||
svr *httptest.Server
|
||||
|
||||
// test data
|
||||
TestUsername string
|
||||
TestPassword string
|
||||
TestToken string
|
||||
}
|
||||
|
||||
func (t *Test) Setup(t2 *testing.T) {
|
||||
//if err := controllers.InitControllers(); err != nil {
|
||||
// panic(err)
|
||||
//}
|
||||
//t2.Cleanup(t.Cleanup)
|
||||
}
|
||||
|
||||
func (t *Test) Cleanup() {
|
||||
_ = t.modelSvc.DropAll()
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
|
||||
func (t *Test) NewExpect(t2 *testing.T) (e *httpexpect.Expect) {
|
||||
e = httpexpect.New(t2, t.svr.URL)
|
||||
res := e.POST("/login").WithJSON(map[string]string{
|
||||
"username": t.TestUsername,
|
||||
"password": t.TestPassword,
|
||||
}).Expect().JSON().Object()
|
||||
t.TestToken = res.Path("$.data").String().Raw()
|
||||
require.NotEmpty(t2, t.TestToken)
|
||||
return e
|
||||
}
|
||||
|
||||
func (t *Test) WithAuth(req *httpexpect.Request) *httpexpect.Request {
|
||||
return req.WithHeader("Authorization", t.TestToken)
|
||||
}
|
||||
|
||||
var T *Test
|
||||
|
||||
func NewTest() (res *Test, err error) {
|
||||
// test
|
||||
t := &Test{}
|
||||
|
||||
// gin app
|
||||
t.app = gin.New()
|
||||
|
||||
// http test server
|
||||
t.svr = httptest.NewServer(t.app)
|
||||
|
||||
// init controllers
|
||||
if err := controllers.InitControllers(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// init routes
|
||||
if err := routes.InitRoutes(t.app); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// dependency injection
|
||||
c := dig.New()
|
||||
if err := c.Provide(service.NewService); err != nil {
|
||||
return nil, trace.TraceError(err)
|
||||
}
|
||||
if err := c.Invoke(func(modelSvc service.ModelService) {
|
||||
t.modelSvc = modelSvc
|
||||
}); err != nil {
|
||||
return nil, trace.TraceError(err)
|
||||
}
|
||||
|
||||
// test data
|
||||
t.TestUsername = "admin"
|
||||
t.TestPassword = "admin"
|
||||
|
||||
return t, nil
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/constants"
|
||||
"github.com/crawlab-team/crawlab/db/mongo"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func init() {
|
||||
viper.Set("mongo.db", "crawlab_test")
|
||||
}
|
||||
|
||||
func TestExportController_Csv(t *testing.T) {
|
||||
T.Setup(t)
|
||||
e := T.NewExpect(t)
|
||||
|
||||
// mongo collection
|
||||
colName := "test_collection_for_export"
|
||||
col := mongo.GetMongoCol(colName)
|
||||
|
||||
// insert test data to mongo collection
|
||||
for i := 0; i < 10; i++ {
|
||||
_, err := col.Insert(bson.M{
|
||||
"field1": i + 1,
|
||||
"field2": i + 2,
|
||||
"field3": i + 3,
|
||||
"field4": i + 4,
|
||||
})
|
||||
require.Nil(t, err)
|
||||
}
|
||||
|
||||
// export from mongo collection
|
||||
res := T.WithAuth(e.POST("/export/csv")).
|
||||
WithQuery("target", colName).
|
||||
Expect().Status(http.StatusOK).JSON().Object()
|
||||
res.Path("$.data").NotNull()
|
||||
|
||||
// export id
|
||||
exportId := res.Path("$.data").String().Raw()
|
||||
|
||||
// poll export with export id
|
||||
for i := 0; i < 10; i++ {
|
||||
res = T.WithAuth(e.GET("/export/csv/" + exportId)).Expect().Status(http.StatusOK).JSON().Object()
|
||||
status := res.Path("$.data.status").String().Raw()
|
||||
if status == constants.TaskStatusFinished {
|
||||
break
|
||||
}
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
|
||||
// download exported csv file
|
||||
csvFileBody := T.WithAuth(e.GET("/export/csv/" + exportId + "/download")).Expect().Status(http.StatusOK).Body()
|
||||
csvFileBody.NotEmpty()
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/crawlab-team/crawlab/core/constants"
|
||||
"github.com/crawlab-team/crawlab/core/entity"
|
||||
"github.com/crawlab-team/crawlab/db/mongo"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func init() {
|
||||
viper.Set("mongo.db", "crawlab_test")
|
||||
}
|
||||
|
||||
func TestFilterController_GetColFieldOptions(t *testing.T) {
|
||||
T.Setup(t)
|
||||
e := T.NewExpect(t)
|
||||
|
||||
// mongo collection
|
||||
colName := "test_collection_for_filter"
|
||||
field1 := "field1"
|
||||
field2 := "field2"
|
||||
value1 := "value1"
|
||||
col := mongo.GetMongoCol(colName)
|
||||
n := 10
|
||||
var ids []primitive.ObjectID
|
||||
var names []string
|
||||
for i := 0; i < n; i++ {
|
||||
_id := primitive.NewObjectID()
|
||||
ids = append(ids, _id)
|
||||
name := fmt.Sprintf("name_%d", i)
|
||||
names = append(names, name)
|
||||
_, err := col.Insert(bson.M{field1: value1, field2: i % 2, "name": name, "_id": _id})
|
||||
require.Nil(t, err)
|
||||
}
|
||||
|
||||
// validate filter options field 1
|
||||
res := T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field1, field1))).
|
||||
Expect().Status(http.StatusOK).JSON().Object()
|
||||
res.Path("$.data").NotNull()
|
||||
res.Path("$.data").Array().Length().Equal(1)
|
||||
res.Path("$.data").Array().Element(0).Path("$.value").Equal(value1)
|
||||
|
||||
// validate filter options field 2
|
||||
res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))).
|
||||
Expect().Status(http.StatusOK).JSON().Object()
|
||||
res.Path("$.data").NotNull()
|
||||
res.Path("$.data").Array().Length().Equal(2)
|
||||
|
||||
// validate filter options with query
|
||||
conditions := []entity.Condition{{field2, constants.FilterOpEqual, 0}}
|
||||
conditionsJson, err := json.Marshal(conditions)
|
||||
conditionsJsonStr := string(conditionsJson)
|
||||
require.Nil(t, err)
|
||||
res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))).
|
||||
WithQuery(constants.FilterQueryFieldConditions, conditionsJsonStr).
|
||||
Expect().Status(http.StatusOK).JSON().Object()
|
||||
res.Path("$.data").NotNull()
|
||||
res.Path("$.data").Array().Length().Equal(1)
|
||||
|
||||
// validate filter options (basic path)
|
||||
res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s", colName))).
|
||||
Expect().Status(http.StatusOK).JSON().Object()
|
||||
res.Path("$.data").NotNull()
|
||||
res.Path("$.data").Array().Length().Equal(n)
|
||||
for i := 0; i < n; i++ {
|
||||
res.Path("$.data").Array().Element(i).Object().Value("value").Equal(ids[i])
|
||||
res.Path("$.data").Array().Element(i).Object().Value("label").Equal(names[i])
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/constants"
|
||||
"github.com/crawlab-team/crawlab/core/controllers"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/crawlab-team/crawlab/core/user"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// init user
|
||||
modelSvc, err := service.GetService()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
_, err = modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil)
|
||||
if err != nil {
|
||||
if err.Error() != mongo.ErrNoDocuments.Error() {
|
||||
panic(err)
|
||||
}
|
||||
userSvc, err := user.GetUserService()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := userSvc.Create(&interfaces.UserCreateOptions{
|
||||
Username: constants.DefaultAdminUsername,
|
||||
Password: constants.DefaultAdminPassword,
|
||||
Role: constants.RoleAdmin,
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := controllers.InitControllers(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
m.Run()
|
||||
|
||||
T.Cleanup()
|
||||
}
|
||||
@@ -1,305 +0,0 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/crawlab-team/crawlab/core/constants"
|
||||
"github.com/crawlab-team/crawlab/core/entity"
|
||||
"github.com/crawlab-team/crawlab/core/models/delegate"
|
||||
"github.com/crawlab-team/crawlab/core/models/models"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestProjectController_Get(t *testing.T) {
|
||||
T.Setup(t)
|
||||
e := T.NewExpect(t)
|
||||
|
||||
p := models.Project{
|
||||
Name: "test project",
|
||||
}
|
||||
res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
|
||||
res.Path("$.data._id").NotNull()
|
||||
id := res.Path("$.data._id").String().Raw()
|
||||
oid, err := primitive.ObjectIDFromHex(id)
|
||||
require.Nil(t, err)
|
||||
require.False(t, oid.IsZero())
|
||||
|
||||
res = T.WithAuth(e.GET("/projects/" + id)).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
|
||||
res.Path("$.data._id").NotNull()
|
||||
res.Path("$.data.name").Equal("test project")
|
||||
}
|
||||
|
||||
func TestProjectController_Put(t *testing.T) {
|
||||
T.Setup(t)
|
||||
e := T.NewExpect(t)
|
||||
|
||||
p := models.Project{
|
||||
Name: "old name",
|
||||
Description: "old description",
|
||||
}
|
||||
|
||||
// add
|
||||
res := T.WithAuth(e.POST("/projects")).
|
||||
WithJSON(p).
|
||||
Expect().Status(http.StatusOK).
|
||||
JSON().Object()
|
||||
res.Path("$.data._id").NotNull()
|
||||
id := res.Path("$.data._id").String().Raw()
|
||||
oid, err := primitive.ObjectIDFromHex(id)
|
||||
require.Nil(t, err)
|
||||
require.False(t, oid.IsZero())
|
||||
|
||||
// change object
|
||||
p.Id = oid
|
||||
p.Name = "new name"
|
||||
p.Description = "new description"
|
||||
|
||||
// update
|
||||
T.WithAuth(e.PUT("/projects/" + id)).
|
||||
WithJSON(p).
|
||||
Expect().Status(http.StatusOK)
|
||||
|
||||
// check
|
||||
res = T.WithAuth(e.GET("/projects/" + id)).Expect().Status(http.StatusOK).JSON().Object()
|
||||
res.Path("$.data._id").Equal(id)
|
||||
res.Path("$.data.name").Equal("new name")
|
||||
res.Path("$.data.description").Equal("new description")
|
||||
}
|
||||
|
||||
func TestProjectController_Post(t *testing.T) {
|
||||
T.Setup(t)
|
||||
e := T.NewExpect(t)
|
||||
|
||||
p := models.Project{
|
||||
Name: "test project",
|
||||
Description: "this is a test project",
|
||||
}
|
||||
|
||||
res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
|
||||
res.Path("$.data._id").NotNull()
|
||||
res.Path("$.data.name").Equal("test project")
|
||||
res.Path("$.data.description").Equal("this is a test project")
|
||||
}
|
||||
|
||||
func TestProjectController_Delete(t *testing.T) {
|
||||
T.Setup(t)
|
||||
e := T.NewExpect(t)
|
||||
|
||||
p := models.Project{
|
||||
Name: "test project",
|
||||
Description: "this is a test project",
|
||||
}
|
||||
|
||||
// add
|
||||
res := T.WithAuth(e.POST("/projects")).
|
||||
WithJSON(p).
|
||||
Expect().Status(http.StatusOK).
|
||||
JSON().Object()
|
||||
res.Path("$.data._id").NotNull()
|
||||
id := res.Path("$.data._id").String().Raw()
|
||||
oid, err := primitive.ObjectIDFromHex(id)
|
||||
require.Nil(t, err)
|
||||
require.False(t, oid.IsZero())
|
||||
|
||||
// get
|
||||
res = T.WithAuth(e.GET("/projects/" + id)).
|
||||
Expect().Status(http.StatusOK).
|
||||
JSON().Object()
|
||||
res.Path("$.data._id").NotNull()
|
||||
id = res.Path("$.data._id").String().Raw()
|
||||
oid, err = primitive.ObjectIDFromHex(id)
|
||||
require.Nil(t, err)
|
||||
require.False(t, oid.IsZero())
|
||||
|
||||
// delete
|
||||
T.WithAuth(e.DELETE("/projects/" + id)).
|
||||
Expect().Status(http.StatusOK).
|
||||
JSON().Object()
|
||||
|
||||
// get
|
||||
T.WithAuth(e.GET("/projects/" + id)).
|
||||
Expect().Status(http.StatusNotFound)
|
||||
}
|
||||
|
||||
func TestProjectController_GetList(t *testing.T) {
|
||||
T.Setup(t)
|
||||
e := T.NewExpect(t)
|
||||
|
||||
n := 100 // total
|
||||
bn := 10 // batch
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
p := models.Project{
|
||||
Name: fmt.Sprintf("test name %d", i+1),
|
||||
}
|
||||
obj := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object()
|
||||
obj.Path("$.data._id").NotNull()
|
||||
}
|
||||
|
||||
f := entity.Filter{
|
||||
//IsOr: false,
|
||||
Conditions: []*entity.Condition{
|
||||
{Key: "name", Op: constants.FilterOpContains, Value: "test name"},
|
||||
},
|
||||
}
|
||||
condBytes, err := json.Marshal(&f.Conditions)
|
||||
require.Nil(t, err)
|
||||
|
||||
pagination := entity.Pagination{
|
||||
Page: 1,
|
||||
Size: bn,
|
||||
}
|
||||
|
||||
// get list with pagination
|
||||
res := T.WithAuth(e.GET("/projects")).
|
||||
WithQuery("conditions", string(condBytes)).
|
||||
WithQueryObject(pagination).
|
||||
Expect().Status(http.StatusOK).JSON().Object()
|
||||
res.Path("$.data").Array().Length().Equal(bn)
|
||||
res.Path("$.total").Number().Equal(n)
|
||||
|
||||
data := res.Path("$.data").Array()
|
||||
for i := 0; i < bn; i++ {
|
||||
obj := data.Element(i)
|
||||
obj.Path("$.name").Equal(fmt.Sprintf("test name %d", i+1))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestProjectController_PostList(t *testing.T) {
|
||||
T.Setup(t)
|
||||
e := T.NewExpect(t)
|
||||
|
||||
n := 10
|
||||
var docs []models.Project
|
||||
for i := 0; i < n; i++ {
|
||||
docs = append(docs, models.Project{
|
||||
Name: fmt.Sprintf("project %d", i+1),
|
||||
Description: "this is a project",
|
||||
})
|
||||
}
|
||||
|
||||
T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK)
|
||||
|
||||
res := T.WithAuth(e.GET("/projects")).
|
||||
WithQueryObject(entity.Pagination{Page: 1, Size: 10}).
|
||||
Expect().Status(http.StatusOK).
|
||||
JSON().Object()
|
||||
res.Path("$.data").Array().Length().Equal(n)
|
||||
}
|
||||
|
||||
func TestProjectController_DeleteList(t *testing.T) {
|
||||
T.Setup(t)
|
||||
e := T.NewExpect(t)
|
||||
|
||||
n := 10
|
||||
var docs []models.Project
|
||||
for i := 0; i < n; i++ {
|
||||
docs = append(docs, models.Project{
|
||||
Name: fmt.Sprintf("project %d", i+1),
|
||||
Description: "this is a project",
|
||||
})
|
||||
}
|
||||
|
||||
// add
|
||||
res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK).
|
||||
JSON().Object()
|
||||
var ids []primitive.ObjectID
|
||||
data := res.Path("$.data").Array()
|
||||
for i := 0; i < n; i++ {
|
||||
obj := data.Element(i)
|
||||
id := obj.Path("$._id").String().Raw()
|
||||
oid, err := primitive.ObjectIDFromHex(id)
|
||||
require.Nil(t, err)
|
||||
require.False(t, oid.IsZero())
|
||||
ids = append(ids, oid)
|
||||
}
|
||||
|
||||
// delete
|
||||
payload := entity.BatchRequestPayload{
|
||||
Ids: ids,
|
||||
}
|
||||
T.WithAuth(e.DELETE("/projects")).
|
||||
WithJSON(payload).
|
||||
Expect().Status(http.StatusOK)
|
||||
|
||||
// check
|
||||
for _, id := range ids {
|
||||
T.WithAuth(e.GET("/projects/" + id.Hex())).
|
||||
Expect().Status(http.StatusNotFound)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestProjectController_PutList(t *testing.T) {
|
||||
T.Setup(t)
|
||||
e := T.NewExpect(t)
|
||||
|
||||
// now
|
||||
now := time.Now()
|
||||
|
||||
n := 10
|
||||
var docs []models.Project
|
||||
for i := 0; i < n; i++ {
|
||||
docs = append(docs, models.Project{
|
||||
Name: "old name",
|
||||
Description: "old description",
|
||||
})
|
||||
}
|
||||
|
||||
// add
|
||||
res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK).
|
||||
JSON().Object()
|
||||
var ids []primitive.ObjectID
|
||||
data := res.Path("$.data").Array()
|
||||
for i := 0; i < n; i++ {
|
||||
obj := data.Element(i)
|
||||
id := obj.Path("$._id").String().Raw()
|
||||
oid, err := primitive.ObjectIDFromHex(id)
|
||||
require.Nil(t, err)
|
||||
require.False(t, oid.IsZero())
|
||||
ids = append(ids, oid)
|
||||
}
|
||||
|
||||
// wait for 100 millisecond
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// update
|
||||
p := models.Project{
|
||||
Name: "new name",
|
||||
Description: "new description",
|
||||
}
|
||||
dataBytes, err := json.Marshal(&p)
|
||||
require.Nil(t, err)
|
||||
payload := entity.BatchRequestPayloadWithStringData{
|
||||
Ids: ids,
|
||||
Data: string(dataBytes),
|
||||
Fields: []string{
|
||||
"name",
|
||||
"description",
|
||||
},
|
||||
}
|
||||
T.WithAuth(e.PUT("/projects")).WithJSON(payload).Expect().Status(http.StatusOK)
|
||||
|
||||
// check response data
|
||||
for i := 0; i < n; i++ {
|
||||
res = T.WithAuth(e.GET("/projects/" + ids[i].Hex())).Expect().Status(http.StatusOK).JSON().Object()
|
||||
res.Path("$.data.name").Equal("new name")
|
||||
res.Path("$.data.description").Equal("new description")
|
||||
}
|
||||
|
||||
// check artifacts
|
||||
pl, err := T.modelSvc.GetProjectList(bson.M{"_id": bson.M{"$in": ids}}, nil)
|
||||
require.Nil(t, err)
|
||||
for _, p := range pl {
|
||||
a, err := delegate.NewModelDelegate(&p).GetArtifact()
|
||||
require.Nil(t, err)
|
||||
require.True(t, a.GetSys().GetUpdateTs().After(now))
|
||||
require.True(t, a.GetSys().GetUpdateTs().After(a.GetSys().GetCreateTs()))
|
||||
}
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/entity"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/crawlab-team/crawlab/core/models/delegate"
|
||||
"github.com/crawlab-team/crawlab/core/models/models"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSpiderController_Delete(t *testing.T) {
|
||||
T.Setup(t)
|
||||
e := T.NewExpect(t)
|
||||
|
||||
s := models.Spider{
|
||||
Name: "test spider",
|
||||
Description: "this is a test spider",
|
||||
ColName: "test col name",
|
||||
}
|
||||
|
||||
// add spider
|
||||
res := T.WithAuth(e.POST("/spiders")).
|
||||
WithJSON(s).
|
||||
Expect().Status(http.StatusOK).
|
||||
JSON().Object()
|
||||
res.Path("$.data._id").NotNull()
|
||||
id := res.Path("$.data._id").String().Raw()
|
||||
oid, err := primitive.ObjectIDFromHex(id)
|
||||
require.Nil(t, err)
|
||||
require.False(t, oid.IsZero())
|
||||
|
||||
// add tasks
|
||||
var taskIds []primitive.ObjectID
|
||||
tasks := []models.Task{
|
||||
{
|
||||
Id: primitive.NewObjectID(),
|
||||
SpiderId: oid,
|
||||
},
|
||||
{
|
||||
Id: primitive.NewObjectID(),
|
||||
SpiderId: oid,
|
||||
},
|
||||
}
|
||||
for _, task := range tasks {
|
||||
// add task
|
||||
err := delegate.NewModelDelegate(&task).Add()
|
||||
require.Nil(t, err)
|
||||
|
||||
// add task stat
|
||||
err = delegate.NewModelDelegate(&models.TaskStat{
|
||||
Id: task.Id,
|
||||
}).Add()
|
||||
require.Nil(t, err)
|
||||
|
||||
taskIds = append(taskIds, task.Id)
|
||||
}
|
||||
|
||||
// delete
|
||||
T.WithAuth(e.DELETE("/spiders/" + id)).
|
||||
Expect().Status(http.StatusOK)
|
||||
|
||||
// get
|
||||
T.WithAuth(e.GET("/spiders/" + id)).
|
||||
Expect().Status(http.StatusNotFound)
|
||||
|
||||
// get tasks
|
||||
for _, task := range tasks {
|
||||
T.WithAuth(e.GET("/tasks/" + task.Id.Hex())).
|
||||
Expect().Status(http.StatusNotFound)
|
||||
}
|
||||
|
||||
// spider stat
|
||||
modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat)
|
||||
spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{
|
||||
"_id": oid,
|
||||
})
|
||||
require.Nil(t, err)
|
||||
require.Zero(t, spiderStatCount)
|
||||
|
||||
// task stats
|
||||
modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
|
||||
taskStatCount, err := modelTaskStatSvc.Count(bson.M{
|
||||
"_id": bson.M{
|
||||
"$in": taskIds,
|
||||
},
|
||||
})
|
||||
require.Nil(t, err)
|
||||
require.Zero(t, taskStatCount)
|
||||
}
|
||||
|
||||
func TestSpiderController_DeleteList(t *testing.T) {
|
||||
T.Setup(t)
|
||||
e := T.NewExpect(t)
|
||||
|
||||
spiders := []models.Spider{
|
||||
{
|
||||
Id: primitive.NewObjectID(),
|
||||
Name: "test spider 1",
|
||||
Description: "this is a test spider 1",
|
||||
ColName: "test col name 1",
|
||||
},
|
||||
{
|
||||
Id: primitive.NewObjectID(),
|
||||
Name: "test spider 2",
|
||||
Description: "this is a test spider 2",
|
||||
ColName: "test col name 2",
|
||||
},
|
||||
}
|
||||
|
||||
// add spiders
|
||||
for _, spider := range spiders {
|
||||
T.WithAuth(e.POST("/spiders")).
|
||||
WithJSON(spider).
|
||||
Expect().Status(http.StatusOK)
|
||||
}
|
||||
|
||||
var spiderIds []primitive.ObjectID
|
||||
var taskIds []primitive.ObjectID
|
||||
for _, spider := range spiders {
|
||||
// task id
|
||||
taskId := primitive.NewObjectID()
|
||||
|
||||
// add task
|
||||
err := delegate.NewModelDelegate(&models.Task{
|
||||
Id: taskId,
|
||||
SpiderId: spider.Id,
|
||||
}).Add()
|
||||
require.Nil(t, err)
|
||||
|
||||
// add task stats
|
||||
err = delegate.NewModelDelegate(&models.TaskStat{
|
||||
Id: taskId,
|
||||
}).Add()
|
||||
require.Nil(t, err)
|
||||
|
||||
spiderIds = append(spiderIds, spider.Id)
|
||||
taskIds = append(taskIds, taskId)
|
||||
}
|
||||
|
||||
// delete spiders
|
||||
T.WithAuth(e.DELETE("/spiders")).
|
||||
WithJSON(entity.BatchRequestPayload{
|
||||
Ids: spiderIds,
|
||||
}).Expect().Status(http.StatusOK)
|
||||
|
||||
// get spiders
|
||||
for _, spider := range spiders {
|
||||
// get
|
||||
T.WithAuth(e.GET("/spiders/" + spider.Id.Hex())).
|
||||
Expect().Status(http.StatusNotFound)
|
||||
}
|
||||
|
||||
// get tasks
|
||||
for _, taskId := range taskIds {
|
||||
T.WithAuth(e.GET("/tasks/" + taskId.Hex())).
|
||||
Expect().Status(http.StatusNotFound)
|
||||
}
|
||||
|
||||
// spider stat
|
||||
modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat)
|
||||
spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{
|
||||
"_id": bson.M{
|
||||
"$in": spiderIds,
|
||||
},
|
||||
})
|
||||
require.Nil(t, err)
|
||||
require.Zero(t, spiderStatCount)
|
||||
|
||||
// task stats
|
||||
modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
|
||||
taskStatCount, err := modelTaskStatSvc.Count(bson.M{
|
||||
"_id": bson.M{
|
||||
"$in": taskIds,
|
||||
},
|
||||
})
|
||||
require.Nil(t, err)
|
||||
require.Zero(t, taskStatCount)
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/entity"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/crawlab-team/crawlab/core/models/delegate"
|
||||
"github.com/crawlab-team/crawlab/core/models/models"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTaskController_Delete(t *testing.T) {
|
||||
T.Setup(t)
|
||||
e := T.NewExpect(t)
|
||||
|
||||
task := models.Task{
|
||||
Id: primitive.NewObjectID(),
|
||||
}
|
||||
|
||||
// add task
|
||||
err := delegate.NewModelDelegate(&task).Add()
|
||||
require.Nil(t, err)
|
||||
|
||||
// add task stat
|
||||
err = delegate.NewModelDelegate(&models.TaskStat{
|
||||
Id: task.Id,
|
||||
}).Add()
|
||||
require.Nil(t, err)
|
||||
|
||||
// delete
|
||||
T.WithAuth(e.DELETE("/tasks/" + task.Id.Hex())).
|
||||
Expect().Status(http.StatusOK)
|
||||
|
||||
// get
|
||||
T.WithAuth(e.GET("/tasks/" + task.Id.Hex())).
|
||||
Expect().Status(http.StatusNotFound)
|
||||
|
||||
// task stats
|
||||
modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
|
||||
taskStatCount, err := modelTaskStatSvc.Count(bson.M{
|
||||
"_id": task.Id,
|
||||
})
|
||||
require.Nil(t, err)
|
||||
require.Zero(t, taskStatCount)
|
||||
}
|
||||
|
||||
func TestTaskController_DeleteList(t *testing.T) {
|
||||
T.Setup(t)
|
||||
e := T.NewExpect(t)
|
||||
|
||||
tasks := []models.Task{
|
||||
{
|
||||
Id: primitive.NewObjectID(),
|
||||
},
|
||||
{
|
||||
Id: primitive.NewObjectID(),
|
||||
},
|
||||
}
|
||||
|
||||
// add spiders
|
||||
var taskIds []primitive.ObjectID
|
||||
for _, task := range tasks {
|
||||
// add task
|
||||
err := delegate.NewModelDelegate(&task).Add()
|
||||
require.Nil(t, err)
|
||||
|
||||
// add task stat
|
||||
err = delegate.NewModelDelegate(&models.TaskStat{
|
||||
Id: task.Id,
|
||||
}).Add()
|
||||
require.Nil(t, err)
|
||||
|
||||
taskIds = append(taskIds, task.Id)
|
||||
}
|
||||
|
||||
// delete tasks
|
||||
T.WithAuth(e.DELETE("/tasks")).
|
||||
WithJSON(entity.BatchRequestPayload{
|
||||
Ids: taskIds,
|
||||
}).Expect().Status(http.StatusOK)
|
||||
|
||||
// get tasks
|
||||
for _, task := range tasks {
|
||||
// get
|
||||
T.WithAuth(e.GET("/tasks/" + task.Id.Hex())).
|
||||
Expect().Status(http.StatusNotFound)
|
||||
}
|
||||
|
||||
// task stats
|
||||
modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat)
|
||||
taskStatCount, err := modelTaskStatSvc.Count(bson.M{
|
||||
"_id": bson.M{
|
||||
"$in": taskIds,
|
||||
},
|
||||
})
|
||||
require.Nil(t, err)
|
||||
require.Zero(t, taskStatCount)
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/container"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/crawlab-team/crawlab/core/models/delegate"
|
||||
"github.com/crawlab-team/crawlab/core/models/models"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
var TokenController *tokenController
|
||||
|
||||
var TokenActions []Action
|
||||
|
||||
type tokenController struct {
|
||||
ListActionControllerDelegate
|
||||
d ListActionControllerDelegate
|
||||
ctx *tokenContext
|
||||
}
|
||||
|
||||
func (ctr *tokenController) Post(c *gin.Context) {
|
||||
var err error
|
||||
var t models.Token
|
||||
if err := c.ShouldBindJSON(&t); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
u, err := ctr.ctx.userSvc.GetCurrentUser(c)
|
||||
if err != nil {
|
||||
HandleErrorUnauthorized(c, err)
|
||||
return
|
||||
}
|
||||
t.Token, err = ctr.ctx.userSvc.MakeToken(u)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
if err := delegate.NewModelDelegate(&t, GetUserFromContext(c)).Add(); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
type tokenContext struct {
|
||||
modelSvc service.ModelService
|
||||
userSvc interfaces.UserService
|
||||
}
|
||||
|
||||
func newTokenContext() *tokenContext {
|
||||
// context
|
||||
ctx := &tokenContext{}
|
||||
|
||||
// dependency injection
|
||||
if err := container.GetContainer().Invoke(func(
|
||||
modelSvc service.ModelService,
|
||||
userSvc interfaces.UserService,
|
||||
) {
|
||||
ctx.modelSvc = modelSvc
|
||||
ctx.userSvc = userSvc
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return ctx
|
||||
}
|
||||
|
||||
func newTokenController() *tokenController {
|
||||
modelSvc, err := service.GetService()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
ctr := NewListPostActionControllerDelegate(ControllerIdToken, modelSvc.GetBaseService(interfaces.ModelIdToken), TokenActions)
|
||||
d := NewListPostActionControllerDelegate(ControllerIdToken, modelSvc.GetBaseService(interfaces.ModelIdToken), TokenActions)
|
||||
ctx := newTokenContext()
|
||||
|
||||
return &tokenController{
|
||||
ListActionControllerDelegate: *ctr,
|
||||
d: *d,
|
||||
ctx: ctx,
|
||||
}
|
||||
}
|
||||
@@ -1,244 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/crawlab-team/crawlab/core/constants"
|
||||
"github.com/crawlab-team/crawlab/core/container"
|
||||
"github.com/crawlab-team/crawlab/core/entity"
|
||||
"github.com/crawlab-team/crawlab/core/errors"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
delegate2 "github.com/crawlab-team/crawlab/core/models/delegate"
|
||||
"github.com/crawlab-team/crawlab/core/models/models"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/crawlab-team/crawlab/core/utils"
|
||||
"github.com/crawlab-team/crawlab/trace"
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
var UserController *userController
|
||||
|
||||
func getUserActions() []Action {
|
||||
userCtx := newUserContext()
|
||||
return []Action{
|
||||
{
|
||||
Method: http.MethodPost,
|
||||
Path: "/:id/change-password",
|
||||
HandlerFunc: userCtx.changePassword,
|
||||
},
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "/me",
|
||||
HandlerFunc: userCtx.getMe,
|
||||
},
|
||||
{
|
||||
Method: http.MethodPut,
|
||||
Path: "/me",
|
||||
HandlerFunc: userCtx.putMe,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type userController struct {
|
||||
ListActionControllerDelegate
|
||||
d ListActionControllerDelegate
|
||||
ctx *userContext
|
||||
}
|
||||
|
||||
func (ctr *userController) Post(c *gin.Context) {
|
||||
var u models.User
|
||||
if err := c.ShouldBindJSON(&u); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
if err := ctr.ctx.userSvc.Create(&interfaces.UserCreateOptions{
|
||||
Username: u.Username,
|
||||
Password: u.Password,
|
||||
Email: u.Email,
|
||||
Role: u.Role,
|
||||
}); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctr *userController) PostList(c *gin.Context) {
|
||||
// users
|
||||
var users []models.User
|
||||
if err := c.ShouldBindJSON(&users); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, u := range users {
|
||||
if err := ctr.ctx.userSvc.Create(&interfaces.UserCreateOptions{
|
||||
Username: u.Username,
|
||||
Password: u.Password,
|
||||
Email: u.Email,
|
||||
Role: u.Role,
|
||||
}); err != nil {
|
||||
trace.PrintError(err)
|
||||
}
|
||||
}
|
||||
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctr *userController) PutList(c *gin.Context) {
|
||||
// payload
|
||||
var payload entity.BatchRequestPayloadWithStringData
|
||||
if err := c.ShouldBindJSON(&payload); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// doc to update
|
||||
var doc models.User
|
||||
if err := json.Unmarshal([]byte(payload.Data), &doc); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// query
|
||||
query := bson.M{
|
||||
"_id": bson.M{
|
||||
"$in": payload.Ids,
|
||||
},
|
||||
}
|
||||
|
||||
// update users
|
||||
if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdUser).UpdateDoc(query, &doc, payload.Fields); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// update passwords
|
||||
if utils.Contains(payload.Fields, "password") {
|
||||
for _, id := range payload.Ids {
|
||||
if err := ctr.ctx.userSvc.ChangePassword(id, doc.Password); err != nil {
|
||||
trace.PrintError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
type userContext struct {
|
||||
modelSvc service.ModelService
|
||||
userSvc interfaces.UserService
|
||||
}
|
||||
|
||||
func (ctx *userContext) changePassword(c *gin.Context) {
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
var payload map[string]string
|
||||
if err := c.ShouldBindJSON(&payload); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
password, ok := payload["password"]
|
||||
if !ok {
|
||||
HandleErrorBadRequest(c, errors.ErrorUserMissingRequiredFields)
|
||||
return
|
||||
}
|
||||
if len(password) < 5 {
|
||||
HandleErrorBadRequest(c, errors.ErrorUserInvalidPassword)
|
||||
return
|
||||
}
|
||||
if err := ctx.userSvc.ChangePassword(id, password); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func (ctx *userContext) getMe(c *gin.Context) {
|
||||
u, err := ctx._getMe(c)
|
||||
if err != nil {
|
||||
HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized)
|
||||
return
|
||||
}
|
||||
HandleSuccessWithData(c, u)
|
||||
}
|
||||
|
||||
func (ctx *userContext) putMe(c *gin.Context) {
|
||||
// current user
|
||||
u, err := ctx._getMe(c)
|
||||
if err != nil {
|
||||
HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
// payload
|
||||
doc, err := NewJsonBinder(ControllerIdUser).Bind(c)
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
if doc.GetId() != u.GetId() {
|
||||
HandleErrorBadRequest(c, errors.ErrorHttpBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// save to db
|
||||
if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Save(); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
HandleSuccessWithData(c, doc)
|
||||
}
|
||||
|
||||
func (ctx *userContext) _getMe(c *gin.Context) (u interfaces.User, err error) {
|
||||
res, ok := c.Get(constants.UserContextKey)
|
||||
if !ok {
|
||||
return nil, trace.TraceError(errors.ErrorUserNotExistsInContext)
|
||||
}
|
||||
u, ok = res.(interfaces.User)
|
||||
if !ok {
|
||||
return nil, trace.TraceError(errors.ErrorUserInvalidType)
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func newUserContext() *userContext {
|
||||
// context
|
||||
ctx := &userContext{}
|
||||
|
||||
// dependency injection
|
||||
if err := container.GetContainer().Invoke(func(
|
||||
modelSvc service.ModelService,
|
||||
userSvc interfaces.UserService,
|
||||
) {
|
||||
ctx.modelSvc = modelSvc
|
||||
ctx.userSvc = userSvc
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return ctx
|
||||
}
|
||||
|
||||
func newUserController() *userController {
|
||||
modelSvc, err := service.GetService()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
ctr := NewListPostActionControllerDelegate(ControllerIdUser, modelSvc.GetBaseService(interfaces.ModelIdUser), getUserActions())
|
||||
d := NewListPostActionControllerDelegate(ControllerIdUser, modelSvc.GetBaseService(interfaces.ModelIdUser), getUserActions())
|
||||
ctx := newUserContext()
|
||||
|
||||
return &userController{
|
||||
ListActionControllerDelegate: *ctr,
|
||||
d: *d,
|
||||
ctx: ctx,
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/config"
|
||||
"github.com/gin-gonic/gin"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func GetVersion(c *gin.Context) {
|
||||
HandleSuccessWithData(c, config.GetVersion())
|
||||
}
|
||||
|
||||
func getVersionActions() []Action {
|
||||
return []Action{
|
||||
{
|
||||
Method: http.MethodGet,
|
||||
Path: "",
|
||||
HandlerFunc: GetVersion,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
var VersionController ActionController
|
||||
Reference in New Issue
Block a user