git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: code cleanup

Browse Source
This commit is contained in:
Marvin Zhang
2024-10-20 17:22:35 +08:00
parent 418af7c215
commit 3c53fde7e2
25 changed files with 21 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
core/controllers/project_v2.go
View File

@@ -1,7 +1,7 @@
package controllers
import (
"github.com/crawlab-team/crawlab/core/errors"
"errors"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
@@ -73,7 +73,7 @@ func GetProjectList(c *gin.Context) {
for _, s := range spiders {
_, ok := cache[s.ProjectId]
if !ok {
HandleErrorInternalServerError(c, errors.ErrorControllerMissingInCache)
HandleErrorInternalServerError(c, errors.New("project id not found"))
return
}
cache[s.ProjectId]++

4
core/controllers/schedule_v2.go
View File

@@ -1,7 +1,7 @@
package controllers
import (
"github.com/crawlab-team/crawlab/core/errors"
errors2 "errors"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/schedule"
@@ -56,7 +56,7 @@ func PutScheduleById(c *gin.Context) {
return
}
if s.Id != id {
HandleErrorBadRequest(c, errors.ErrorHttpBadRequest)
HandleErrorBadRequest(c, errors2.New("id in path does not match id in body"))
return
}

4
core/controllers/utils_filter.go
View File

@@ -2,9 +2,9 @@ package controllers
import (
"encoding/json"
errors2 "errors"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/gin-gonic/gin"
@@ -111,7 +111,7 @@ func GetFilterAll(c *gin.Context) (res bool, err error) {
case "FALSE":
return false, nil
default:
return false, errors.ErrorFilterInvalidOperation
return false, errors2.New("invalid value")
}
}

23
core/errors/base.go
View File

@@ -6,24 +6,11 @@ import (
)
const (
ErrorPrefixController = "controller"
ErrorPrefixModel = "model"
ErrorPrefixFilter = "filter"
ErrorPrefixHttp = "http"
ErrorPrefixGrpc = "grpc"
ErrorPrefixNode = "node"
ErrorPrefixInject = "inject"
ErrorPrefixSpider = "spider"
ErrorPrefixFs = "fs"
ErrorPrefixTask = "task"
ErrorPrefixSchedule = "schedule"
ErrorPrefixUser = "user"
ErrorPrefixStats = "stats"
ErrorPrefixEvent = "event"
ErrorPrefixProcess = "process"
ErrorPrefixGit = "git"
ErrorPrefixResult = "result"
ErrorPrefixDataSource = "data_source"
ErrorPrefixModel = "model"
ErrorPrefixGrpc = "grpc"
ErrorPrefixNode = "node"
ErrorPrefixTask = "task"
ErrorPrefixUser = "user"
)
type ErrorPrefix string

19
core/errors/controller.go
View File

@@ -1,19 +0,0 @@
package errors
func NewControllerError(msg string) (err error) {
return NewError(ErrorPrefixController, msg)
}
var ErrorControllerInvalidControllerId = NewControllerError("invalid controller id")
var ErrorControllerInvalidType = NewControllerError("invalid type")
var ErrorControllerAddError = NewControllerError("add error")
var ErrorControllerUpdateError = NewControllerError("update error")
var ErrorControllerDeleteError = NewControllerError("delete error")
var ErrorControllerNotImplemented = NewControllerError("not implemented")
var ErrorControllerNoModelService = NewControllerError("no model service")
var ErrorControllerRequestPayloadInvalid = NewControllerError("request payload invalid")
var ErrorControllerMissingInCache = NewControllerError("missing in cache")
var ErrorControllerNotCancellable = NewControllerError("not cancellable")
var ErrorControllerMissingRequestFields = NewControllerError("missing request fields")
var ErrorControllerEmptyResponse = NewControllerError("empty response")
var ErrorControllerFilerNotFound = NewControllerError("filer not found")

15
core/errors/ds.go
View File

@@ -1,15 +0,0 @@
package errors
func NewDataSourceError(msg string) (err error) {
return NewError(ErrorPrefixDataSource, msg)
}
var (
ErrorDataSourceInvalidType = NewDataSourceError("invalid type")
ErrorDataSourceNotExists = NewDataSourceError("not exists")
ErrorDataSourceNotExistsInContext = NewDataSourceError("not exists in context")
ErrorDataSourceAlreadyExists = NewDataSourceError("already exists")
ErrorDataSourceMismatch = NewDataSourceError("mismatch")
ErrorDataSourceMissingRequiredFields = NewDataSourceError("missing required fields")
ErrorDataSourceUnauthorized = NewDataSourceError("unauthorized")
)

10
core/errors/event.go
View File

@@ -1,10 +0,0 @@
package errors
func NewEventError(msg string) (err error) {
return NewError(ErrorPrefixEvent, msg)
}
var ErrorEventNotFound = NewEventError("not found")
var ErrorEventInvalidType = NewEventError("invalid type")
var ErrorEventAlreadyExists = NewEventError("already exists")
var ErrorEventUnknownAction = NewEventError("unknown action")

8
core/errors/filter.go
View File

@@ -1,8 +0,0 @@
package errors
func NewFilterError(msg string) (err error) {
return NewError(ErrorPrefixFilter, msg)
}
var ErrorFilterInvalidOperation = NewFilterError("invalid operation")
var ErrorFilterUnableToParseQuery = NewFilterError("unable to parse query")

11
core/errors/fs.go
View File

@@ -1,11 +0,0 @@
package errors
func NewFsError(msg string) (err error) {
return NewError(ErrorPrefixFs, msg)
}
var ErrorFsForbidden = NewFsError("forbidden")
var ErrorFsEmptyWorkspacePath = NewFsError("empty workspace path")
var ErrorFsInvalidType = NewFsError("invalid type")
var ErrorFsAlreadyExists = NewFsError("already exists")
var ErrorFsInvalidContent = NewFsError("invalid content")

10
core/errors/git.go
View File

@@ -1,10 +0,0 @@
package errors
func NewGitError(msg string) (err error) {
return NewError(ErrorPrefixGit, msg)
}
var (
ErrorGitInvalidAuthType = NewGitError("invalid auth type")
ErrorGitNoMainBranch = NewGitError("no main branch")
)

6
core/errors/grpc.go
View File

@@ -8,13 +8,7 @@ var (
ErrorGrpcClientFailedToStart = NewGrpcError("client failed to start")
ErrorGrpcServerFailedToListen = NewGrpcError("server failed to listen")
ErrorGrpcServerFailedToServe = NewGrpcError("server failed to serve")
ErrorGrpcClientNotExists = NewGrpcError("client not exists")
ErrorGrpcClientAlreadyExists = NewGrpcError("client already exists")
ErrorGrpcInvalidType = NewGrpcError("invalid type")
ErrorGrpcNotAllowed = NewGrpcError("not allowed")
ErrorGrpcSubscribeNotExists = NewGrpcError("subscribe not exists")
ErrorGrpcStreamNotFound = NewGrpcError("stream not found")
ErrorGrpcInvalidCode = NewGrpcError("invalid code")
ErrorGrpcUnauthorized = NewGrpcError("unauthorized")
ErrorGrpcInvalidNodeKey = NewGrpcError("invalid node key")
)

9
core/errors/http.go
View File

@@ -1,9 +0,0 @@
package errors
func NewHttpError(msg string) (err error) {
return NewError(ErrorPrefixHttp, msg)
}
var ErrorHttpBadRequest = NewHttpError("bad request")
var ErrorHttpUnauthorized = NewHttpError("unauthorized")
var ErrorHttpNotFound = NewHttpError("not found")

11
core/errors/model.go
View File

@@ -1,19 +1,8 @@
package errors
import "errors"
func NewModelError(msg string) (err error) {
return NewError(ErrorPrefixModel, msg)
}
var ErrorModelInvalidType = NewModelError("invalid type")
var ErrorModelInvalidModelId = NewModelError("invalid model id")
var ErrorModelNotImplemented = NewModelError("not implemented")
var ErrorModelNotFound = NewModelError("not found")
var ErrorModelAlreadyExists = NewModelError("already exists")
var ErrorModelNotExists = NewModelError("not exists")
var ErrorModelMissingRequiredData = NewModelError("missing required data")
var ErrorModelMissingId = errors.New("missing _id")
var ErrorModelNotAllowed = NewModelError("not allowed")
var ErrorModelDeleteListError = NewModelError("delete list error")
var ErrorModelNilPointer = NewModelError("nil pointer")

6
core/errors/node.go
View File

@@ -5,10 +5,4 @@ func NewNodeError(msg string) (err error) {
}
var ErrorNodeUnregistered = NewNodeError("unregistered")
var ErrorNodeServiceNotExists = NewNodeError("service not exists")
var ErrorNodeInvalidType = NewNodeError("invalid type")
var ErrorNodeInvalidStatus = NewNodeError("invalid status")
var ErrorNodeInvalidCode = NewNodeError("invalid code")
var ErrorNodeInvalidNodeKey = NewNodeError("invalid node key")
var ErrorNodeMonitorError = NewNodeError("monitor error")
var ErrorNodeNotExists = NewNodeError("not exists")

10
core/errors/process.go
View File

@@ -1,10 +0,0 @@
package errors
func NewProcessError(msg string) (err error) {
return NewError(ErrorPrefixProcess, msg)
}
var (
ErrorProcessReachedMaxErrors = NewProcessError("reached max errors")
ErrorProcessDaemonProcessExited = NewProcessError("daemon process exited")
)

5
core/errors/result.go
View File

@@ -1,5 +0,0 @@
package errors
func NewResultError(msg string) (err error) {
return NewError(ErrorPrefixResult, msg)
}

7
core/errors/schedule.go
View File

@@ -1,7 +0,0 @@
package errors
func NewScheduleError(msg string) (err error) {
return NewError(ErrorPrefixSchedule, msg)
}
//var ErrorSchedule = NewScheduleError("unregistered")

10
core/errors/spider.go
View File

@@ -1,10 +0,0 @@
package errors
func NewSpiderError(msg string) (err error) {
return NewError(ErrorPrefixSpider, msg)
}
var (
ErrorSpiderMissingRequiredOption = NewSpiderError("missing required option")
ErrorSpiderForbidden = NewSpiderError("forbidden")
)

7
core/errors/stats.go
View File

@@ -1,7 +0,0 @@
package errors
func NewStatsError(msg string) (err error) {
return NewError(ErrorPrefixStats, msg)
}
var ErrorStatsInvalidType = NewStatsError("invalid type")

9
core/errors/store.go
View File

@@ -1,9 +0,0 @@
package errors
func NewInjectError(msg string) (err error) {
return NewError(ErrorPrefixInject, msg)
}
var ErrorInjectEmptyValue = NewInjectError("empty value")
var ErrorInjectNotExists = NewInjectError("not exists")
var ErrorInjectInvalidType = NewInjectError("invalid type")

15
core/errors/task.go
View File

@@ -5,15 +5,8 @@ func NewTaskError(msg string) (err error) {
}
var (
ErrorTaskNotExists = NewTaskError("not exists")
ErrorTaskAlreadyExists = NewTaskError("already exists")
ErrorTaskInvalidType = NewTaskError("invalid type")
ErrorTaskProcessStillExists = NewTaskError("process still exists")
ErrorTaskUnableToCancel = NewTaskError("unable to cancel")
ErrorTaskForbidden = NewTaskError("forbidden")
ErrorTaskNoAvailableRunners = NewTaskError("no available runner")
ErrorTaskEmptySpiderId = NewTaskError("empty spider id")
ErrorTaskNoNodeId = NewTaskError("no node id")
ErrorTaskNodeNotFound = NewTaskError("node not found")
ErrorTaskMissingRequiredOption = NewSpiderError("missing required option")
ErrorTaskNotExists = NewTaskError("not exists")
ErrorTaskAlreadyExists = NewTaskError("already exists")
ErrorTaskNoNodeId = NewTaskError("no node id")
ErrorTaskNodeNotFound = NewTaskError("node not found")
)

1
core/errors/user.go
View File

@@ -8,7 +8,6 @@ var (
ErrorUserInvalidType = NewUserError("invalid type")
ErrorUserInvalidToken = NewUserError("invalid token")
ErrorUserNotExists = NewUserError("not exists")
ErrorUserNotExistsInContext = NewUserError("not exists in context")
ErrorUserAlreadyExists = NewUserError("already exists")
ErrorUserMismatch = NewUserError("mismatch")
ErrorUserMissingRequiredFields = NewUserError("missing required fields")

4
core/middlewares/auth_v2.go
View File

@@ -1,8 +1,8 @@
package middlewares
import (
"errors"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/user"
@@ -34,7 +34,7 @@ func AuthorizationMiddlewareV2() gin.HandlerFunc {
u, err := userSvc.CheckToken(tokenStr)
if err != nil {
// validation failed, return error response
utils.HandleErrorUnauthorized(c, errors.ErrorHttpUnauthorized)
utils.HandleErrorUnauthorized(c, errors.New("invalid token"))
return
}

4
core/spider/admin/service_v2.go
View File

@@ -1,10 +1,10 @@
package admin
import (
"errors"
log2 "github.com/apex/log"
config2 "github.com/crawlab-team/crawlab/core/config"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
@@ -142,7 +142,7 @@ func newSpiderAdminServiceV2() (svc2 *ServiceV2, err error) {
// validate node type
if !svc.nodeCfgSvc.IsMaster() {
return nil, trace.TraceError(errors.ErrorSpiderForbidden)
return nil, errors.New("only master node can run spider admin service")
}
return svc, nil

4
core/utils/filter.go
View File

@@ -1,8 +1,8 @@
package utils
import (
errors2 "errors"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"go.mongodb.org/mongo-driver/bson"
)
@@ -42,7 +42,7 @@ func FilterToQuery(f interfaces.Filter) (q bson.M, err error) {
case constants.FilterOpLessThanEqual:
q[key] = bson.M{"$lte": value}
default:
return nil, errors.ErrorFilterInvalidOperation
return nil, errors2.New("invalid operation")
}
}
return q, nil