From fd0f6fc37ab4a672c811ccd1ae6c8d490d0fbc41 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 29 Apr 2021 10:44:46 +0800 Subject: [PATCH] updated code structure --- backend/apps/api.go | 19 +++++++------ backend/apps/base.go | 57 +++------------------------------------ backend/apps/handler.go | 13 +++++---- backend/apps/master.go | 23 ++++++++++++++++ backend/apps/scheduler.go | 25 +++++++++++++++++ backend/apps/utils.go | 37 +++++++++++++++++++++++++ backend/apps/worker.go | 28 +++++++++++++++++++ backend/cmd/api.go | 10 +++---- backend/cmd/handler.go | 12 ++++----- backend/cmd/master.go | 22 +++++++++++++++ backend/cmd/worker.go | 23 ++++++++++++++++ backend/go.sum | 3 +++ 12 files changed, 192 insertions(+), 80 deletions(-) create mode 100644 backend/apps/master.go create mode 100644 backend/apps/scheduler.go create mode 100644 backend/apps/utils.go create mode 100644 backend/apps/worker.go create mode 100644 backend/cmd/master.go create mode 100644 backend/cmd/worker.go diff --git a/backend/apps/api.go b/backend/apps/api.go index 2cfd1cee..a650687f 100644 --- a/backend/apps/api.go +++ b/backend/apps/api.go @@ -18,26 +18,25 @@ import ( ) type Api struct { - BaseApp app *gin.Engine srv *http.Server } -func (app *Api) init() { +func (app *Api) Init() { // initialize config - _ = app.initModule("config", config.InitConfig) + _ = initModule("config", config.InitConfig) // initialize mongo - _ = app.initModule("mongo", mongo.InitMongo) + _ = initModule("mongo", mongo.InitMongo) // initialize redis - _ = app.initModule("redis", redis.InitRedis) + _ = initModule("redis", redis.InitRedis) // initialize model services - _ = app.initModule("mode-services", models.InitModelServices) + _ = initModule("mode-services", models.InitModelServices) // initialize controllers - _ = app.initModule("controllers", controllers.InitControllers) + _ = initModule("controllers", controllers.InitControllers) // initialize middlewares _ = app.initModuleWithApp("middlewares", middlewares.InitMiddlewares) @@ -46,7 +45,7 @@ func (app *Api) init() { _ = app.initModuleWithApp("routes", routes.InitRoutes) } -func (app *Api) run() { +func (app *Api) Run() { host := viper.GetString("server.host") port := viper.GetString("server.port") address := net.JoinHostPort(host, port) @@ -63,7 +62,7 @@ func (app *Api) run() { } } -func (app *Api) stop() { +func (app *Api) Stop() { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() if err := app.srv.Shutdown(ctx); err != nil { @@ -72,7 +71,7 @@ func (app *Api) stop() { } func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) { - return app.initModule(name, func() error { + return initModule(name, func() error { return fn(app.app) }) } diff --git a/backend/apps/base.go b/backend/apps/base.go index 2c84fa13..92770361 100644 --- a/backend/apps/base.go +++ b/backend/apps/base.go @@ -1,58 +1,7 @@ package apps -import ( - "fmt" - "github.com/apex/log" - "github.com/crawlab-team/go-trace" - "os" - "os/signal" - "syscall" -) - type App interface { - init() - run() - stop() -} - -type BaseApp struct { -} - -func (app *BaseApp) init() { - panic("implement me") -} - -func (app *BaseApp) run() { - panic("implement me") -} - -func (app *BaseApp) stop() { - panic("implement me") -} - -func (app *BaseApp) start() { - app.init() - go app.run() - app.waitForStop() - app.stop() -} - -func (app *BaseApp) Start() { - app.start() -} - -func (app *BaseApp) waitForStop() { - quit := make(chan os.Signal, 1) - signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) - <-quit -} - -func (app *BaseApp) initModule(name string, fn func() error) (err error) { - if err := fn(); err != nil { - log.Error(fmt.Sprintf("init %s error: %s", name, err.Error())) - _ = trace.TraceError(err) - panic(err) - } - log.Info(fmt.Sprintf("initialized %s successfully", name)) - return nil + Init() + Run() + Stop() } diff --git a/backend/apps/handler.go b/backend/apps/handler.go index 935143c6..0701a105 100644 --- a/backend/apps/handler.go +++ b/backend/apps/handler.go @@ -2,21 +2,24 @@ package apps import ( "github.com/apex/log" - "github.com/crawlab-team/crawlab-core/services" + "github.com/crawlab-team/crawlab-core/grpc" ) type Handler struct { - BaseApp } -func (app *Handler) init() { - _ = app.initModule("task-service", services.InitTaskService) +func (app *Handler) Init() { + _ = initModule("grpc", grpc.InitGrpcServices) } -func (app *Handler) run() { +func (app *Handler) Run() { log.Info("handler has started") } +func (app *Handler) Stop() { + log.Info("handler has stopped") +} + func NewHandler() *Handler { return &Handler{} } diff --git a/backend/apps/master.go b/backend/apps/master.go new file mode 100644 index 00000000..bfbba2fa --- /dev/null +++ b/backend/apps/master.go @@ -0,0 +1,23 @@ +package apps + +type Master struct { + api *Api +} + +func (app *Master) Init() { + panic("implement me") +} + +func (app *Master) Run() { + panic("implement me") +} + +func (app *Master) Stop() { + panic("implement me") +} + +func NewMaster() *Master { + return &Master{ + api: NewApi(), + } +} diff --git a/backend/apps/scheduler.go b/backend/apps/scheduler.go new file mode 100644 index 00000000..56938767 --- /dev/null +++ b/backend/apps/scheduler.go @@ -0,0 +1,25 @@ +package apps + +import ( + "github.com/apex/log" + "github.com/crawlab-team/crawlab-core/grpc" +) + +type Scheduler struct { +} + +func (app *Scheduler) Init() { + _ = initModule("grpc", grpc.InitGrpcServices) +} + +func (app *Scheduler) Run() { + log.Info("scheduler has started") +} + +func (app *Scheduler) Stop() { + log.Info("scheduler has stopped") +} + +func NewScheduler() *Scheduler { + return &Scheduler{} +} diff --git a/backend/apps/utils.go b/backend/apps/utils.go new file mode 100644 index 00000000..17987882 --- /dev/null +++ b/backend/apps/utils.go @@ -0,0 +1,37 @@ +package apps + +import ( + "fmt" + "github.com/apex/log" + "github.com/crawlab-team/go-trace" + "os" + "os/signal" + "syscall" +) + +func Start(app App) { + start(app) +} + +func start(app App) { + app.Init() + go app.Run() + waitForStop() + app.Stop() +} + +func waitForStop() { + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit +} + +func initModule(name string, fn func() error) (err error) { + if err := fn(); err != nil { + log.Error(fmt.Sprintf("init %s error: %s", name, err.Error())) + _ = trace.TraceError(err) + panic(err) + } + log.Info(fmt.Sprintf("initialized %s successfully", name)) + return nil +} diff --git a/backend/apps/worker.go b/backend/apps/worker.go new file mode 100644 index 00000000..db1ccf09 --- /dev/null +++ b/backend/apps/worker.go @@ -0,0 +1,28 @@ +package apps + +import ( + "github.com/apex/log" + "github.com/crawlab-team/crawlab-core/grpc" +) + +type Worker struct { + handler *Handler +} + +func (app *Worker) Init() { + _ = initModule("grpc", grpc.InitGrpcServices) +} + +func (app *Worker) Run() { + log.Info("worker has started") +} + +func (app *Worker) Stop() { + log.Info("worker has stopped") +} + +func NewWorker() *Worker { + return &Worker{ + handler: NewHandler(), + } +} diff --git a/backend/cmd/api.go b/backend/cmd/api.go index fd265273..931bdbe9 100644 --- a/backend/cmd/api.go +++ b/backend/cmd/api.go @@ -10,12 +10,12 @@ func init() { } var apiCmd = &cobra.Command{ - Use: "api", - Short: "Start API server", - Long: `Start API server of Crawlab which serves data to frontend`, + Use: "api", + Aliases: []string{"A"}, + Short: "Start API server", + Long: `Start API server of Crawlab which serves data to frontend`, Run: func(cmd *cobra.Command, args []string) { api := apps.NewApi() - api.Init() - api.Run() + apps.Start(api) }, } diff --git a/backend/cmd/handler.go b/backend/cmd/handler.go index 885d1281..67abf39f 100644 --- a/backend/cmd/handler.go +++ b/backend/cmd/handler.go @@ -10,13 +10,13 @@ func init() { } var handlerCmd = &cobra.Command{ - Use: "handler", - Short: "Start task handler", - Long: `Start task handler service (worker) of Crawlab -which runs tasks assigned by master node`, + Use: "handler", + Aliases: []string{"H"}, + Short: "Start handler", + Long: `Start a handler instance of Crawlab +which runs tasks with given parameters`, Run: func(cmd *cobra.Command, args []string) { handler := apps.NewHandler() - handler.Init() - handler.Run() + apps.Start(handler) }, } diff --git a/backend/cmd/master.go b/backend/cmd/master.go new file mode 100644 index 00000000..91ba5584 --- /dev/null +++ b/backend/cmd/master.go @@ -0,0 +1,22 @@ +package cmd + +import ( + "crawlab/apps" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(masterCmd) +} + +var masterCmd = &cobra.Command{ + Use: "master", + Aliases: []string{"M"}, + Short: "Start master", + Long: `Start a master instance of Crawlab +which runs api and assign tasks to worker nodes`, + Run: func(cmd *cobra.Command, args []string) { + master := apps.NewMaster() + apps.Start(master) + }, +} diff --git a/backend/cmd/worker.go b/backend/cmd/worker.go new file mode 100644 index 00000000..30e85566 --- /dev/null +++ b/backend/cmd/worker.go @@ -0,0 +1,23 @@ +package cmd + +import ( + "crawlab/apps" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(workerCmd) +} + +var workerCmd = &cobra.Command{ + Use: "worker", + Aliases: []string{"W"}, + Short: "Start worker", + Long: `Start a worker instance of Crawlab +serving in the worker node and executes tasks +assigned by the master node`, + Run: func(cmd *cobra.Command, args []string) { + worker := apps.NewWorker() + apps.Start(worker) + }, +} diff --git a/backend/go.sum b/backend/go.sum index 236692df..ba4f6b7f 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -208,6 +208,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/gorilla/websocket v1.0.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -654,6 +655,7 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d h1:HV9Z9qMhQEsdlvxNFELgQ11RkMzO3CMkjEySjCtuLes= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -661,6 +663,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=