updated code structure

backend/apps/api.go

@@ -18,26 +18,25 @@ import (
 )
 
 type Api struct {
-	BaseApp
 	app *gin.Engine
 	srv *http.Server
 }
 
-func (app *Api) init() {
+func (app *Api) Init() {
 	// initialize config
-	_ = app.initModule("config", config.InitConfig)
+	_ = initModule("config", config.InitConfig)
 
 	// initialize mongo
-	_ = app.initModule("mongo", mongo.InitMongo)
+	_ = initModule("mongo", mongo.InitMongo)
 
 	// initialize redis
-	_ = app.initModule("redis", redis.InitRedis)
+	_ = initModule("redis", redis.InitRedis)
 
 	// initialize model services
-	_ = app.initModule("mode-services", models.InitModelServices)
+	_ = initModule("mode-services", models.InitModelServices)
 
 	// initialize controllers
-	_ = app.initModule("controllers", controllers.InitControllers)
+	_ = initModule("controllers", controllers.InitControllers)
 
 	// initialize middlewares
 	_ = app.initModuleWithApp("middlewares", middlewares.InitMiddlewares)
@@ -46,7 +45,7 @@ func (app *Api) init() {
 	_ = app.initModuleWithApp("routes", routes.InitRoutes)
 }
 
-func (app *Api) run() {
+func (app *Api) Run() {
 	host := viper.GetString("server.host")
 	port := viper.GetString("server.port")
 	address := net.JoinHostPort(host, port)
@@ -63,7 +62,7 @@ func (app *Api) run() {
 	}
 }
 
-func (app *Api) stop() {
+func (app *Api) Stop() {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	if err := app.srv.Shutdown(ctx); err != nil {
@@ -72,7 +71,7 @@ func (app *Api) stop() {
 }
 
 func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) {
-	return app.initModule(name, func() error {
+	return initModule(name, func() error {
 		return fn(app.app)
 	})
 }

backend/apps/base.go

@@ -1,58 +1,7 @@
 package apps
 
-import (
-	"fmt"
-	"github.com/apex/log"
-	"github.com/crawlab-team/go-trace"
-	"os"
-	"os/signal"
-	"syscall"
-)
-
 type App interface {
-	init()
-	run()
-	stop()
-}
-
-type BaseApp struct {
-}
-
-func (app *BaseApp) init() {
-	panic("implement me")
-}
-
-func (app *BaseApp) run() {
-	panic("implement me")
-}
-
-func (app *BaseApp) stop() {
-	panic("implement me")
-}
-
-func (app *BaseApp) start() {
-	app.init()
-	go app.run()
-	app.waitForStop()
-	app.stop()
-}
-
-func (app *BaseApp) Start() {
-	app.start()
-}
-
-func (app *BaseApp) waitForStop() {
-	quit := make(chan os.Signal, 1)
-	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
-	<-quit
-}
-
-func (app *BaseApp) initModule(name string, fn func() error) (err error) {
-	if err := fn(); err != nil {
-		log.Error(fmt.Sprintf("init %s error: %s", name, err.Error()))
-		_ = trace.TraceError(err)
-		panic(err)
-	}
-	log.Info(fmt.Sprintf("initialized %s successfully", name))
-	return nil
+	Init()
+	Run()
+	Stop()
 }

backend/apps/handler.go

@@ -2,21 +2,24 @@ package apps
 
 import (
 	"github.com/apex/log"
-	"github.com/crawlab-team/crawlab-core/services"
+	"github.com/crawlab-team/crawlab-core/grpc"
 )
 
 type Handler struct {
-	BaseApp
 }
 
-func (app *Handler) init() {
-	_ = app.initModule("task-service", services.InitTaskService)
+func (app *Handler) Init() {
+	_ = initModule("grpc", grpc.InitGrpcServices)
 }
 
-func (app *Handler) run() {
+func (app *Handler) Run() {
 	log.Info("handler has started")
 }
 
+func (app *Handler) Stop() {
+	log.Info("handler has stopped")
+}
+
 func NewHandler() *Handler {
 	return &Handler{}
 }

backend/apps/master.go

@@ -0,0 +1,23 @@
+package apps
+
+type Master struct {
+	api *Api
+}
+
+func (app *Master) Init() {
+	panic("implement me")
+}
+
+func (app *Master) Run() {
+	panic("implement me")
+}
+
+func (app *Master) Stop() {
+	panic("implement me")
+}
+
+func NewMaster() *Master {
+	return &Master{
+		api: NewApi(),
+	}
+}

backend/apps/scheduler.go

@@ -0,0 +1,25 @@
+package apps
+
+import (
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab-core/grpc"
+)
+
+type Scheduler struct {
+}
+
+func (app *Scheduler) Init() {
+	_ = initModule("grpc", grpc.InitGrpcServices)
+}
+
+func (app *Scheduler) Run() {
+	log.Info("scheduler has started")
+}
+
+func (app *Scheduler) Stop() {
+	log.Info("scheduler has stopped")
+}
+
+func NewScheduler() *Scheduler {
+	return &Scheduler{}
+}

backend/apps/utils.go

@@ -0,0 +1,37 @@
+package apps
+
+import (
+	"fmt"
+	"github.com/apex/log"
+	"github.com/crawlab-team/go-trace"
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+func Start(app App) {
+	start(app)
+}
+
+func start(app App) {
+	app.Init()
+	go app.Run()
+	waitForStop()
+	app.Stop()
+}
+
+func waitForStop() {
+	quit := make(chan os.Signal, 1)
+	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
+	<-quit
+}
+
+func initModule(name string, fn func() error) (err error) {
+	if err := fn(); err != nil {
+		log.Error(fmt.Sprintf("init %s error: %s", name, err.Error()))
+		_ = trace.TraceError(err)
+		panic(err)
+	}
+	log.Info(fmt.Sprintf("initialized %s successfully", name))
+	return nil
+}

backend/apps/worker.go

@@ -0,0 +1,28 @@
+package apps
+
+import (
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab-core/grpc"
+)
+
+type Worker struct {
+	handler *Handler
+}
+
+func (app *Worker) Init() {
+	_ = initModule("grpc", grpc.InitGrpcServices)
+}
+
+func (app *Worker) Run() {
+	log.Info("worker has started")
+}
+
+func (app *Worker) Stop() {
+	log.Info("worker has stopped")
+}
+
+func NewWorker() *Worker {
+	return &Worker{
+		handler: NewHandler(),
+	}
+}

backend/cmd/api.go

@@ -10,12 +10,12 @@ func init() {
 }
 
 var apiCmd = &cobra.Command{
-	Use:   "api",
-	Short: "Start API server",
-	Long:  `Start API server of Crawlab which serves data to frontend`,
+	Use:     "api",
+	Aliases: []string{"A"},
+	Short:   "Start API server",
+	Long:    `Start API server of Crawlab which serves data to frontend`,
 	Run: func(cmd *cobra.Command, args []string) {
 		api := apps.NewApi()
-		api.Init()
-		api.Run()
+		apps.Start(api)
 	},
 }

backend/cmd/handler.go

@@ -10,13 +10,13 @@ func init() {
 }
 
 var handlerCmd = &cobra.Command{
-	Use:   "handler",
-	Short: "Start task handler",
-	Long: `Start task handler service (worker) of Crawlab 
-which runs tasks assigned by master node`,
+	Use:     "handler",
+	Aliases: []string{"H"},
+	Short:   "Start handler",
+	Long: `Start a handler instance of Crawlab 
+which runs tasks with given parameters`,
 	Run: func(cmd *cobra.Command, args []string) {
 		handler := apps.NewHandler()
-		handler.Init()
-		handler.Run()
+		apps.Start(handler)
 	},
 }

backend/cmd/master.go

@@ -0,0 +1,22 @@
+package cmd
+
+import (
+	"crawlab/apps"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(masterCmd)
+}
+
+var masterCmd = &cobra.Command{
+	Use:     "master",
+	Aliases: []string{"M"},
+	Short:   "Start master",
+	Long: `Start a master instance of Crawlab 
+which runs api and assign tasks to worker nodes`,
+	Run: func(cmd *cobra.Command, args []string) {
+		master := apps.NewMaster()
+		apps.Start(master)
+	},
+}

backend/cmd/worker.go

@@ -0,0 +1,23 @@
+package cmd
+
+import (
+	"crawlab/apps"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(workerCmd)
+}
+
+var workerCmd = &cobra.Command{
+	Use:     "worker",
+	Aliases: []string{"W"},
+	Short:   "Start worker",
+	Long: `Start a worker instance of Crawlab 
+serving in the worker node and executes tasks
+assigned by the master node`,
+	Run: func(cmd *cobra.Command, args []string) {
+		worker := apps.NewWorker()
+		apps.Start(worker)
+	},
+}

backend/go.sum

@@ -208,6 +208,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR
 github.com/gorilla/websocket v1.0.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
@@ -654,6 +655,7 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
 google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d h1:HV9Z9qMhQEsdlvxNFELgQ11RkMzO3CMkjEySjCtuLes=
 google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@@ -661,6 +663,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI=
 google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=