diff --git a/backend/apps/interfaces.go b/backend/apps/interfaces.go index 8a96450e..dbdd24fd 100644 --- a/backend/apps/interfaces.go +++ b/backend/apps/interfaces.go @@ -9,8 +9,17 @@ type App interface { Stop() } -type MasterApp interface { +type NodeApp interface { App interfaces.WithConfigPath + SetGrpcAddress(address interfaces.Address) +} + +type MasterApp interface { + NodeApp SetRunOnMaster(ok bool) } + +type WorkerApp interface { + NodeApp +} diff --git a/backend/apps/master.go b/backend/apps/master.go index 8b222c6b..1fc86345 100644 --- a/backend/apps/master.go +++ b/backend/apps/master.go @@ -10,6 +10,7 @@ import ( type Master struct { // settings runOnMaster bool + grpcAddress interfaces.Address // dependencies interfaces.WithConfigPath @@ -22,6 +23,10 @@ type Master struct { quit chan int } +func (app *Master) SetGrpcAddress(address interfaces.Address) { + app.grpcAddress = address +} + func (app *Master) SetRunOnMaster(ok bool) { app.runOnMaster = ok } @@ -56,9 +61,15 @@ func NewMaster(opts ...MasterOption) (app MasterApp) { opt(m) } + // service options + var svcOpts []service.Option + if m.grpcAddress != nil { + svcOpts = append(svcOpts, service.WithAddress(m.grpcAddress)) + } + // dependency injection c := dig.New() - if err := c.Provide(service.ProvideMasterService(m.GetConfigPath())); err != nil { + if err := c.Provide(service.ProvideMasterService(m.GetConfigPath(), svcOpts...)); err != nil { panic(err) } if err := c.Invoke(func(masterSvc interfaces.NodeMasterService) { diff --git a/backend/apps/options.go b/backend/apps/options.go index 80b5bdc1..70280058 100644 --- a/backend/apps/options.go +++ b/backend/apps/options.go @@ -1,15 +1,37 @@ package apps +import "github.com/crawlab-team/crawlab-core/interfaces" + type MasterOption func(app MasterApp) -func WithConfigPath(path string) MasterOption { +func WithMasterConfigPath(path string) MasterOption { return func(app MasterApp) { app.SetConfigPath(path) } } +func WithMasterGrpcAddress(address interfaces.Address) MasterOption { + return func(app MasterApp) { + app.SetGrpcAddress(address) + } +} + func WithRunOnMaster(ok bool) MasterOption { return func(app MasterApp) { app.SetRunOnMaster(ok) } } + +type WorkerOption func(app WorkerApp) + +func WithWorkerConfigPath(path string) WorkerOption { + return func(app WorkerApp) { + app.SetConfigPath(path) + } +} + +func WithWorkerGrpcAddress(address interfaces.Address) WorkerOption { + return func(app WorkerApp) { + app.SetGrpcAddress(address) + } +} diff --git a/backend/apps/worker.go b/backend/apps/worker.go index a3fe5d83..7d1200d7 100644 --- a/backend/apps/worker.go +++ b/backend/apps/worker.go @@ -1,16 +1,33 @@ package apps +import ( + "github.com/crawlab-team/crawlab-core/config" + "github.com/crawlab-team/crawlab-core/interfaces" + "github.com/crawlab-team/crawlab-core/node/service" + "go.uber.org/dig" +) + type Worker struct { - handler *Handler - quit chan int + // settings + grpcAddress interfaces.Address + + // dependencies + interfaces.WithConfigPath + workerSvc interfaces.NodeWorkerService + + // internals + quit chan int +} + +func (app *Worker) SetGrpcAddress(address interfaces.Address) { + app.grpcAddress = address } func (app *Worker) Init() { - initApp("handler", app.handler) // handler } func (app *Worker) Start() { - go app.handler.Start() + go app.workerSvc.Start() } func (app *Worker) Wait() { @@ -18,14 +35,37 @@ func (app *Worker) Wait() { } func (app *Worker) Stop() { - app.handler.Stop() - app.quit <- 1 } -func NewWorker() *Worker { - return &Worker{ - handler: NewHandler(), - quit: make(chan int, 1), +func NewWorker(opts ...WorkerOption) (app *Worker) { + // worker + w := &Worker{ + WithConfigPath: config.NewConfigPathService(), + quit: make(chan int, 1), } + + // apply options + for _, opt := range opts { + opt(w) + } + + // service options + var svcOpts []service.Option + if w.grpcAddress != nil { + svcOpts = append(svcOpts, service.WithAddress(w.grpcAddress)) + } + + // dependency injection + c := dig.New() + if err := c.Provide(service.ProvideWorkerService(w.GetConfigPath(), svcOpts...)); err != nil { + panic(err) + } + if err := c.Invoke(func(workerSvc interfaces.NodeWorkerService) { + w.workerSvc = workerSvc + }); err != nil { + panic(err) + } + + return w } diff --git a/backend/cmd/master.go b/backend/cmd/master.go index 065d37b8..2e739d29 100644 --- a/backend/cmd/master.go +++ b/backend/cmd/master.go @@ -2,11 +2,26 @@ package cmd import ( "crawlab/apps" + "fmt" + "github.com/crawlab-team/crawlab-core/entity" "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var ( + runOnMaster bool + masterConfigPath string + masterGrpcAddress string ) func init() { rootCmd.AddCommand(masterCmd) + + masterCmd.PersistentFlags().StringVarP(&masterConfigPath, "config-path", "c", "", "Config path of master node") + _ = viper.BindPFlag("configPath", masterCmd.PersistentFlags().Lookup("configPath")) + + masterCmd.PersistentFlags().StringVarP(&masterGrpcAddress, "grpc-address", "g", "", "gRPC address of master node") + _ = viper.BindPFlag("grpcAddress", masterCmd.PersistentFlags().Lookup("grpcAddress")) } var masterCmd = &cobra.Command{ @@ -16,9 +31,26 @@ var masterCmd = &cobra.Command{ Long: `Start a master instance of Crawlab which runs api and assign tasks to worker nodes`, Run: func(cmd *cobra.Command, args []string) { - master := apps.NewMaster( - apps.WithRunOnMaster(runOnMaster), - ) + // options + var opts []apps.MasterOption + if masterConfigPath != "" { + opts = append(opts, apps.WithMasterConfigPath(masterConfigPath)) + viper.Set("config.path", masterConfigPath) + } + opts = append(opts, apps.WithRunOnMaster(runOnMaster)) + if masterGrpcAddress != "" { + address, err := entity.NewAddressFromString(masterGrpcAddress) + if err != nil { + fmt.Println(fmt.Sprintf("invalid grpc-address: %s", masterGrpcAddress)) + } + opts = append(opts, apps.WithMasterGrpcAddress(address)) + viper.Set("grpc.client.address", masterGrpcAddress) + } + + // app + master := apps.NewMaster(opts...) + + // start apps.Start(master) }, } diff --git a/backend/cmd/root.go b/backend/cmd/root.go index aa53dc1b..067d4d74 100644 --- a/backend/cmd/root.go +++ b/backend/cmd/root.go @@ -10,8 +10,7 @@ import ( var ( // Used for flags. - cfgFile string - runOnMaster bool + cfgFile string rootCmd = &cobra.Command{ Use: "crawlab", @@ -30,9 +29,6 @@ func Execute() error { func init() { cobra.OnInitialize(initConfig) - - rootCmd.PersistentFlags().BoolVar(&runOnMaster, "runOnMaster", false, "Whether to run tasks on master node (default: false)") - _ = viper.BindPFlag("runOnMaster", rootCmd.PersistentFlags().Lookup("runOnMaster")) } func initConfig() { diff --git a/backend/cmd/worker.go b/backend/cmd/worker.go index 30e85566..93a6ca1d 100644 --- a/backend/cmd/worker.go +++ b/backend/cmd/worker.go @@ -2,11 +2,25 @@ package cmd import ( "crawlab/apps" + "fmt" + "github.com/crawlab-team/crawlab-core/entity" "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var ( + workerConfigPath string + workerGrpcAddress string ) func init() { rootCmd.AddCommand(workerCmd) + + workerCmd.PersistentFlags().StringVarP(&workerConfigPath, "config-path", "c", "", "Config path of worker node") + _ = viper.BindPFlag("configPath", workerCmd.PersistentFlags().Lookup("configPath")) + + workerCmd.PersistentFlags().StringVarP(&workerGrpcAddress, "grpc-address", "g", "", "gRPC address of worker node") + _ = viper.BindPFlag("grpcAddress", workerCmd.PersistentFlags().Lookup("grpcAddress")) } var workerCmd = &cobra.Command{ @@ -17,7 +31,26 @@ var workerCmd = &cobra.Command{ serving in the worker node and executes tasks assigned by the master node`, Run: func(cmd *cobra.Command, args []string) { - worker := apps.NewWorker() - apps.Start(worker) + // options + var opts []apps.WorkerOption + if workerConfigPath != "" { + opts = append(opts, apps.WithWorkerConfigPath(workerConfigPath)) + viper.Set("config.path", workerConfigPath) + } + if workerGrpcAddress != "" { + address, err := entity.NewAddressFromString(workerGrpcAddress) + if err != nil { + fmt.Println(fmt.Sprintf("invalid grpc-address: %s", workerGrpcAddress)) + return + } + opts = append(opts, apps.WithWorkerGrpcAddress(address)) + viper.Set("grpc.client.address", workerGrpcAddress) + } + + // app + master := apps.NewWorker(opts...) + + // start + apps.Start(master) }, } diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 8f69d28f..302fe70f 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -61,4 +61,10 @@ notification: senderIdentity: '' smtp: user: '' - password: '' \ No newline at end of file + password: '' +config: + path: '' +grpc: + client: + address: localhost:9666 + authKey: Crawlab2021! \ No newline at end of file diff --git a/backend/test/config-master.json b/backend/test/config-master.json new file mode 100644 index 00000000..98ead88c --- /dev/null +++ b/backend/test/config-master.json @@ -0,0 +1,10 @@ +{ + "key": "master", + "is_master": true, + "name": "master", + "ip": "", + "mac": "", + "hostname": "", + "description": "", + "auth_key": "Crawlab2021!" +} \ No newline at end of file diff --git a/backend/test/config-worker-invalid-auth-key.json b/backend/test/config-worker-invalid-auth-key.json new file mode 100644 index 00000000..5c724985 --- /dev/null +++ b/backend/test/config-worker-invalid-auth-key.json @@ -0,0 +1,10 @@ +{ + "key": "worker-invalid-auth-key", + "is_master": false, + "name": "worker", + "ip": "", + "mac": "", + "hostname": "", + "description": "", + "auth_key": "invalid-auth-key" +} \ No newline at end of file diff --git a/backend/test/config-worker.json b/backend/test/config-worker.json new file mode 100644 index 00000000..f88c1630 --- /dev/null +++ b/backend/test/config-worker.json @@ -0,0 +1,10 @@ +{ + "key": "worker", + "is_master": false, + "name": "worker", + "ip": "", + "mac": "", + "hostname": "", + "description": "", + "auth_key": "Crawlab2021!" +} \ No newline at end of file