feat: added modules

core/node/config/config.go

@@ -0,0 +1,68 @@
+package config
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/spf13/viper"
+)
+
+type Config entity.NodeInfo
+
+type Options struct {
+	Key        string
+	Name       string
+	IsMaster   bool
+	AuthKey    string
+	MaxRunners int
+}
+
+var DefaultMaxRunner = 8
+
+var DefaultConfigOptions = &Options{
+	Key:        utils.NewUUIDString(),
+	IsMaster:   utils.IsMaster(),
+	AuthKey:    constants.DefaultGrpcAuthKey,
+	MaxRunners: 0,
+}
+
+func NewConfig(opts *Options) (cfg *Config) {
+	if opts == nil {
+		opts = DefaultConfigOptions
+	}
+	if opts.Key == "" {
+		if viper.GetString("node.key") != "" {
+			opts.Key = viper.GetString("node.key")
+		} else {
+			opts.Key = utils.NewUUIDString()
+		}
+	}
+	if opts.Name == "" {
+		if viper.GetString("node.name") != "" {
+			opts.Name = viper.GetString("node.name")
+		} else {
+			opts.Name = opts.Key
+		}
+	}
+	if opts.AuthKey == "" {
+		if viper.GetString("grpc.authKey") != "" {
+			opts.AuthKey = viper.GetString("grpc.authKey")
+		} else {
+			opts.AuthKey = constants.DefaultGrpcAuthKey
+		}
+	}
+	if opts.MaxRunners == 0 {
+		if viper.GetInt("task.handler.maxRunners") != 0 {
+			opts.MaxRunners = viper.GetInt("task.handler.maxRunners")
+		} else {
+			opts.MaxRunners = DefaultMaxRunner
+		}
+	}
+	return &Config{
+		Key:        opts.Key,
+		Name:       opts.Name,
+		IsMaster:   opts.IsMaster,
+		AuthKey:    opts.AuthKey,
+		MaxRunners: opts.MaxRunners,
+	}
+}

core/node/config/config_service.go

@@ -0,0 +1,130 @@
+package config
+
+import (
+	"encoding/json"
+	"github.com/crawlab-team/crawlab/core/config"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"os"
+	"path"
+)
+
+type Service struct {
+	cfg  *Config
+	path string
+}
+
+func (svc *Service) Init() (err error) {
+	// check config directory path
+	configDirPath := path.Dir(svc.path)
+	if !utils.Exists(configDirPath) {
+		if err := os.MkdirAll(configDirPath, os.FileMode(0766)); err != nil {
+			return trace.TraceError(err)
+		}
+	}
+
+	if !utils.Exists(svc.path) {
+		// not exists, set to default config
+		// and create a config file for persistence
+		svc.cfg = NewConfig(nil)
+		data, err := json.Marshal(svc.cfg)
+		if err != nil {
+			return trace.TraceError(err)
+		}
+		if err := os.WriteFile(svc.path, data, os.FileMode(0766)); err != nil {
+			return trace.TraceError(err)
+		}
+	} else {
+		// exists, read and set to config
+		data, err := os.ReadFile(svc.path)
+		if err != nil {
+			return trace.TraceError(err)
+		}
+		if err := json.Unmarshal(data, svc.cfg); err != nil {
+			return trace.TraceError(err)
+		}
+	}
+
+	return nil
+}
+
+func (svc *Service) Reload() (err error) {
+	return svc.Init()
+}
+
+func (svc *Service) GetBasicNodeInfo() (res interfaces.Entity) {
+	return &entity.NodeInfo{
+		Key:        svc.GetNodeKey(),
+		Name:       svc.GetNodeName(),
+		IsMaster:   svc.IsMaster(),
+		AuthKey:    svc.GetAuthKey(),
+		MaxRunners: svc.GetMaxRunners(),
+	}
+}
+
+func (svc *Service) GetNodeKey() (res string) {
+	return svc.cfg.Key
+}
+
+func (svc *Service) GetNodeName() (res string) {
+	return svc.cfg.Name
+}
+
+func (svc *Service) IsMaster() (res bool) {
+	return svc.cfg.IsMaster
+}
+
+func (svc *Service) GetAuthKey() (res string) {
+	return svc.cfg.AuthKey
+}
+
+func (svc *Service) GetMaxRunners() (res int) {
+	return svc.cfg.MaxRunners
+}
+
+func (svc *Service) GetConfigPath() (path string) {
+	return svc.path
+}
+
+func (svc *Service) SetConfigPath(path string) {
+	svc.path = path
+}
+
+func NewNodeConfigService() (svc2 interfaces.NodeConfigService, err error) {
+	// cfg
+	cfg := NewConfig(nil)
+
+	// config service
+	svc := &Service{
+		cfg: cfg,
+	}
+
+	// normalize config path
+	cfgPath := config.GetConfigPath()
+	svc.SetConfigPath(cfgPath)
+
+	// init
+	if err := svc.Init(); err != nil {
+		return nil, err
+	}
+
+	return svc, nil
+}
+
+var _service interfaces.NodeConfigService
+
+func GetNodeConfigService() interfaces.NodeConfigService {
+	if _service != nil {
+		return _service
+	}
+
+	var err error
+	_service, err = NewNodeConfigService()
+	if err != nil {
+		panic(err)
+	}
+
+	return _service
+}

core/node/config/options.go

@@ -0,0 +1,13 @@
+package config
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+)
+
+type Option func(svc interfaces.NodeConfigService)
+
+func WithConfigPath(path string) Option {
+	return func(svc interfaces.NodeConfigService) {
+		svc.SetConfigPath(path)
+	}
+}