git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix: test case issue

Browse Source
This commit is contained in:
Marvin Zhang
2024-11-19 15:53:40 +08:00
parent d436087404
commit 47cf368f26
16 changed files with 102 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
core/node/config/config.go
View File

@@ -1,68 +1,15 @@
package config
import (
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/spf13/viper"
)
type Config entity.NodeInfo
type Options struct {
Key string
Name string
IsMaster bool
AuthKey string
MaxRunners int
}
var DefaultMaxRunner = 8
var DefaultConfigOptions = &Options{
Key: utils.NewUUIDString(),
IsMaster: utils.IsMaster(),
AuthKey: constants.DefaultGrpcAuthKey,
MaxRunners: 0,
}
func NewConfig(opts *Options) (cfg *Config) {
if opts == nil {
opts = DefaultConfigOptions
}
if opts.Key == "" {
if viper.GetString("node.key") != "" {
opts.Key = viper.GetString("node.key")
} else {
opts.Key = utils.NewUUIDString()
}
}
if opts.Name == "" {
if viper.GetString("node.name") != "" {
opts.Name = viper.GetString("node.name")
} else {
opts.Name = opts.Key
}
}
if opts.AuthKey == "" {
if viper.GetString("grpc.authKey") != "" {
opts.AuthKey = viper.GetString("grpc.authKey")
} else {
opts.AuthKey = constants.DefaultGrpcAuthKey
}
}
if opts.MaxRunners == 0 {
if viper.GetInt("task.handler.maxRunners") != 0 {
opts.MaxRunners = viper.GetInt("task.handler.maxRunners")
} else {
opts.MaxRunners = DefaultMaxRunner
}
}
return &Config{
Key: opts.Key,
Name: opts.Name,
IsMaster: opts.IsMaster,
AuthKey: opts.AuthKey,
MaxRunners: opts.MaxRunners,
func newConfig() (cfg *entity.NodeInfo) {
return &entity.NodeInfo{
Key: utils.GetNodeKey(),
Name: utils.GetNodeName(),
IsMaster: utils.IsMaster(),
MaxRunners: utils.GetNodeMaxRunners(),
}
}

49
core/node/config/config_service.go
View File

@@ -2,7 +2,7 @@ package config
import (
"encoding/json"
"github.com/crawlab-team/crawlab/core/config"
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/utils"
@@ -13,38 +13,42 @@ import (
)
type Service struct {
cfg *Config
path string
cfg *entity.NodeInfo
}
func (svc *Service) Init() (err error) {
metadataConfigPath := utils.GetMetadataConfigPath()
// check config directory path
configDirPath := filepath.Dir(svc.path)
configDirPath := filepath.Dir(metadataConfigPath)
if !utils.Exists(configDirPath) {
if err := os.MkdirAll(configDirPath, os.FileMode(0766)); err != nil {
return trace.TraceError(err)
}
}
if !utils.Exists(svc.path) {
// not exists, set to default config
// and create a config file for persistence
svc.cfg = NewConfig(nil)
if !utils.Exists(metadataConfigPath) {
// not exists, set to default config, and create a config file for persistence
svc.cfg = newConfig()
data, err := json.Marshal(svc.cfg)
if err != nil {
return trace.TraceError(err)
log.Errorf("marshal config error: %v", err)
return err
}
if err := os.WriteFile(svc.path, data, os.FileMode(0766)); err != nil {
return trace.TraceError(err)
if err := os.WriteFile(metadataConfigPath, data, os.FileMode(0766)); err != nil {
log.Errorf("write config file error: %v", err)
return err
}
} else {
// exists, read and set to config
data, err := os.ReadFile(svc.path)
data, err := os.ReadFile(metadataConfigPath)
if err != nil {
return trace.TraceError(err)
log.Errorf("read config file error: %v", err)
return err
}
if err := json.Unmarshal(data, svc.cfg); err != nil {
return trace.TraceError(err)
log.Errorf("unmarshal config error: %v", err)
return err
}
}
@@ -86,27 +90,12 @@ func (svc *Service) GetMaxRunners() (res int) {
return svc.cfg.MaxRunners
}
func (svc *Service) GetConfigPath() (path string) {
return svc.path
}
func (svc *Service) SetConfigPath(path string) {
svc.path = path
}
func newNodeConfigService() (svc2 interfaces.NodeConfigService, err error) {
// cfg
cfg := NewConfig(nil)
// config service
svc := &Service{
cfg: cfg,
cfg: newConfig(),
}
// normalize config path
cfgPath := config.GetConfigPath()
svc.SetConfigPath(cfgPath)
// init
if err := svc.Init(); err != nil {
return nil, err

42
core/node/service/master_service.go
View File

@@ -4,7 +4,6 @@ import (
"errors"
"github.com/apex/log"
"github.com/cenkalti/backoff/v4"
config2 "github.com/crawlab-team/crawlab/core/config"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/grpc/server"
"github.com/crawlab-team/crawlab/core/interfaces"
@@ -37,10 +36,7 @@ type MasterService struct {
systemSvc *system.Service
// settings
cfgPath string
address interfaces.Address
monitorInterval time.Duration
stopOnError bool
}
func (svc *MasterService) Start() {
@@ -83,11 +79,11 @@ func (svc *MasterService) Wait() {
func (svc *MasterService) Stop() {
_ = svc.server.Stop()
svc.taskHandlerSvc.Stop()
log.Infof("master[%s] service has stopped", svc.GetConfigService().GetNodeKey())
log.Infof("master[%s] service has stopped", svc.cfgSvc.GetNodeKey())
}
func (svc *MasterService) Monitor() {
log.Infof("master[%s] monitoring started", svc.GetConfigService().GetNodeKey())
log.Infof("master[%s] monitoring started", svc.cfgSvc.GetNodeKey())
// ticker
ticker := time.NewTicker(svc.monitorInterval)
@@ -96,12 +92,7 @@ func (svc *MasterService) Monitor() {
// monitor
err := svc.monitor()
if err != nil {
trace.PrintError(err)
if svc.stopOnError {
log.Errorf("master[%s] monitor error, now stopping...", svc.GetConfigService().GetNodeKey())
svc.Stop()
return
}
log.Errorf("master[%s] monitor error: %v", svc.cfgSvc.GetNodeKey(), err)
}
// wait
@@ -109,25 +100,10 @@ func (svc *MasterService) Monitor() {
}
}
func (svc *MasterService) GetConfigService() (cfgSvc interfaces.NodeConfigService) {
return svc.cfgSvc
}
func (svc *MasterService) GetConfigPath() (path string) {
return svc.cfgPath
}
func (svc *MasterService) SetConfigPath(path string) {
svc.cfgPath = path
}
func (svc *MasterService) SetMonitorInterval(duration time.Duration) {
svc.monitorInterval = duration
}
func (svc *MasterService) Register() (err error) {
nodeKey := svc.GetConfigService().GetNodeKey()
nodeName := svc.GetConfigService().GetNodeName()
nodeKey := svc.cfgSvc.GetNodeKey()
nodeName := svc.cfgSvc.GetNodeName()
nodeMaxRunners := svc.cfgSvc.GetMaxRunners()
node, err := service.NewModelService[models.Node]().GetOne(bson.M{"key": nodeKey}, nil)
if err != nil && err.Error() == mongo2.ErrNoDocuments.Error() {
// not exists
@@ -135,7 +111,7 @@ func (svc *MasterService) Register() (err error) {
node := models.Node{
Key: nodeKey,
Name: nodeName,
MaxRunners: config.DefaultConfigOptions.MaxRunners,
MaxRunners: nodeMaxRunners,
IsMaster: true,
Status: constants.NodeStatusOnline,
Enabled: true,
@@ -233,7 +209,7 @@ func (svc *MasterService) getAllWorkerNodes() (nodes []models.Node, err error) {
}
func (svc *MasterService) updateMasterNodeStatus() (err error) {
nodeKey := svc.GetConfigService().GetNodeKey()
nodeKey := svc.cfgSvc.GetNodeKey()
node, err := service.NewModelService[models.Node]().GetOne(bson.M{"key": nodeKey}, nil)
if err != nil {
return err
@@ -318,10 +294,8 @@ func (svc *MasterService) sendNotification(node *models.Node) {
func newMasterService() *MasterService {
return &MasterService{
cfgPath: config2.GetConfigPath(),
cfgSvc: config.GetNodeConfigService(),
monitorInterval: 15 * time.Second,
stopOnError: false,
server: server.GetGrpcServer(),
taskSchedulerSvc: scheduler.GetTaskSchedulerService(),
taskHandlerSvc: handler.GetTaskHandlerService(),

11
core/node/service/worker_service.go
View File

@@ -9,7 +9,6 @@ import (
"github.com/apex/log"
"github.com/cenkalti/backoff/v4"
"github.com/crawlab-team/crawlab/core/config"
"github.com/crawlab-team/crawlab/core/grpc/client"
"github.com/crawlab-team/crawlab/core/interfaces"
client2 "github.com/crawlab-team/crawlab/core/models/client"
@@ -28,7 +27,6 @@ type WorkerService struct {
handlerSvc *handler.Service
// settings
cfgPath string
address interfaces.Address
heartbeatInterval time.Duration
@@ -116,14 +114,6 @@ func (svc *WorkerService) GetConfigService() (cfgSvc interfaces.NodeConfigServic
return svc.cfgSvc
}
func (svc *WorkerService) GetConfigPath() (path string) {
return svc.cfgPath
}
func (svc *WorkerService) SetConfigPath(path string) {
svc.cfgPath = path
}
func (svc *WorkerService) subscribe() {
// Configure exponential backoff
b := backoff.NewExponentialBackOff()
@@ -195,7 +185,6 @@ func (svc *WorkerService) sendHeartbeat() {
func newWorkerService() *WorkerService {
return &WorkerService{
cfgPath: config.GetConfigPath(),
heartbeatInterval: 15 * time.Second,
cfgSvc: nodeconfig.GetNodeConfigService(),
client: client.GetGrpcClient(),